├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── analyzer.go ├── bootstrap ├── css │ ├── bootstrap-responsive.min.css │ ├── bootstrap.min.css │ └── custom.css ├── img │ ├── glyphicons-halflings-white.png │ └── glyphicons-halflings.png └── js │ └── bootstrap.min.js ├── config.go ├── db.go ├── helpers-md5.go ├── helpers.go ├── hlsproberc-mintest ├── html ├── ccchart-min.js ├── index.html └── menu.html ├── http-api.go ├── http-client.go ├── logger.go ├── monitor-prober.go ├── monitor.go ├── pics ├── favicon.ico └── logo-64.png ├── reports.go ├── sample-config ├── source-loader.go ├── stats.go ├── streamsurfer.go ├── streamsurfer_test.go ├── structure.go ├── templates.go ├── templates ├── activity-index.tmpl ├── index.tmpl ├── page-footer.tmpl ├── page-header.tmpl ├── report-index.tmpl ├── report-stream-history.tmpl ├── report-stream-info.tmpl └── top-page-menu.tmpl ├── webui-report.go └── zabbix.go /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 3 | *.o 4 | *.a 5 | *.so 6 | 7 | # Folders 8 | _obj 9 | _test 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. {http://fsf.org/} 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see {http://www.gnu.org/licenses/}. 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | hlsprobe2 Copyright (C) 2013 Alexander I.Grafov aka Axel 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | {http://www.gnu.org/licenses/}. 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | {http://www.gnu.org/philosophy/why-not-lgpl.html}. 675 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export GOPATH=/home/axel/go 2 | SOURCES=streamsurfer.go structure.go config.go monitor.go monitor-prober.go http-client.go http-api.go stats.go logger.go zabbix.go helpers.go helpers-md5.go templates.go webui-report.go analyzer.go db.go # reports.go source-loader.go 3 | HTML=html/*.html 4 | LDFLAGS="-X main.build_date `date -u +%Y%m%d%H%M%S`" 5 | 6 | streamsurfer: $(SOURCES) $(HTML) 7 | go build -ldflags $(LDFLAGS) $(SOURCES) 8 | gcc: $(SOURCES) $(HTML) 9 | go build -o streamsurfer -compiler gccgo -ldflags $(LDFLAGS) $(SOURCES) 10 | gccbuild: gcc 11 | build: streamsurfer 12 | paxbuild: streamsurfer 13 | # use sudo or run as root 14 | paxctl -cm streamsurfer 15 | run: $(SOURCES) 16 | go run $(SOURCES) 17 | install: streamsurfer 18 | # use sudo or run as root 19 | strip streamsurfer 20 | cp -a streamsurfer /usr/local/bin 21 | clean: 22 | rm streamsurfer 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **As I moved away from videostreaming development I not more interested in this project. So I leave code as is in unfinished state. I'll continue to maintain [m3u8](https://github.com/grafov/m3u8) library as it is used in some other projects.** 2 | 3 | Stream Surfer 4 | ============= 5 | 6 | **Current state: code broken. Don't use!** 7 | 8 | Stream Surfer — probe utiliy for HTTP video streaming. The utility detects errors in 9 | HTTP Streams (Apple HLS now supported) and monitor health of any HTTP resources. It may 10 | be used as regular monitoring tool and stress testing for mediaservers (and any 11 | HTTP-servers too). 12 | 13 | Features are: 14 | 15 | * parse M3U8-playlists (master and single-bitrate playlists) 16 | * detect bad playlists format (empty playlists, incorrect chunk durations) 17 | * check HTTP response statuses 18 | * collects response time statistics 19 | * webreports to represent collected statistics 20 | * integration with Zabbix (http://zabbix.com) monitoring software 21 | 22 | Planned features: 23 | 24 | * probe more formats beside HLS (planned HDS, DASH, Widevine VOD) 25 | * probe chunks with `mediainfo` utility (from ffmpeg) 26 | * REST HTTP to represent collected data and utility control 27 | * aggregate and analyze statistics from other streamsurfer nodes 28 | * visualize monitoring information in Web UI 29 | * persistent storage for statistics and reports generation 30 | 31 | This software can't be used for HLS playback. 32 | 33 | `streamsurfer` is an furfer development of Python `hlsprobe` (https://github.com/grafov/hlsprobe). 34 | 35 | Install 36 | ------- 37 | 38 | You need Go language (http://golang.org) environment installed. 39 | Then: 40 | 41 | go get github.com/grafov/bcast 42 | go get github.com/grafov/m3u8 43 | go get github.com/gorilla/mux 44 | git clone https://github.com/grafov/streamsurfer 45 | cd streamsurfer 46 | sudo make install 47 | 48 | The code includes Bootstrap 2 (http://getbootstrap.com) library (under Apache License). 49 | It may be packaged with `streamsurfer` due GPLv3 license. 50 | To simplify installation Bootstrap code yet included in `streamsurfer` package. 51 | Later it will be splitted and Bootstrap will be downloaded separately. 52 | 53 | Usage 54 | ----- 55 | 56 | Setup configuration file (copy one of templates from package) and start utility: 57 | 58 | streamsurfer --config=config.yml 59 | 60 | All stream problems logged to error log (`error-log` parameter in the config `params` section). 61 | Web reports available at `localhost:8088` (define listener with `http-api-listen`). 62 | 63 | Similar projects 64 | ---------------- 65 | 66 | * http://code.google.com/p/hls-player 67 | * http://github.com/brookemckim/hlspider 68 | 69 | Build status 70 | ------------ 71 | 72 | [![Build Status](https://travis-ci.org/grafov/hlsprobe2.png?branch=master)](https://travis-ci.org/grafov/hlsprobe2) 73 | 74 | [![Is maintained?](http://stillmaintained.com/grafov/hlsprobe2.png)](http://stillmaintained.com/grafov/hlsprobe2) 75 | -------------------------------------------------------------------------------- /analyzer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | /* TODO 9 | 10 | хранить lastAnalyzed в виде [time, taskId] в базе 11 | при старте problemanalyzer забирает карту lastanalyzed из бд 12 | при проверке канала, если данных lastanalyzed нет, то ставится [now-6h, nil] 13 | в анализаторе проходить по задачам, выбирать макс. errtype 14 | если ошибка, то выставлять флаг ошибки и записывать время старта 15 | идти по проверкам задачи записывая макс.флаг ошибки 16 | при переходе к след. задаче смотреть, если ошибка устранилась, то записывать время окончания 17 | сбрасывать флаг ошибки, записывать диапазон в список 18 | затем проходить по диапазону, выбирая ошибки > минДлительностьДляОтчета → формировать отчет 19 | меньшие периоды, если их > N за М времени рапортовать как сводный отчёт (warning level) 20 | если 0 { 63 | fmt.Printf("conclusion for %s %v\n", key.String(), conclusion) 64 | } 65 | // TODO дополнительно делать анализ для всей группы 66 | } 67 | // TODO RemoveExpiredReports(cfg.ExpireDurationDB) 68 | } 69 | } 70 | 71 | // Report found problems from problem analyzer 72 | // Maybe several methods to report problems of different prirority 73 | func ProblemReporter() { 74 | 75 | } 76 | 77 | func LoadReports() []Report { 78 | return nil 79 | } 80 | 81 | /* 82 | * Private realization 83 | */ 84 | 85 | // Analyze HLS and form report with error states. 86 | // We check for an each check in an each task in results set. Task interpreted as problem if error occured in 87 | // the any single check. Then we aggregate problem tasks into error ranges ([]ErrRange). Report builder then 88 | // analyze error ranges and make reports on them. 89 | func analyzeHLS(key Key, hist []KeepedResult, lastCheck *CheckPoint) (reports []Report) { 90 | var ( 91 | isRangeOpened bool // problem under analyzator cursor 92 | isTaskOK bool = true // statuses for current check and task 93 | start, stop time.Time // start and stop timestamps of error period 94 | prevTid, fromTid, toTid int64 // task id 95 | errlevel ErrType // error level 96 | errorRanges []ErrRange // ranges with error states of the stream 97 | forSave *ErrRange // continious range of failed tasks 98 | ) 99 | 100 | for _, hitem := range hist { 101 | if hitem.Started.Before(lastCheck.Occured) { 102 | continue 103 | } 104 | 105 | if key.Name == "sd_2014_game_of_thrones_04_02_film" { 106 | fmt.Printf("+++ %d %d %d %d %s %d %+v\n", prevTid, hitem.Tid, fromTid, toTid, isTaskOK, errlevel, forSave) 107 | } 108 | 109 | if prevTid > 0 && prevTid != hitem.Tid { // переход задач 110 | if isTaskOK && forSave != nil { // период ошибок кончился 111 | errorRanges = append(errorRanges, *forSave) 112 | forSave = nil 113 | isRangeOpened = false 114 | } else { 115 | isTaskOK = true 116 | } 117 | prevTid = hitem.Tid 118 | } 119 | 120 | if prevTid == 0 { 121 | prevTid = hitem.Tid 122 | } 123 | 124 | if hitem.ErrType > ERROR_LEVEL { 125 | isTaskOK = false 126 | if hitem.ErrType > errlevel { 127 | errlevel = hitem.ErrType 128 | } 129 | if !isRangeOpened { 130 | isRangeOpened = true 131 | fromTid = hitem.Tid 132 | start = hitem.Started 133 | toTid = fromTid 134 | stop = start 135 | } else { 136 | toTid = hitem.Tid 137 | stop = hitem.Started.Add(hitem.Elapsed) 138 | } 139 | forSave = &ErrRange{fromTid, hitem.Tid, start, stop, errlevel} 140 | } 141 | } 142 | 143 | lastCheck = &CheckPoint{toTid, stop, nil} 144 | if isRangeOpened && errlevel > 0 { // период остался незакрыт 145 | errorRanges = append(errorRanges, ErrRange{fromTid, toTid, start, stop, errlevel}) 146 | } 147 | if len(errorRanges) > 0 { 148 | fmt.Printf("err range for %s %v\n", key.String(), errorRanges) 149 | } 150 | // Permanent errors report. Error is permanent if it continued more than 10 minute. 151 | for _, val := range errorRanges { 152 | if val.Discontinued.Sub(val.Occured) > 10*time.Minute { 153 | reports = append(reports, generatePermanentErrorsReport(key, errorRanges, isRangeOpened)) 154 | } 155 | } 156 | return reports 157 | } 158 | 159 | func analyzeHDS(hist []KeepedResult) []Report { 160 | return []Report{} 161 | } 162 | 163 | func analyzeHTTP(hist []KeepedResult) []Report { 164 | return []Report{} 165 | } 166 | 167 | /* 168 | * Report generators 169 | */ 170 | 171 | // Permanent errors report generator 172 | func generatePermanentErrorsReport(key Key, ranges []ErrRange, errorPersists bool) Report { 173 | return Report{Title: "Sample report"} 174 | } 175 | -------------------------------------------------------------------------------- /bootstrap/css/bootstrap-responsive.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap Responsive v2.3.2 3 | * 4 | * Copyright 2013 Twitter, Inc 5 | * Licensed under the Apache License v2.0 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Designed and built with all the love in the world by @mdo and @fat. 9 | */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}@-ms-viewport{width:device-width}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}.visible-print{display:none!important}@media print{.visible-print{display:inherit!important}.hidden-print{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.564102564102564%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .controls-row [class*="span"]+[class*="span"]{margin-left:2.7624309392265194%}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.uneditable-input[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="offset"]:first-child{margin-left:0}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.controls-row [class*="span"]+[class*="span"]{margin-left:0}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade{top:-100px}.modal.fade.in{top:20px}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.media .pull-left,.media .pull-right{display:block;float:none;margin-bottom:10px}.media-object{margin-right:0;margin-left:0}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#777;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .nav>li>a:focus,.nav-collapse .dropdown-menu a:hover,.nav-collapse .dropdown-menu a:focus{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a,.navbar-inverse .nav-collapse .dropdown-menu a{color:#999}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .nav>li>a:focus,.navbar-inverse .nav-collapse .dropdown-menu a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:focus{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:none;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .open>.dropdown-menu{display:block}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111;border-bottom-color:#111}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}} 10 | -------------------------------------------------------------------------------- /bootstrap/css/custom.css: -------------------------------------------------------------------------------- 1 | .table tbody tr.critical > td { 2 | background-color: #f2dede; 3 | } 4 | 5 | .table tbody tr > td.success { 6 | background-color: #dff0d8 !important; 7 | } 8 | 9 | .table tbody tr > td.error { 10 | background-color: #f2dede !important; 11 | } 12 | 13 | .table tbody tr > td.critical { 14 | background-color: #f2dede !important; 15 | } 16 | 17 | .table tbody tr > td.warning { 18 | background-color: #fcf8e3 !important; 19 | } 20 | 21 | .table tbody tr > td.info { 22 | background-color: #d9edf7 !important; 23 | } 24 | 25 | .table-hover tbody tr:hover > td.success { 26 | background-color: #d0e9c6 !important; 27 | } 28 | 29 | .table-hover tbody tr:hover > td.error { 30 | background-color: #ebcccc !important; 31 | } 32 | 33 | .table-hover tbody tr:hover > td.warning { 34 | background-color: #faf2cc !important; 35 | } 36 | 37 | .table-hover tbody tr:hover > td.info { 38 | background-color: #c4e3f3 !important; 39 | } 40 | -------------------------------------------------------------------------------- /bootstrap/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafov/streamsurfer/da3ff432f6da1b01b77856ca877c9ef3741d2b23/bootstrap/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /bootstrap/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafov/streamsurfer/da3ff432f6da1b01b77856ca877c9ef3741d2b23/bootstrap/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /bootstrap/js/bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap.js by @fat & @mdo 3 | * Copyright 2013 Twitter, Inc. 4 | * http://www.apache.org/licenses/LICENSE-2.0.txt 5 | */ 6 | !function(e){"use strict";e(function(){e.support.transition=function(){var e=function(){var e=document.createElement("bootstrap"),t={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"},n;for(n in t)if(e.style[n]!==undefined)return t[n]}();return e&&{end:e}}()})}(window.jQuery),!function(e){"use strict";var t='[data-dismiss="alert"]',n=function(n){e(n).on("click",t,this.close)};n.prototype.close=function(t){function s(){i.trigger("closed").remove()}var n=e(this),r=n.attr("data-target"),i;r||(r=n.attr("href"),r=r&&r.replace(/.*(?=#[^\s]*$)/,"")),i=e(r),t&&t.preventDefault(),i.length||(i=n.hasClass("alert")?n:n.parent()),i.trigger(t=e.Event("close"));if(t.isDefaultPrevented())return;i.removeClass("in"),e.support.transition&&i.hasClass("fade")?i.on(e.support.transition.end,s):s()};var r=e.fn.alert;e.fn.alert=function(t){return this.each(function(){var r=e(this),i=r.data("alert");i||r.data("alert",i=new n(this)),typeof t=="string"&&i[t].call(r)})},e.fn.alert.Constructor=n,e.fn.alert.noConflict=function(){return e.fn.alert=r,this},e(document).on("click.alert.data-api",t,n.prototype.close)}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.button.defaults,n)};t.prototype.setState=function(e){var t="disabled",n=this.$element,r=n.data(),i=n.is("input")?"val":"html";e+="Text",r.resetText||n.data("resetText",n[i]()),n[i](r[e]||this.options[e]),setTimeout(function(){e=="loadingText"?n.addClass(t).attr(t,t):n.removeClass(t).removeAttr(t)},0)},t.prototype.toggle=function(){var e=this.$element.closest('[data-toggle="buttons-radio"]');e&&e.find(".active").removeClass("active"),this.$element.toggleClass("active")};var n=e.fn.button;e.fn.button=function(n){return this.each(function(){var r=e(this),i=r.data("button"),s=typeof n=="object"&&n;i||r.data("button",i=new t(this,s)),n=="toggle"?i.toggle():n&&i.setState(n)})},e.fn.button.defaults={loadingText:"loading..."},e.fn.button.Constructor=t,e.fn.button.noConflict=function(){return e.fn.button=n,this},e(document).on("click.button.data-api","[data-toggle^=button]",function(t){var n=e(t.target);n.hasClass("btn")||(n=n.closest(".btn")),n.button("toggle")})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.$indicators=this.$element.find(".carousel-indicators"),this.options=n,this.options.pause=="hover"&&this.$element.on("mouseenter",e.proxy(this.pause,this)).on("mouseleave",e.proxy(this.cycle,this))};t.prototype={cycle:function(t){return t||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(e.proxy(this.next,this),this.options.interval)),this},getActiveIndex:function(){return this.$active=this.$element.find(".item.active"),this.$items=this.$active.parent().children(),this.$items.index(this.$active)},to:function(t){var n=this.getActiveIndex(),r=this;if(t>this.$items.length-1||t<0)return;return this.sliding?this.$element.one("slid",function(){r.to(t)}):n==t?this.pause().cycle():this.slide(t>n?"next":"prev",e(this.$items[t]))},pause:function(t){return t||(this.paused=!0),this.$element.find(".next, .prev").length&&e.support.transition.end&&(this.$element.trigger(e.support.transition.end),this.cycle(!0)),clearInterval(this.interval),this.interval=null,this},next:function(){if(this.sliding)return;return this.slide("next")},prev:function(){if(this.sliding)return;return this.slide("prev")},slide:function(t,n){var r=this.$element.find(".item.active"),i=n||r[t](),s=this.interval,o=t=="next"?"left":"right",u=t=="next"?"first":"last",a=this,f;this.sliding=!0,s&&this.pause(),i=i.length?i:this.$element.find(".item")[u](),f=e.Event("slide",{relatedTarget:i[0],direction:o});if(i.hasClass("active"))return;this.$indicators.length&&(this.$indicators.find(".active").removeClass("active"),this.$element.one("slid",function(){var t=e(a.$indicators.children()[a.getActiveIndex()]);t&&t.addClass("active")}));if(e.support.transition&&this.$element.hasClass("slide")){this.$element.trigger(f);if(f.isDefaultPrevented())return;i.addClass(t),i[0].offsetWidth,r.addClass(o),i.addClass(o),this.$element.one(e.support.transition.end,function(){i.removeClass([t,o].join(" ")).addClass("active"),r.removeClass(["active",o].join(" ")),a.sliding=!1,setTimeout(function(){a.$element.trigger("slid")},0)})}else{this.$element.trigger(f);if(f.isDefaultPrevented())return;r.removeClass("active"),i.addClass("active"),this.sliding=!1,this.$element.trigger("slid")}return s&&this.cycle(),this}};var n=e.fn.carousel;e.fn.carousel=function(n){return this.each(function(){var r=e(this),i=r.data("carousel"),s=e.extend({},e.fn.carousel.defaults,typeof n=="object"&&n),o=typeof n=="string"?n:s.slide;i||r.data("carousel",i=new t(this,s)),typeof n=="number"?i.to(n):o?i[o]():s.interval&&i.pause().cycle()})},e.fn.carousel.defaults={interval:5e3,pause:"hover"},e.fn.carousel.Constructor=t,e.fn.carousel.noConflict=function(){return e.fn.carousel=n,this},e(document).on("click.carousel.data-api","[data-slide], [data-slide-to]",function(t){var n=e(this),r,i=e(n.attr("data-target")||(r=n.attr("href"))&&r.replace(/.*(?=#[^\s]+$)/,"")),s=e.extend({},i.data(),n.data()),o;i.carousel(s),(o=n.attr("data-slide-to"))&&i.data("carousel").pause().to(o).cycle(),t.preventDefault()})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.collapse.defaults,n),this.options.parent&&(this.$parent=e(this.options.parent)),this.options.toggle&&this.toggle()};t.prototype={constructor:t,dimension:function(){var e=this.$element.hasClass("width");return e?"width":"height"},show:function(){var t,n,r,i;if(this.transitioning||this.$element.hasClass("in"))return;t=this.dimension(),n=e.camelCase(["scroll",t].join("-")),r=this.$parent&&this.$parent.find("> .accordion-group > .in");if(r&&r.length){i=r.data("collapse");if(i&&i.transitioning)return;r.collapse("hide"),i||r.data("collapse",null)}this.$element[t](0),this.transition("addClass",e.Event("show"),"shown"),e.support.transition&&this.$element[t](this.$element[0][n])},hide:function(){var t;if(this.transitioning||!this.$element.hasClass("in"))return;t=this.dimension(),this.reset(this.$element[t]()),this.transition("removeClass",e.Event("hide"),"hidden"),this.$element[t](0)},reset:function(e){var t=this.dimension();return this.$element.removeClass("collapse")[t](e||"auto")[0].offsetWidth,this.$element[e!==null?"addClass":"removeClass"]("collapse"),this},transition:function(t,n,r){var i=this,s=function(){n.type=="show"&&i.reset(),i.transitioning=0,i.$element.trigger(r)};this.$element.trigger(n);if(n.isDefaultPrevented())return;this.transitioning=1,this.$element[t]("in"),e.support.transition&&this.$element.hasClass("collapse")?this.$element.one(e.support.transition.end,s):s()},toggle:function(){this[this.$element.hasClass("in")?"hide":"show"]()}};var n=e.fn.collapse;e.fn.collapse=function(n){return this.each(function(){var r=e(this),i=r.data("collapse"),s=e.extend({},e.fn.collapse.defaults,r.data(),typeof n=="object"&&n);i||r.data("collapse",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.collapse.defaults={toggle:!0},e.fn.collapse.Constructor=t,e.fn.collapse.noConflict=function(){return e.fn.collapse=n,this},e(document).on("click.collapse.data-api","[data-toggle=collapse]",function(t){var n=e(this),r,i=n.attr("data-target")||t.preventDefault()||(r=n.attr("href"))&&r.replace(/.*(?=#[^\s]+$)/,""),s=e(i).data("collapse")?"toggle":n.data();n[e(i).hasClass("in")?"addClass":"removeClass"]("collapsed"),e(i).collapse(s)})}(window.jQuery),!function(e){"use strict";function r(){e(".dropdown-backdrop").remove(),e(t).each(function(){i(e(this)).removeClass("open")})}function i(t){var n=t.attr("data-target"),r;n||(n=t.attr("href"),n=n&&/#/.test(n)&&n.replace(/.*(?=#[^\s]*$)/,"")),r=n&&e(n);if(!r||!r.length)r=t.parent();return r}var t="[data-toggle=dropdown]",n=function(t){var n=e(t).on("click.dropdown.data-api",this.toggle);e("html").on("click.dropdown.data-api",function(){n.parent().removeClass("open")})};n.prototype={constructor:n,toggle:function(t){var n=e(this),s,o;if(n.is(".disabled, :disabled"))return;return s=i(n),o=s.hasClass("open"),r(),o||("ontouchstart"in document.documentElement&&e('
').insertBefore(e(this)).on("click",r),s.toggleClass("open")),n.focus(),!1},keydown:function(n){var r,s,o,u,a,f;if(!/(38|40|27)/.test(n.keyCode))return;r=e(this),n.preventDefault(),n.stopPropagation();if(r.is(".disabled, :disabled"))return;u=i(r),a=u.hasClass("open");if(!a||a&&n.keyCode==27)return n.which==27&&u.find(t).focus(),r.click();s=e("[role=menu] li:not(.divider):visible a",u);if(!s.length)return;f=s.index(s.filter(":focus")),n.keyCode==38&&f>0&&f--,n.keyCode==40&&f').appendTo(document.body),this.$backdrop.click(this.options.backdrop=="static"?e.proxy(this.$element[0].focus,this.$element[0]):e.proxy(this.hide,this)),i&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in");if(!t)return;i?this.$backdrop.one(e.support.transition.end,t):t()}else!this.isShown&&this.$backdrop?(this.$backdrop.removeClass("in"),e.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one(e.support.transition.end,t):t()):t&&t()}};var n=e.fn.modal;e.fn.modal=function(n){return this.each(function(){var r=e(this),i=r.data("modal"),s=e.extend({},e.fn.modal.defaults,r.data(),typeof n=="object"&&n);i||r.data("modal",i=new t(this,s)),typeof n=="string"?i[n]():s.show&&i.show()})},e.fn.modal.defaults={backdrop:!0,keyboard:!0,show:!0},e.fn.modal.Constructor=t,e.fn.modal.noConflict=function(){return e.fn.modal=n,this},e(document).on("click.modal.data-api",'[data-toggle="modal"]',function(t){var n=e(this),r=n.attr("href"),i=e(n.attr("data-target")||r&&r.replace(/.*(?=#[^\s]+$)/,"")),s=i.data("modal")?"toggle":e.extend({remote:!/#/.test(r)&&r},i.data(),n.data());t.preventDefault(),i.modal(s).one("hide",function(){n.focus()})})}(window.jQuery),!function(e){"use strict";var t=function(e,t){this.init("tooltip",e,t)};t.prototype={constructor:t,init:function(t,n,r){var i,s,o,u,a;this.type=t,this.$element=e(n),this.options=this.getOptions(r),this.enabled=!0,o=this.options.trigger.split(" ");for(a=o.length;a--;)u=o[a],u=="click"?this.$element.on("click."+this.type,this.options.selector,e.proxy(this.toggle,this)):u!="manual"&&(i=u=="hover"?"mouseenter":"focus",s=u=="hover"?"mouseleave":"blur",this.$element.on(i+"."+this.type,this.options.selector,e.proxy(this.enter,this)),this.$element.on(s+"."+this.type,this.options.selector,e.proxy(this.leave,this)));this.options.selector?this._options=e.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},getOptions:function(t){return t=e.extend({},e.fn[this.type].defaults,this.$element.data(),t),t.delay&&typeof t.delay=="number"&&(t.delay={show:t.delay,hide:t.delay}),t},enter:function(t){var n=e.fn[this.type].defaults,r={},i;this._options&&e.each(this._options,function(e,t){n[e]!=t&&(r[e]=t)},this),i=e(t.currentTarget)[this.type](r).data(this.type);if(!i.options.delay||!i.options.delay.show)return i.show();clearTimeout(this.timeout),i.hoverState="in",this.timeout=setTimeout(function(){i.hoverState=="in"&&i.show()},i.options.delay.show)},leave:function(t){var n=e(t.currentTarget)[this.type](this._options).data(this.type);this.timeout&&clearTimeout(this.timeout);if(!n.options.delay||!n.options.delay.hide)return n.hide();n.hoverState="out",this.timeout=setTimeout(function(){n.hoverState=="out"&&n.hide()},n.options.delay.hide)},show:function(){var t,n,r,i,s,o,u=e.Event("show");if(this.hasContent()&&this.enabled){this.$element.trigger(u);if(u.isDefaultPrevented())return;t=this.tip(),this.setContent(),this.options.animation&&t.addClass("fade"),s=typeof this.options.placement=="function"?this.options.placement.call(this,t[0],this.$element[0]):this.options.placement,t.detach().css({top:0,left:0,display:"block"}),this.options.container?t.appendTo(this.options.container):t.insertAfter(this.$element),n=this.getPosition(),r=t[0].offsetWidth,i=t[0].offsetHeight;switch(s){case"bottom":o={top:n.top+n.height,left:n.left+n.width/2-r/2};break;case"top":o={top:n.top-i,left:n.left+n.width/2-r/2};break;case"left":o={top:n.top+n.height/2-i/2,left:n.left-r};break;case"right":o={top:n.top+n.height/2-i/2,left:n.left+n.width}}this.applyPlacement(o,s),this.$element.trigger("shown")}},applyPlacement:function(e,t){var n=this.tip(),r=n[0].offsetWidth,i=n[0].offsetHeight,s,o,u,a;n.offset(e).addClass(t).addClass("in"),s=n[0].offsetWidth,o=n[0].offsetHeight,t=="top"&&o!=i&&(e.top=e.top+i-o,a=!0),t=="bottom"||t=="top"?(u=0,e.left<0&&(u=e.left*-2,e.left=0,n.offset(e),s=n[0].offsetWidth,o=n[0].offsetHeight),this.replaceArrow(u-r+s,s,"left")):this.replaceArrow(o-i,o,"top"),a&&n.offset(e)},replaceArrow:function(e,t,n){this.arrow().css(n,e?50*(1-e/t)+"%":"")},setContent:function(){var e=this.tip(),t=this.getTitle();e.find(".tooltip-inner")[this.options.html?"html":"text"](t),e.removeClass("fade in top bottom left right")},hide:function(){function i(){var t=setTimeout(function(){n.off(e.support.transition.end).detach()},500);n.one(e.support.transition.end,function(){clearTimeout(t),n.detach()})}var t=this,n=this.tip(),r=e.Event("hide");this.$element.trigger(r);if(r.isDefaultPrevented())return;return n.removeClass("in"),e.support.transition&&this.$tip.hasClass("fade")?i():n.detach(),this.$element.trigger("hidden"),this},fixTitle:function(){var e=this.$element;(e.attr("title")||typeof e.attr("data-original-title")!="string")&&e.attr("data-original-title",e.attr("title")||"").attr("title","")},hasContent:function(){return this.getTitle()},getPosition:function(){var t=this.$element[0];return e.extend({},typeof t.getBoundingClientRect=="function"?t.getBoundingClientRect():{width:t.offsetWidth,height:t.offsetHeight},this.$element.offset())},getTitle:function(){var e,t=this.$element,n=this.options;return e=t.attr("data-original-title")||(typeof n.title=="function"?n.title.call(t[0]):n.title),e},tip:function(){return this.$tip=this.$tip||e(this.options.template)},arrow:function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},validate:function(){this.$element[0].parentNode||(this.hide(),this.$element=null,this.options=null)},enable:function(){this.enabled=!0},disable:function(){this.enabled=!1},toggleEnabled:function(){this.enabled=!this.enabled},toggle:function(t){var n=t?e(t.currentTarget)[this.type](this._options).data(this.type):this;n.tip().hasClass("in")?n.hide():n.show()},destroy:function(){this.hide().$element.off("."+this.type).removeData(this.type)}};var n=e.fn.tooltip;e.fn.tooltip=function(n){return this.each(function(){var r=e(this),i=r.data("tooltip"),s=typeof n=="object"&&n;i||r.data("tooltip",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.tooltip.Constructor=t,e.fn.tooltip.defaults={animation:!0,placement:"top",selector:!1,template:'
',trigger:"hover focus",title:"",delay:0,html:!1,container:!1},e.fn.tooltip.noConflict=function(){return e.fn.tooltip=n,this}}(window.jQuery),!function(e){"use strict";var t=function(e,t){this.init("popover",e,t)};t.prototype=e.extend({},e.fn.tooltip.Constructor.prototype,{constructor:t,setContent:function(){var e=this.tip(),t=this.getTitle(),n=this.getContent();e.find(".popover-title")[this.options.html?"html":"text"](t),e.find(".popover-content")[this.options.html?"html":"text"](n),e.removeClass("fade top bottom left right in")},hasContent:function(){return this.getTitle()||this.getContent()},getContent:function(){var e,t=this.$element,n=this.options;return e=(typeof n.content=="function"?n.content.call(t[0]):n.content)||t.attr("data-content"),e},tip:function(){return this.$tip||(this.$tip=e(this.options.template)),this.$tip},destroy:function(){this.hide().$element.off("."+this.type).removeData(this.type)}});var n=e.fn.popover;e.fn.popover=function(n){return this.each(function(){var r=e(this),i=r.data("popover"),s=typeof n=="object"&&n;i||r.data("popover",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.popover.Constructor=t,e.fn.popover.defaults=e.extend({},e.fn.tooltip.defaults,{placement:"right",trigger:"click",content:"",template:'

'}),e.fn.popover.noConflict=function(){return e.fn.popover=n,this}}(window.jQuery),!function(e){"use strict";function t(t,n){var r=e.proxy(this.process,this),i=e(t).is("body")?e(window):e(t),s;this.options=e.extend({},e.fn.scrollspy.defaults,n),this.$scrollElement=i.on("scroll.scroll-spy.data-api",r),this.selector=(this.options.target||(s=e(t).attr("href"))&&s.replace(/.*(?=#[^\s]+$)/,"")||"")+" .nav li > a",this.$body=e("body"),this.refresh(),this.process()}t.prototype={constructor:t,refresh:function(){var t=this,n;this.offsets=e([]),this.targets=e([]),n=this.$body.find(this.selector).map(function(){var n=e(this),r=n.data("target")||n.attr("href"),i=/^#\w/.test(r)&&e(r);return i&&i.length&&[[i.position().top+(!e.isWindow(t.$scrollElement.get(0))&&t.$scrollElement.scrollTop()),r]]||null}).sort(function(e,t){return e[0]-t[0]}).each(function(){t.offsets.push(this[0]),t.targets.push(this[1])})},process:function(){var e=this.$scrollElement.scrollTop()+this.options.offset,t=this.$scrollElement[0].scrollHeight||this.$body[0].scrollHeight,n=t-this.$scrollElement.height(),r=this.offsets,i=this.targets,s=this.activeTarget,o;if(e>=n)return s!=(o=i.last()[0])&&this.activate(o);for(o=r.length;o--;)s!=i[o]&&e>=r[o]&&(!r[o+1]||e<=r[o+1])&&this.activate(i[o])},activate:function(t){var n,r;this.activeTarget=t,e(this.selector).parent(".active").removeClass("active"),r=this.selector+'[data-target="'+t+'"],'+this.selector+'[href="'+t+'"]',n=e(r).parent("li").addClass("active"),n.parent(".dropdown-menu").length&&(n=n.closest("li.dropdown").addClass("active")),n.trigger("activate")}};var n=e.fn.scrollspy;e.fn.scrollspy=function(n){return this.each(function(){var r=e(this),i=r.data("scrollspy"),s=typeof n=="object"&&n;i||r.data("scrollspy",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.scrollspy.Constructor=t,e.fn.scrollspy.defaults={offset:10},e.fn.scrollspy.noConflict=function(){return e.fn.scrollspy=n,this},e(window).on("load",function(){e('[data-spy="scroll"]').each(function(){var t=e(this);t.scrollspy(t.data())})})}(window.jQuery),!function(e){"use strict";var t=function(t){this.element=e(t)};t.prototype={constructor:t,show:function(){var t=this.element,n=t.closest("ul:not(.dropdown-menu)"),r=t.attr("data-target"),i,s,o;r||(r=t.attr("href"),r=r&&r.replace(/.*(?=#[^\s]*$)/,""));if(t.parent("li").hasClass("active"))return;i=n.find(".active:last a")[0],o=e.Event("show",{relatedTarget:i}),t.trigger(o);if(o.isDefaultPrevented())return;s=e(r),this.activate(t.parent("li"),n),this.activate(s,s.parent(),function(){t.trigger({type:"shown",relatedTarget:i})})},activate:function(t,n,r){function o(){i.removeClass("active").find("> .dropdown-menu > .active").removeClass("active"),t.addClass("active"),s?(t[0].offsetWidth,t.addClass("in")):t.removeClass("fade"),t.parent(".dropdown-menu")&&t.closest("li.dropdown").addClass("active"),r&&r()}var i=n.find("> .active"),s=r&&e.support.transition&&i.hasClass("fade");s?i.one(e.support.transition.end,o):o(),i.removeClass("in")}};var n=e.fn.tab;e.fn.tab=function(n){return this.each(function(){var r=e(this),i=r.data("tab");i||r.data("tab",i=new t(this)),typeof n=="string"&&i[n]()})},e.fn.tab.Constructor=t,e.fn.tab.noConflict=function(){return e.fn.tab=n,this},e(document).on("click.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"]',function(t){t.preventDefault(),e(this).tab("show")})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.$element=e(t),this.options=e.extend({},e.fn.typeahead.defaults,n),this.matcher=this.options.matcher||this.matcher,this.sorter=this.options.sorter||this.sorter,this.highlighter=this.options.highlighter||this.highlighter,this.updater=this.options.updater||this.updater,this.source=this.options.source,this.$menu=e(this.options.menu),this.shown=!1,this.listen()};t.prototype={constructor:t,select:function(){var e=this.$menu.find(".active").attr("data-value");return this.$element.val(this.updater(e)).change(),this.hide()},updater:function(e){return e},show:function(){var t=e.extend({},this.$element.position(),{height:this.$element[0].offsetHeight});return this.$menu.insertAfter(this.$element).css({top:t.top+t.height,left:t.left}).show(),this.shown=!0,this},hide:function(){return this.$menu.hide(),this.shown=!1,this},lookup:function(t){var n;return this.query=this.$element.val(),!this.query||this.query.length"+t+""})},render:function(t){var n=this;return t=e(t).map(function(t,r){return t=e(n.options.item).attr("data-value",r),t.find("a").html(n.highlighter(r)),t[0]}),t.first().addClass("active"),this.$menu.html(t),this},next:function(t){var n=this.$menu.find(".active").removeClass("active"),r=n.next();r.length||(r=e(this.$menu.find("li")[0])),r.addClass("active")},prev:function(e){var t=this.$menu.find(".active").removeClass("active"),n=t.prev();n.length||(n=this.$menu.find("li").last()),n.addClass("active")},listen:function(){this.$element.on("focus",e.proxy(this.focus,this)).on("blur",e.proxy(this.blur,this)).on("keypress",e.proxy(this.keypress,this)).on("keyup",e.proxy(this.keyup,this)),this.eventSupported("keydown")&&this.$element.on("keydown",e.proxy(this.keydown,this)),this.$menu.on("click",e.proxy(this.click,this)).on("mouseenter","li",e.proxy(this.mouseenter,this)).on("mouseleave","li",e.proxy(this.mouseleave,this))},eventSupported:function(e){var t=e in this.$element;return t||(this.$element.setAttribute(e,"return;"),t=typeof this.$element[e]=="function"),t},move:function(e){if(!this.shown)return;switch(e.keyCode){case 9:case 13:case 27:e.preventDefault();break;case 38:e.preventDefault(),this.prev();break;case 40:e.preventDefault(),this.next()}e.stopPropagation()},keydown:function(t){this.suppressKeyPressRepeat=~e.inArray(t.keyCode,[40,38,9,13,27]),this.move(t)},keypress:function(e){if(this.suppressKeyPressRepeat)return;this.move(e)},keyup:function(e){switch(e.keyCode){case 40:case 38:case 16:case 17:case 18:break;case 9:case 13:if(!this.shown)return;this.select();break;case 27:if(!this.shown)return;this.hide();break;default:this.lookup()}e.stopPropagation(),e.preventDefault()},focus:function(e){this.focused=!0},blur:function(e){this.focused=!1,!this.mousedover&&this.shown&&this.hide()},click:function(e){e.stopPropagation(),e.preventDefault(),this.select(),this.$element.focus()},mouseenter:function(t){this.mousedover=!0,this.$menu.find(".active").removeClass("active"),e(t.currentTarget).addClass("active")},mouseleave:function(e){this.mousedover=!1,!this.focused&&this.shown&&this.hide()}};var n=e.fn.typeahead;e.fn.typeahead=function(n){return this.each(function(){var r=e(this),i=r.data("typeahead"),s=typeof n=="object"&&n;i||r.data("typeahead",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.typeahead.defaults={source:[],items:8,menu:'
    ',item:'
    • ',minLength:1},e.fn.typeahead.Constructor=t,e.fn.typeahead.noConflict=function(){return e.fn.typeahead=n,this},e(document).on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(t){var n=e(this);if(n.data("typeahead"))return;n.typeahead(n.data())})}(window.jQuery),!function(e){"use strict";var t=function(t,n){this.options=e.extend({},e.fn.affix.defaults,n),this.$window=e(window).on("scroll.affix.data-api",e.proxy(this.checkPosition,this)).on("click.affix.data-api",e.proxy(function(){setTimeout(e.proxy(this.checkPosition,this),1)},this)),this.$element=e(t),this.checkPosition()};t.prototype.checkPosition=function(){if(!this.$element.is(":visible"))return;var t=e(document).height(),n=this.$window.scrollTop(),r=this.$element.offset(),i=this.options.offset,s=i.bottom,o=i.top,u="affix affix-top affix-bottom",a;typeof i!="object"&&(s=o=i),typeof o=="function"&&(o=i.top()),typeof s=="function"&&(s=i.bottom()),a=this.unpin!=null&&n+this.unpin<=r.top?!1:s!=null&&r.top+this.$element.height()>=t-s?"bottom":o!=null&&n<=o?"top":!1;if(this.affixed===a)return;this.affixed=a,this.unpin=a=="bottom"?r.top-n:null,this.$element.removeClass(u).addClass("affix"+(a?"-"+a:""))};var n=e.fn.affix;e.fn.affix=function(n){return this.each(function(){var r=e(this),i=r.data("affix"),s=typeof n=="object"&&n;i||r.data("affix",i=new t(this,s)),typeof n=="string"&&i[n]()})},e.fn.affix.Constructor=t,e.fn.affix.defaults={offset:0},e.fn.affix.noConflict=function(){return e.fn.affix=n,this},e(window).on("load",function(){e('[data-spy="affix"]').each(function(){var t=e(this),n=t.data();n.offset=n.offset||{},n.offsetBottom&&(n.offset.bottom=n.offsetBottom),n.offsetTop&&(n.offset.top=n.offsetTop),t.affix(n)})})}(window.jQuery); -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | // Load configuration file. 2 | package main 3 | 4 | import ( 5 | "bufio" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "launchpad.net/goyaml" 10 | "net/http" 11 | "regexp" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | // parsed and validated config data 17 | type Config struct { 18 | GroupParams map[string]*configGroup 19 | GroupStreams map[string]*[]Stream // map[groupname]stream 20 | Stubs configStub 21 | Zabbix configZabbix 22 | Samples []string 23 | ListenHTTP string 24 | User string 25 | Pass string 26 | UserAgents []string 27 | ErrorLog string 28 | ExpireDurationDB time.Duration // measured in hours 29 | IsReady chan bool // config parsed and ready to use 30 | } 31 | 32 | // parsed grup config 33 | type configGroup struct { 34 | Type StreamType 35 | Probers int 36 | MediaProbers int 37 | CheckBrokenTime int 38 | ConnectTimeout time.Duration 39 | RWTimeout time.Duration 40 | SlowWarningTimeout time.Duration 41 | VerySlowWarningTimeout time.Duration 42 | TimeBetweenTasks time.Duration 43 | TaskTTL time.Duration 44 | TryOneSegment bool 45 | MethodHTTP string 46 | ParseMethod string 47 | User string 48 | Pass string 49 | } 50 | 51 | type configZabbix struct { 52 | DiscoveryPath string `yaml:"discovery-path,omitempty"` 53 | DiscoveryGroups []string `yaml:"discovery-groups,omitempty"` 54 | NameTemplate string `yaml:"name-template,omitempty"` 55 | TitleTemplate string `yaml:"title-template,omitempty"` 56 | } 57 | 58 | // custom values for HTML-templates and reports 59 | type configStub struct { 60 | Name string `yaml:"name,omitempty"` 61 | } 62 | 63 | // raw config data 64 | type configYAML struct { 65 | ListenHTTP string `yaml:"http-api-listen,omitempty"` 66 | User string `yaml:"http-api-user,omitempty"` 67 | Pass string `yaml:"http-api-pass,omitempty"` 68 | Stubs configStub `yaml:"stubs,omitempty"` 69 | Zabbix configZabbix `yaml:"zabbix,omitempty"` 70 | Samples []string `yaml:"unmortal,omitempty"` 71 | UserAgents []string `yaml:"user-agents,omitempty"` 72 | Defaults configGroupYAML `yaml:"defaults,omitempty"` 73 | Groups map[string]configGroupYAML `yaml:"groups,omitempty"` 74 | ExpireDurationDB time.Duration `yaml:"db-expired"` // measured in hours 75 | } 76 | 77 | // rawconfig group data 78 | type configGroupYAML struct { 79 | Type string `yaml:"type,omitempty"` 80 | URI string `yaml:"streams-uri,omitempty"` // external link list 81 | Streams []string `yaml:"streams,omitempty"` // link list 82 | Probers int `yaml:"probers,omitempty"` // num of 83 | MediaProbers int `yaml:"media-probers,omitempty"` // num of 84 | CheckBrokenTime int `yaml:"check-broken-time"` // ms 85 | ConnectTimeout time.Duration `yaml:"connect-timeout,omitempty"` // sec 86 | RWTimeout time.Duration `yaml:"rw-timeout,omitempty"` // sec 87 | SlowWarningTimeout time.Duration `yaml:"slow-warning-timeout,omitempty"` // sec 88 | VerySlowWarningTimeout time.Duration `yaml:"very-slow-warning-timeout,omitempty"` // sec 89 | TimeBetweenTasks time.Duration `yaml:"time-between-tasks,omitempty"` // sec 90 | TaskTTL time.Duration `yaml:"task-ttl,omitempty"` // sec 91 | TryOneSegment bool `yaml:"one-segment,omitempty"` 92 | MethodHTTP string `yaml:"http-method,omitempty"` // GET, HEAD 93 | ErrorLog string `yaml:"error-log,omitempty"` 94 | ParseMethod string `yaml:"parse-method,omitempty"` // regexp for alternative method of title/name parsing from the URL 95 | User string `yaml:"user,omitempty"` 96 | Pass string `yaml:"pass,omitempty"` 97 | } 98 | 99 | var cfg *Config 100 | 101 | // TODO Dynamic configuration without program restart. 102 | // Elder. 103 | func InitConfig(confile string) { 104 | rawcfg := rawConfig(confile) 105 | cfg = new(Config) 106 | cfg.IsReady = make(chan bool, 1) 107 | parseOptionsConfig(rawcfg) 108 | parseGroupsConfig(rawcfg) 109 | cfg.IsReady <- true 110 | } 111 | 112 | // XXX for future 113 | func ConfigKeeper() { 114 | <-cfg.IsReady 115 | select {} // TODO reload config by query 116 | } 117 | 118 | // 119 | func (cfg *Config) Params(gname string) configGroup { 120 | if data, ok := cfg.GroupParams[gname]; ok { 121 | return *data 122 | } else { 123 | return configGroup{} 124 | } 125 | } 126 | 127 | // 128 | func (cfg *Config) Streams() { 129 | 130 | } 131 | 132 | // Read raw config with YAML validation 133 | func rawConfig(confile string) *configYAML { 134 | cfg := new(configYAML) 135 | 136 | // Hardcoded defaults: 137 | cfg.Stubs = configStub{Name: "Stream Surfer"} 138 | 139 | if confile == "" { 140 | confile = "/etc/streamsurfer.yaml" 141 | } 142 | data, e := ioutil.ReadFile(FullPath(confile)) 143 | if e == nil { 144 | e = goyaml.Unmarshal(data, &cfg) 145 | if e != nil { 146 | print("Config file parsing failed. Hardcoded defaults used.\n") 147 | } 148 | } 149 | 150 | return cfg 151 | } 152 | 153 | // 154 | func parseOptionsConfig(rawcfg *configYAML) { 155 | cfg.ListenHTTP = rawcfg.ListenHTTP 156 | cfg.User = rawcfg.User 157 | cfg.Pass = rawcfg.Pass 158 | cfg.Stubs = rawcfg.Stubs 159 | cfg.Zabbix = rawcfg.Zabbix 160 | cfg.Samples = rawcfg.Samples 161 | cfg.UserAgents = rawcfg.UserAgents 162 | cfg.ExpireDurationDB = rawcfg.ExpireDurationDB * time.Hour 163 | } 164 | 165 | // 166 | func parseGroupsConfig(rawcfg *configYAML) { 167 | cfg.GroupParams = make(map[string]*configGroup) 168 | cfg.GroupStreams = make(map[string]*[]Stream) 169 | 170 | for gname, gdata := range rawcfg.Groups { 171 | stype := String2StreamType(gdata.Type) 172 | cfg.GroupParams[gname] = &configGroup{ 173 | Type: stype, 174 | Probers: gdata.Probers, 175 | MediaProbers: gdata.MediaProbers, 176 | CheckBrokenTime: gdata.CheckBrokenTime, 177 | ParseMethod: gdata.ParseMethod, 178 | TimeBetweenTasks: gdata.TimeBetweenTasks, 179 | ConnectTimeout: gdata.ConnectTimeout, 180 | RWTimeout: gdata.RWTimeout, 181 | SlowWarningTimeout: gdata.SlowWarningTimeout, 182 | VerySlowWarningTimeout: gdata.VerySlowWarningTimeout, 183 | TaskTTL: gdata.TaskTTL, 184 | TryOneSegment: gdata.TryOneSegment, 185 | MethodHTTP: strings.ToUpper(gdata.MethodHTTP), 186 | User: gdata.User, 187 | Pass: gdata.Pass, 188 | } 189 | 190 | if gdata.URI != "" { 191 | cfg.GroupStreams[gname] = new([]Stream) 192 | addRemoteConfig(cfg.GroupStreams[gname], cfg.GroupParams[gname], gname, gdata.URI, gdata.User, gdata.Pass) 193 | } else { 194 | cfg.GroupStreams[gname] = new([]Stream) 195 | addLocalConfig(cfg.GroupStreams[gname], cfg.GroupParams[gname], gname, gdata.Streams) 196 | } 197 | } 198 | // println("----------------------------------------") 199 | // for key, data := range cfg.GroupParams { 200 | // fmt.Printf("%s %#v\n", key, data) 201 | // } 202 | // println("----------------------------------------") 203 | // for key, data := range cfg.GroupStreams { 204 | // fmt.Printf("%s %#v\n", key, data) 205 | // } 206 | } 207 | 208 | // Helper. Get remote list of streams. 209 | func addRemoteConfig(dest *[]Stream, params *configGroup, group string, uri, remoteUser, remotePass string) error { 210 | defer func() error { 211 | if r := recover(); r != nil { 212 | return errors.New(fmt.Sprintf("Can't get remote config for (%s) %s %s", params.Type, group, uri)) 213 | } 214 | return nil 215 | }() 216 | 217 | client := NewTimeoutClient(10*time.Second, 10*time.Second) 218 | req, err := http.NewRequest("GET", uri, nil) 219 | if err != nil { 220 | return err 221 | } 222 | if remoteUser != "" { 223 | req.SetBasicAuth(remoteUser, remotePass) 224 | } 225 | result, err := client.Do(req) 226 | if err == nil { 227 | body := bufio.NewReader(result.Body) 228 | for { 229 | line, err := body.ReadString('\n') 230 | if err != nil { 231 | break 232 | } 233 | uri, name, title := splitName(params.ParseMethod, line) 234 | *dest = append(*dest, Stream{URI: uri, Type: params.Type, Name: name, Title: title, Group: group}) 235 | } 236 | } 237 | return err 238 | } 239 | 240 | // Helper. Parse config of 241 | func addLocalConfig(dest *[]Stream, params *configGroup, group string, sources []string) { 242 | for _, source := range sources { 243 | uri, name, title := splitName(params.ParseMethod, source) 244 | *dest = append(*dest, Stream{URI: uri, Type: params.Type, Name: name, Title: title, Group: group}) 245 | } 246 | } 247 | 248 | // Helper. Split stream link to URI and Name parts. 249 | // Supported both cases: titleuri and urititle 250 | // If `re` presents then name parsed from uri by regular expression. 251 | // URI must be prepended by http:// or https:// 252 | func splitName(re, source string) (uri, name, title string) { 253 | source = strings.TrimSpace(source) 254 | sep := regexp.MustCompile("htt(p|ps)://") 255 | loc := sep.FindStringIndex(source) 256 | if loc != nil { 257 | if loc[0] == 0 { // uri title 258 | splitted := strings.SplitN(source, " ", 2) 259 | if len(splitted) > 1 { 260 | title = strings.TrimSpace(splitted[1]) 261 | } 262 | uri = strings.TrimSpace(splitted[0]) 263 | } else { // title uri 264 | title = strings.TrimSpace(source[0:loc[0]]) 265 | uri = source[loc[0]:] 266 | } 267 | if title == "" { 268 | title = uri 269 | } 270 | } 271 | if re != "" { // get name by regexp 272 | compiledRe := regexp.MustCompile(re) 273 | vals := compiledRe.FindStringSubmatch(uri) 274 | if len(vals) > 1 { 275 | name = vals[1] 276 | } else { 277 | name = title 278 | } 279 | } else { 280 | name = title 281 | } 282 | return 283 | } 284 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | // "encoding/gob" 6 | "encoding/json" 7 | "fmt" 8 | "github.com/garyburd/redigo/redis" 9 | "net" 10 | "net/http" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | var redisPool *redis.Pool 16 | 17 | /* 18 | * Public API 19 | */ 20 | 21 | // Initialize subsystem. Must preceed API calls to storage subsys. 22 | func InitStorage() { 23 | redisPool = &redis.Pool{ 24 | MaxIdle: 3, 25 | IdleTimeout: 240 * time.Second, 26 | Dial: func() (redis.Conn, error) { 27 | c, err := redis.Dial("tcp", net.JoinHostPort("127.1", "6379")) 28 | if err != nil { 29 | return nil, err 30 | // TODO при недоступности редиса останавливать мониторинг 31 | } 32 | return c, err 33 | }, 34 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 35 | _, err := c.Do("PING") 36 | return err 37 | }, 38 | } 39 | } 40 | 41 | // Save monitoring data to DB. 42 | func RedKeepResult(key Key, weight time.Time, res Result) error { 43 | //var buf bytes.Buffer 44 | 45 | conn := redisPool.Get() 46 | defer conn.Close() 47 | // enc := gob.NewEncoder(&buf) 48 | keepit := KeepedResult{ 49 | Tid: res.Task.Tid, 50 | Stream: Stream{ 51 | URI: res.Task.URI, 52 | Type: res.Task.Type, 53 | Name: res.Task.Name, 54 | Group: res.Task.Group, 55 | Title: res.Task.Title, 56 | }, 57 | ErrType: res.ErrType, 58 | HTTPCode: res.HTTPCode, 59 | HTTPStatus: res.HTTPStatus, 60 | ContentLength: res.ContentLength, 61 | RealContentLength: res.RealContentLength, 62 | Headers: res.Headers, 63 | Body: res.Body.Bytes(), 64 | Started: res.Started, 65 | Elapsed: res.Elapsed, 66 | TotalErrs: res.TotalErrs, 67 | } 68 | if res.Pid == nil { 69 | keepit.Master = true 70 | } else { 71 | keepit.Master = false 72 | } 73 | buf, err := json.Marshal(keepit) 74 | // err := enc.Encode(&keepit) 75 | if err != nil { 76 | fmt.Printf("redis: %s\n", err) 77 | return err 78 | } 79 | _, err = conn.Do("ZADD", key.String(), strconv.FormatInt(weight.Unix(), 10), buf) 80 | if err != nil { 81 | fmt.Printf("redis RedKeepResult: %s\n", err) 82 | } 83 | return err 84 | } 85 | 86 | // Keeps values only for errors and warngings. 87 | func RedKeepError(key Key, weight time.Time, errtype ErrType) error { 88 | conn := redisPool.Get() 89 | defer conn.Close() 90 | _, err := conn.Do("ZADD", fmt.Sprintf("errors/%s", key.String()), strconv.FormatInt(weight.Unix(), 10), errtype) 91 | if err != nil { 92 | fmt.Printf("redis RedKeepError: %s\n", err) 93 | } 94 | return err 95 | } 96 | 97 | func RedLoadResults(key Key, from, to time.Time) ([]KeepedResult, error) { 98 | var src bytes.Buffer 99 | var dst KeepedResult = KeepedResult{Stream: Stream{}, Headers: make(http.Header)} 100 | var result []KeepedResult 101 | 102 | conn := redisPool.Get() 103 | defer conn.Close() 104 | data, err := redis.Values(conn.Do("ZRANGEBYSCORE", key.String(), strconv.FormatInt(from.Unix(), 10), strconv.FormatInt(to.Unix(), 10))) 105 | // dec := gob.NewDecoder(&src) 106 | for _, val := range data { 107 | src.Write(val.([]byte)) 108 | //err := dec.Decode(&dst) 109 | err := json.Unmarshal(src.Bytes(), &dst) 110 | if err == nil { 111 | result = append(result, dst) 112 | } 113 | src.Reset() 114 | } 115 | return result, err 116 | } 117 | 118 | func RedLoadErrors(key Key, from, to time.Time) (map[time.Time]ErrType, error) { 119 | var retval ErrType 120 | 121 | conn := redisPool.Get() 122 | defer conn.Close() 123 | result := make(map[time.Time]ErrType) 124 | data, err := redis.Values(conn.Do("ZRANGEBYSCORE", fmt.Sprintf("errors/%s", key.String()), strconv.FormatInt(from.Unix(), 10), strconv.FormatInt(to.Unix(), 10), "WITHSCORES")) 125 | if err == nil { 126 | for idx, val := range data { 127 | if idx%2 == 0 { // data 128 | retval = ErrType(val.([]uint8)[0]) 129 | } else { // key 130 | key, err := strconv.ParseInt(string(val.([]byte)), 10, 64) 131 | if err == nil { 132 | result[time.Unix(key, 0)] = retval 133 | } 134 | } 135 | } 136 | return result, nil 137 | } else { 138 | return nil, err 139 | } 140 | } 141 | 142 | // Remove expired errors from the Redis sorted set 143 | func RemoveExpiredErrors(expired time.Duration) { 144 | conn := redisPool.Get() 145 | defer conn.Close() 146 | for gname := range cfg.GroupParams { 147 | for _, stream := range *cfg.GroupStreams[gname] { 148 | key := Key{gname, stream.Name} 149 | if deleted, _ := redis.Int(conn.Do("ZREMRANGEBYSCORE", fmt.Sprintf("errors/%s", key.String()), "-inf", strconv.FormatInt(time.Now().Add(-expired).Unix(), 10))); deleted > 0 { 150 | // fmt.Printf("%d expired elements from `errors` set `%s` deleted\n", deleted, key) 151 | } 152 | } 153 | } 154 | } 155 | 156 | // Remove expired errors from the Redis sorted set 157 | func RemoveExpiredResults(expired time.Duration) { 158 | conn := redisPool.Get() 159 | defer conn.Close() 160 | for gname := range cfg.GroupParams { 161 | for _, stream := range *cfg.GroupStreams[gname] { 162 | key := Key{gname, stream.Name} 163 | if deleted, _ := redis.Int(conn.Do("ZREMRANGEBYSCORE", fmt.Sprintf("%s", key.String()), "-inf", strconv.FormatInt(time.Now().Add(-expired).Unix(), 10))); deleted > 0 { 164 | // fmt.Printf("%d expired elements from `results` set `%s` deleted\n", deleted, key) 165 | } 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /helpers-md5.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "crypto/md5" 4 | import "strings" 5 | 6 | const itoa64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 7 | 8 | var md5_crypt_swaps = [16]int{12, 6, 0, 13, 7, 1, 14, 8, 2, 15, 9, 3, 5, 10, 4, 11} 9 | 10 | type MD5Entry struct { 11 | Magic, Salt, Hash []byte 12 | } 13 | 14 | func NewMD5Entry(e string) *MD5Entry { 15 | parts := strings.SplitN(e, "$", 4) 16 | if len(parts) != 4 { 17 | return nil 18 | } 19 | return &MD5Entry{ 20 | Magic: []byte("$" + parts[1] + "$"), 21 | Salt: []byte(parts[2]), 22 | Hash: []byte(parts[3]), 23 | } 24 | } 25 | 26 | /* 27 | MD5 password crypt implementation 28 | */ 29 | func MD5Crypt(password, salt, magic []byte) []byte { 30 | d := md5.New() 31 | 32 | d.Write(password) 33 | d.Write(magic) 34 | d.Write(salt) 35 | 36 | d2 := md5.New() 37 | d2.Write(password) 38 | d2.Write(salt) 39 | d2.Write(password) 40 | 41 | for i, mixin := 0, d2.Sum(nil); i < len(password); i++ { 42 | d.Write([]byte{mixin[i%16]}) 43 | } 44 | 45 | for i := len(password); i != 0; i >>= 1 { 46 | if i&1 == 0 { 47 | d.Write([]byte{password[0]}) 48 | } else { 49 | d.Write([]byte{0}) 50 | } 51 | } 52 | 53 | final := d.Sum(nil) 54 | 55 | for i := 0; i < 1000; i++ { 56 | d2 := md5.New() 57 | if i&1 == 0 { 58 | d2.Write(final) 59 | } else { 60 | d2.Write(password) 61 | } 62 | 63 | if i%3 != 0 { 64 | d2.Write(salt) 65 | } 66 | 67 | if i%7 != 0 { 68 | d2.Write(password) 69 | } 70 | 71 | if i&1 == 0 { 72 | d2.Write(password) 73 | } else { 74 | d2.Write(final) 75 | } 76 | final = d2.Sum(nil) 77 | } 78 | 79 | result := make([]byte, 0, 22) 80 | v := uint(0) 81 | bits := uint(0) 82 | for _, i := range md5_crypt_swaps { 83 | v |= (uint(final[i]) << bits) 84 | for bits = bits + 8; bits > 6; bits -= 6 { 85 | result = append(result, itoa64[v&0x3f]) 86 | v >>= 6 87 | } 88 | } 89 | result = append(result, itoa64[v&0x3f]) 90 | 91 | return append(append(append(magic, salt...), '$'), result...) 92 | } 93 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | // misc helpers 2 | package main 3 | 4 | import ( 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func FullPath(name string) string { 12 | return os.ExpandEnv(strings.Replace(name, "~", os.Getenv("HOME"), 1)) 13 | } 14 | 15 | // Return text representation for StreamType constants 16 | func StreamType2String(t StreamType) string { 17 | switch t { 18 | case SAMPLE: 19 | return "sample" 20 | case HLS: 21 | return "hls" 22 | case HDS: 23 | return "hds" 24 | case WV: 25 | return "wv" 26 | case HTTP: 27 | return "http" 28 | default: 29 | return "unknown" 30 | } 31 | } 32 | 33 | // 34 | func String2StreamType(s string) StreamType { 35 | switch strings.ToLower(s) { 36 | case "sample": 37 | return SAMPLE 38 | case "hls": 39 | return HLS 40 | case "hds": 41 | return HDS 42 | case "wv": 43 | return WV 44 | case "http": 45 | return HTTP 46 | default: 47 | return UNKSTREAM 48 | } 49 | } 50 | 51 | // Text representation of stream errors 52 | func StreamErr2String(err ErrType) string { 53 | switch err { 54 | case SUCCESS: 55 | return "success" 56 | case HLSPARSER: 57 | return "HLS parser" // debug 58 | case BADREQUEST: 59 | return "invalid request" // debug 60 | case SLOW: 61 | return "slow response" 62 | case VERYSLOW: 63 | return "very slow response" 64 | case BADSTATUS: 65 | return "bad status" 66 | case BADURI: 67 | return "bad URI" 68 | case LISTEMPTY: // HLS specific 69 | return "list empty" 70 | case BADFORMAT: // HLS specific 71 | return "bad format" 72 | case TTLEXPIRED: 73 | return "TTL expired" 74 | case RTIMEOUT: 75 | return "timeout on read" 76 | case CTIMEOUT: 77 | return "connection timeout" 78 | case BADLENGTH: 79 | return "bad content length value" 80 | case BODYREAD: 81 | return "response body error" 82 | case REFUSED: 83 | return "connection refused" 84 | default: 85 | return "unknown" 86 | } 87 | } 88 | 89 | // 90 | func String2StreamErr(s string) ErrType { 91 | switch strings.ToLower(s) { 92 | case "success": 93 | return SUCCESS 94 | case "debug": 95 | return DEBUG_LEVEL 96 | case "hlsparser": 97 | return HLSPARSER 98 | case "badrequest": 99 | return BADREQUEST 100 | case "warning": 101 | return WARNING_LEVEL 102 | case "slow": 103 | return SLOW 104 | case "veryslow": 105 | return VERYSLOW 106 | case "badstatus": 107 | return BADSTATUS 108 | case "baduri": 109 | return BADURI 110 | case "listempty": // HLS specific 111 | return LISTEMPTY 112 | case "badformat": // HLS specific 113 | return BADFORMAT 114 | case "ttlexpired": 115 | return TTLEXPIRED 116 | case "rtimeout": 117 | return RTIMEOUT 118 | case "error": 119 | return ERROR_LEVEL 120 | case "ctimeout": 121 | return CTIMEOUT 122 | case "badlength": 123 | return BADLENGTH 124 | case "bodyread": 125 | return BODYREAD 126 | case "critical": 127 | return CRITICAL_LEVEL 128 | case "refused": 129 | return REFUSED 130 | default: 131 | return UNKERR 132 | } 133 | } 134 | 135 | // Helper to make a href. 136 | // First arg must be URL, second is text. 137 | // Optional args is title (3d) and class (4d). 138 | func href(url, text string, opts ...string) string { 139 | switch len(opts) { 140 | case 1: 141 | return fmt.Sprintf("%s", opts[0], url, text) 142 | case 2: 143 | return fmt.Sprintf("%s", opts[0], opts[1], url, text) 144 | default: 145 | return fmt.Sprintf("%s", url, text) 146 | } 147 | } 148 | 149 | // 150 | func span(text, class string) string { 151 | return fmt.Sprintf("%s", class, text) 152 | } 153 | 154 | func bytewe(res []byte, err error) []byte { 155 | return res 156 | } 157 | 158 | // Returns proper user agent string for the HTTP-headers. 159 | func UserAgent() string { 160 | if len(cfg.UserAgents) > 0 { 161 | return cfg.UserAgents[rand.Intn(len(cfg.UserAgents)-1)] 162 | } else { 163 | return fmt.Sprintf("%s/%s", SURFER, VERSION) 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /hlsproberc-mintest: -------------------------------------------------------------------------------- 1 | # -*- mode: yaml -*- 2 | hls-streams: 3 | # Just place here URIs to variant playlists or chunklists 4 | - http://example.com/playlist1.m3u8 5 | - http://example.com/playlist2.m3u8 6 | - http://notexistent.domain/link.html 7 | - aospijfwoijfwef 8 | http-streams: 9 | - http://example.com/00/70/sd_2012_example_film.wvm 10 | samples: 11 | - http://ya.ru 12 | - http://google.com 13 | params: 14 | http-probers: 4 15 | hls-probers: 4 16 | media-probers: 32 17 | connect-timeout: 2 18 | rw-timeout: 4 19 | warning-timeout: 2 20 | time-between-tasks: 20 21 | http-api-listen: 127.1:8088 22 | options: 23 | one-segment: true 24 | -------------------------------------------------------------------------------- /html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{.Title}} :: Stream Surfer 6 | 7 | 8 | 9 | 10 | 11 | 12 |
    13 |
    14 |

    {{.Title}}

    15 |

    Monitor is {{if .Data.MonitoringState}}running{{else}}stopped{{end}}. {{.Data.TotalMonitoringPoints}} streams monitored ({{.Data.TotalHLSMonitoringPoints}} HLS, {{.Data.TotalHDSMonitoringPoints}} HDS, {{.Data.TotalHTTPMonitoringPoints}} HTTP).

    16 |

    Show streams activity »

    17 |
    18 | 19 |
    20 |
    21 |

    Control

    22 |

    Donec id elit non mi porta gravida at eget metus. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus. Etiam porta sem malesuada magna mollis euismod. Donec sed odio dui.

    23 |

    Control panel »

    24 |
    25 |
    26 |

    Reports

    27 |

    Reports are:

    28 | 33 |

    Reports page »

    34 |
    35 |
    36 |

    Documentation (online)

    37 |

    Donec sed odio dui. Cras justo odio, dapibus ac facilisis in, egestas eget quam. Vestibulum id ligula porta felis euismod semper. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus.

    38 |

    Open site »

    39 |
    40 |
    41 |
    42 | 46 |
    47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /html/menu.html: -------------------------------------------------------------------------------- 1 |
    2 |
    3 | Stream Surfer 4 | 10 |
    11 |
    12 | -------------------------------------------------------------------------------- /http-api.go: -------------------------------------------------------------------------------- 1 | // HTTP API to control probe service for representing collected data. 2 | package main 3 | 4 | import ( 5 | "crypto/sha1" 6 | "encoding/base64" 7 | "expvar" 8 | "fmt" 9 | "github.com/gorilla/mux" 10 | "net/http" 11 | "os" 12 | "strconv" 13 | "strings" 14 | "text/template" 15 | "time" 16 | ) 17 | 18 | var Page *template.Template 19 | 20 | // Elder. 21 | func HttpAPI() { 22 | var err error 23 | 24 | Page, err = template.ParseGlob("templates/*.tmpl") 25 | if err != nil { 26 | fmt.Printf("Error in template with error %s", err) 27 | os.Exit(1) 28 | } 29 | 30 | r := mux.NewRouter() 31 | r.HandleFunc("/debug", HandleHTTP(expvarHandler)).Methods("GET", "HEAD") 32 | r.HandleFunc("/", HandleHTTP(rootAPI)).Methods("GET", "HEAD") 33 | 34 | /* Monitoring interface (for humans and robots) 35 | */ 36 | // Show stream list for all groups 37 | r.HandleFunc("/act", HandleHTTP(ActivityIndex)).Methods("GET") 38 | // Show stream list for the group 39 | r.HandleFunc("/act/{group}", HandleHTTP(ActivityIndex)).Methods("GET") 40 | // Информация о потоке и сводная статистика 41 | r.HandleFunc("/act/{group}/{stream}", HandleHTTP(ActivityStreamInfo)).Methods("GET") 42 | r.HandleFunc("/act/{group}/{stream}/", HandleHTTP(ActivityStreamInfo)).Methods("GET") 43 | // История ошибок 44 | r.HandleFunc("/act/{group}/{stream}/{mode:history|errors}", HandleHTTP(ActivityStreamHistory)).Methods("GET") 45 | // Вывод результата проверки для мастер-плейлиста 46 | r.HandleFunc("/act/{group}/{stream}/{stamp:[0-9]+}/raw", HandleHTTP(ActivityStreamHistory)).Methods("GET") 47 | // Вывод результата проверки для вложенных проверок 48 | r.HandleFunc("/act/{group}/{stream}/{stamp:[0-9]+}/{idx:[0-9]+}/raw", HandleHTTP(ActivityStreamHistory)).Methods("GET") 49 | 50 | /* Zabbix integration 51 | */ 52 | // Discovery data for Zabbix for all groups 53 | r.HandleFunc("/zabbix-discovery", HandleHTTP(zabbixDiscovery())).Methods("GET", "HEAD") 54 | r.HandleFunc("/zabbix-discovery/{group}", HandleHTTP(zabbixDiscovery())).Methods("GET", "HEAD") 55 | // строковое значение ошибки для выбранных группы и канала 56 | r.HandleFunc("/mon/error/{group}/{stream}/{astype:int|str}", HandleHTTP(monError)).Methods("GET", "HEAD") 57 | // числовое значение ошибки для выбранных группы и канала в диапазоне errlevel from-upto 58 | r.HandleFunc("/mon/error/{group}/{stream}/{fromerrlevel:[a-z]+}-{uptoerrlevel:[a-z]+}", HandleHTTP(monErrorLevel)).Methods("GET") 59 | 60 | /* Reports for humans 61 | */ 62 | // Вывод описания ошибки из анализатора 63 | r.HandleFunc("/rpt", HandleHTTP(ReportIndex)).Methods("GET") 64 | r.HandleFunc("/rpt/", HandleHTTP(ReportIndex)).Methods("GET") 65 | r.HandleFunc("/rpt/{rptid:[0-9]+}", HandleHTTP(ReportStreamErrors)).Methods("GET") 66 | 67 | // Obsoleted reports with old API: 68 | // r.HandleFunc("/rprt", rprtMainPage).Methods("GET") 69 | // r.HandleFunc("/rprt/3hours", rprt3Hours).Methods("GET") 70 | // r.HandleFunc("/rprt/last", rprtLast).Methods("GET") 71 | // r.HandleFunc("/rprt/last-critical", rprtLastCritical).Methods("GET") 72 | // r.HandleFunc("/rprt/g/{group}", rprtGroup).Methods("GET") 73 | // r.HandleFunc("/rprt/g/{group}/last", rprtGroupLast).Methods("GET") 74 | // r.HandleFunc("/rprt/g/{group}/last-critical", rprtGroupLastCritical).Methods("GET") 75 | // r.HandleFunc("/zabbix", zabbixStatus).Methods("GET", "HEAD") // text report for all groups to Zabbix 76 | // r.HandleFunc("/mon/status/{group}", zabbixStatus).Methods("GET", "HEAD") // text report for selected group to Zabbix 77 | // r.HandleFunc("/mon/status/{group}/{stream}", zabbixStatus).Methods("GET", "HEAD") // text report for selected group to Zabbix 78 | 79 | // Zabbix autodiscovery protocol (JSON) 80 | // https://www.zabbix.com/documentation/ru/2.0/manual/discovery/low_level_discovery 81 | // new API 82 | 83 | /* Misc static data. Unauthorized access allowed. 84 | */ 85 | r.Handle("/css/{{name}}.css", http.FileServer(http.Dir("bootstrap"))).Methods("GET", "HEAD") 86 | r.Handle("/js/{{name}}.js", http.FileServer(http.Dir("bootstrap"))).Methods("GET", "HEAD") 87 | r.Handle("/{{name}}.png", http.FileServer(http.Dir("pics"))).Methods("GET", "HEAD") 88 | r.Handle("/favicon.ico", http.FileServer(http.Dir("pics"))).Methods("GET", "HEAD") 89 | fmt.Printf("Listen for API connections at %s\n", cfg.ListenHTTP) 90 | srv := &http.Server{ 91 | Addr: cfg.ListenHTTP, 92 | Handler: r, 93 | ReadTimeout: 30 * time.Second, 94 | } 95 | srv.ListenAndServe() 96 | } 97 | 98 | func rootAPI(res http.ResponseWriter, req *http.Request, vars map[string]string) { 99 | data := make(map[string]interface{}) 100 | data["title"] = cfg.Stubs.Name 101 | data["monState"] = StatsGlobals.MonitoringState 102 | data["totalMonPoints"] = StatsGlobals.TotalMonitoringPoints 103 | data["totalHLSMonPoints"] = StatsGlobals.TotalHLSMonitoringPoints 104 | data["totalHDSMonPoints"] = StatsGlobals.TotalHDSMonitoringPoints 105 | data["totalHTTPMonPoints"] = StatsGlobals.TotalHTTPMonitoringPoints 106 | Page.ExecuteTemplate(res, "index", data) 107 | } 108 | 109 | // Webhandler. Возвращает text/plain значение ошибки для выбранных группы и канала. 110 | func monError(res http.ResponseWriter, req *http.Request, vars map[string]string) { 111 | res.Header().Set("Server", SERVER) 112 | res.Header().Set("Content-Type", "text/plain") 113 | 114 | if vars["group"] != "" && vars["stream"] != "" { 115 | if !StatsGlobals.MonitoringState { 116 | switch vars["astype"] { // пока мониторинг остановлен, считаем, что всё ок 117 | case "str": 118 | res.Write([]byte("success")) 119 | case "int": 120 | res.Write([]byte("0")) 121 | } 122 | return 123 | } 124 | if result, err := LoadLastResult(Key{vars["group"], vars["stream"]}); err == nil { 125 | switch vars["astype"] { 126 | case "str": 127 | res.Write([]byte(StreamErr2String(result.ErrType))) 128 | case "int": 129 | res.Write([]byte(strconv.Itoa(int(result.ErrType)))) 130 | } 131 | } else { // пока проверки не проводились, считаем, что всё ок. Чего зря беспокоиться? 132 | switch vars["astype"] { 133 | case "str": 134 | res.Write([]byte("success")) 135 | case "int": 136 | res.Write([]byte("0")) 137 | } 138 | } 139 | } else { 140 | http.Error(res, "Bad parameters in query.", http.StatusBadRequest) 141 | } 142 | } 143 | 144 | // Webhandler. Возвращает text/plain значение ошибки для выбранных группы и канала. 145 | // Если ошибка ниже заданного мин.уровня, то выдаётся 0=OK, если в границах указанных уровней, то 1=PROBLEM, 146 | // если выше макс.уровня, то 2=FATAL 147 | func monErrorLevel(res http.ResponseWriter, req *http.Request, vars map[string]string) { 148 | res.Header().Set("Server", SERVER) 149 | res.Header().Set("Content-Type", "text/plain") 150 | 151 | if vars["group"] != "" && vars["stream"] != "" && vars["fromerrlevel"] != "" && vars["uptoerrlevel"] != "" { 152 | if !StatsGlobals.MonitoringState { 153 | res.Write([]byte("0")) // пока мониторинг остановлен, считаем, что всё ок 154 | return 155 | } 156 | if result, err := LoadLastResult(Key{vars["group"], vars["stream"]}); err == nil { 157 | cur := result.ErrType 158 | switch { 159 | case cur <= String2StreamErr(vars["fromerrlevel"]): 160 | res.Write([]byte("0")) // OK 161 | case cur >= String2StreamErr(vars["uptoerrlevel"]): 162 | res.Write([]byte("2")) // FATAL 163 | default: 164 | res.Write([]byte("1")) // PROBLEM 165 | } 166 | } else { 167 | res.Write([]byte("0")) // пока проверки не проводились, считаем, что всё ок. Чего зря беспокоиться? 168 | //http.Error(res, "Result not found. Probably stream not tested yet.", http.StatusNotFound) 169 | } 170 | } else { 171 | http.Error(res, "Bad parameters in query.", http.StatusBadRequest) 172 | } 173 | } 174 | 175 | // func rprtMainPage(res http.ResponseWriter, req *http.Request) { 176 | // res.Header().Set("Server", SERVER) 177 | // res.Write(ReportMainPage()) 178 | // } 179 | 180 | // func rprtGroupAll(res http.ResponseWriter, req *http.Request) { 181 | // res.Header().Set("Server", SERVER) 182 | // res.Write([]byte("Сводный отчёт по группам.")) 183 | // } 184 | 185 | // func rprtGroup(res http.ResponseWriter, req *http.Request) { 186 | // res.Header().Set("Server", SERVER) 187 | // res.Write([]byte("HLS Probe at service.")) 188 | // } 189 | 190 | // // Group errors report 191 | // func rprtGroupLast(res http.ResponseWriter, req *http.Request) { 192 | // res.Header().Set("Server", SERVER) 193 | // res.Write(ReportLast(mux.Vars(req), false)) 194 | // } 195 | 196 | // // Group errors report 197 | // func rprtGroupLastCritical(res http.ResponseWriter, req *http.Request) { 198 | // res.Header().Set("Server", SERVER) 199 | // res.Write(ReportLast(mux.Vars(req), true)) 200 | // } 201 | 202 | // // Group errors report 203 | // func rprt3Hours(res http.ResponseWriter, req *http.Request) { 204 | // res.Header().Set("Server", SERVER) 205 | // res.Write(Report3Hours(mux.Vars(req))) 206 | // } 207 | 208 | // // Group errors report 209 | // func rprtLast(res http.ResponseWriter, req *http.Request) { 210 | // res.Header().Set("Server", SERVER) 211 | // res.Write(ReportLast(mux.Vars(req), false)) 212 | // } 213 | 214 | // // Group errors report 215 | // func rprtLastCritical(res http.ResponseWriter, req *http.Request) { 216 | // res.Header().Set("Server", SERVER) 217 | // res.Write(ReportLast(mux.Vars(req), true)) 218 | // } 219 | 220 | // Zabbix integration 221 | // func zabbixStatus(res http.ResponseWriter, req *http.Request) { 222 | // res.Header().Set("Server", SERVER) 223 | // res.Write(ZabbixStatus(mux.Vars(req))) 224 | // } 225 | 226 | // Zabbix integration (with cfg curried) 227 | func zabbixDiscovery() func(http.ResponseWriter, *http.Request, map[string]string) { 228 | return func(res http.ResponseWriter, req *http.Request, vars map[string]string) { 229 | res.Write(ZabbixDiscoveryWeb(vars)) 230 | } 231 | } 232 | 233 | func expvarHandler(w http.ResponseWriter, r *http.Request, vars map[string]string) { 234 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 235 | fmt.Fprintf(w, "{\n") 236 | first := true 237 | expvar.Do(func(kv expvar.KeyValue) { 238 | if !first { 239 | fmt.Fprintf(w, ",\n") 240 | } 241 | first = false 242 | fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) 243 | }) 244 | fmt.Fprintf(w, "\n}\n") 245 | } 246 | 247 | // Wrapper for all HTTP handlers. 248 | // Does authorization and preparation of headers. 249 | func HandleHTTP(f func(http.ResponseWriter, *http.Request, map[string]string)) func(http.ResponseWriter, *http.Request) { 250 | var user string 251 | 252 | handler := func(resp http.ResponseWriter, req *http.Request) { 253 | resp.Header().Set("Server", SURFER) 254 | if cfg.User != "" && cfg.Pass != "" { 255 | user = checkAuth(req) 256 | if user != "" { 257 | resp.Header().Set("X-Authenticated-Username", user) 258 | } else { 259 | requireAuth(resp, req, nil) 260 | return 261 | } 262 | } 263 | vars := mux.Vars(req) 264 | f(resp, req, vars) 265 | } 266 | return handler 267 | } 268 | 269 | // Handler for unauthorized access. 270 | func requireAuth(w http.ResponseWriter, r *http.Request, v map[string]string) { 271 | w.Header().Set("WWW-Authenticate", `Basic realm="`+SURFER+`"`) 272 | w.WriteHeader(401) 273 | w.Write([]byte("401 Unauthorized\n")) 274 | } 275 | 276 | /* 277 | TODO адаптировать вместо использования модуля auth 278 | 279 | Checks the username/password combination from the request. Returns 280 | either an empty string (authentication failed) or the name of the 281 | authenticated user. 282 | 283 | Supports MD5 and SHA1 password entries 284 | */ 285 | func checkAuth(r *http.Request) string { 286 | var passwd string 287 | 288 | s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) 289 | if len(s) != 2 || s[0] != "Basic" { 290 | return "" 291 | } 292 | 293 | b, err := base64.StdEncoding.DecodeString(s[1]) 294 | if err != nil { 295 | return "" 296 | } 297 | pair := strings.SplitN(string(b), ":", 2) 298 | if len(pair) != 2 { 299 | return "" 300 | } 301 | if pair[0] == cfg.User { 302 | passwd = cfg.Pass 303 | } else { 304 | return "" 305 | } 306 | if passwd[:5] == "{SHA}" { 307 | d := sha1.New() 308 | d.Write([]byte(pair[1])) 309 | if passwd[5:] != base64.StdEncoding.EncodeToString(d.Sum(nil)) { 310 | return "" 311 | } 312 | } else { 313 | e := NewMD5Entry(passwd) 314 | if e == nil { 315 | return "" 316 | } 317 | if passwd != string(MD5Crypt([]byte(pair[1]), e.Salt, e.Magic)) { 318 | return "" 319 | } 320 | } 321 | return pair[0] 322 | } 323 | -------------------------------------------------------------------------------- /http-client.go: -------------------------------------------------------------------------------- 1 | // HTTP client with timeouts. 2 | // Client from standard library lacks support of timeouts then we use this one. 3 | // Code taken from https://gist.github.com/dmichael/5710968 4 | // and also explained here http://stackoverflow.com/questions/16895294/how-to-set-timeout-for-http-get-requests-in-golang 5 | package main 6 | 7 | import ( 8 | "net" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type HTTPConfig struct { 14 | ConnectTimeout time.Duration 15 | ReadWriteTimeout time.Duration 16 | } 17 | 18 | func TimeoutDialer(config *HTTPConfig) func(net, addr string) (c net.Conn, err error) { 19 | return func(netw, addr string) (net.Conn, error) { 20 | conn, err := net.DialTimeout(netw, addr, config.ConnectTimeout) 21 | if err != nil { 22 | return nil, err 23 | } 24 | conn.SetDeadline(time.Now().Add(config.ReadWriteTimeout)) 25 | tcp_conn := conn.(*net.TCPConn) 26 | tcp_conn.SetLinger(0) // because we have mass connects/disconnects 27 | return conn, nil 28 | } 29 | } 30 | 31 | func NewTimeoutClient(args ...interface{}) *http.Client { 32 | // Default configuration 33 | config := &HTTPConfig{ 34 | ConnectTimeout: 1 * time.Second, 35 | ReadWriteTimeout: 1 * time.Second, 36 | } 37 | 38 | // merge the default with user input if there is one 39 | if len(args) == 1 { 40 | timeout := args[0].(time.Duration) 41 | config.ConnectTimeout = timeout 42 | config.ReadWriteTimeout = timeout 43 | } 44 | 45 | if len(args) == 2 { 46 | config.ConnectTimeout = args[0].(time.Duration) 47 | config.ReadWriteTimeout = args[1].(time.Duration) 48 | } 49 | 50 | return &http.Client{ 51 | Transport: &http.Transport{ 52 | Dial: TimeoutDialer(config), 53 | }, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | // Everything logging 2 | package main 3 | 4 | import ( 5 | "bufio" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | const TimeFormat = "2006-01-02 15:04:05" 13 | 14 | type LogMessage struct { 15 | Severity Severity 16 | Stream 17 | Result 18 | } 19 | 20 | func LogKeeper(verbose bool) { 21 | var skip error 22 | var logw *bufio.Writer 23 | 24 | defer func() { 25 | if r := recover(); r != nil { 26 | fmt.Println("Logger trace:", r) 27 | } 28 | }() 29 | 30 | logq = make(chan LogMessage, 1024) 31 | logf, skip := os.Create(cfg.ErrorLog) 32 | if skip == nil { 33 | logw = bufio.NewWriter(logf) 34 | fmt.Printf("Error log: %s\n", cfg.ErrorLog) 35 | } else { 36 | println("Can't create file for error log. Error logging to file skiped.") 37 | } 38 | 39 | for { 40 | timeout := make(chan bool, 1) 41 | go func() { 42 | time.Sleep(2 * time.Second) 43 | timeout <- true 44 | }() 45 | 46 | select { 47 | case msg := <-logq: 48 | if skip == nil { 49 | logw.WriteString(msg.Started.Format(TimeFormat)) 50 | logw.WriteRune(' ') 51 | switch msg.Severity { 52 | case WARNING: 53 | logw.WriteString("warning") 54 | case ERROR: 55 | logw.WriteString("error") 56 | } 57 | logw.WriteString(": ") 58 | logw.WriteString(StreamErr2String(msg.Result.ErrType)) 59 | logw.WriteRune(' ') 60 | logw.WriteString(strconv.Itoa(msg.HTTPCode)) 61 | logw.WriteRune(' ') 62 | logw.WriteString(strconv.FormatInt(msg.ContentLength, 10)) 63 | logw.WriteRune(' ') 64 | logw.WriteString(msg.Elapsed.String()) 65 | logw.WriteRune(' ') 66 | logw.WriteString(msg.Group) 67 | logw.WriteString(": ") 68 | logw.WriteString(msg.Name) 69 | logw.WriteRune('\n') 70 | } 71 | case <-timeout: 72 | if skip == nil { 73 | _ = logw.Flush() 74 | } 75 | } 76 | } 77 | } 78 | 79 | func Log(severity Severity, stream Stream, taskres Result) { 80 | logq <- LogMessage{Severity: severity, Stream: stream, Result: taskres} 81 | } 82 | -------------------------------------------------------------------------------- /monitor-prober.go: -------------------------------------------------------------------------------- 1 | // Probers for dif 2 | package main 3 | 4 | import ( 5 | "expvar" 6 | "fmt" 7 | "github.com/grafov/bcast" 8 | "github.com/grafov/m3u8" 9 | "net/url" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // Probe HTTP without additional protocol parsing. 15 | // SaveStats timeouts and bad statuses. 16 | func SimpleProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) { 17 | var result *Result 18 | 19 | defer func() { 20 | if r := recover(); r != nil { 21 | fmt.Println("trace dumped in HTTP prober:", r) 22 | } 23 | }() 24 | 25 | for { 26 | queueCount := debugvars.Get("http-tasks-queue") 27 | queueCount.(*expvar.Int).Set(int64(len(tasks))) 28 | task := <-tasks 29 | if time.Now().Before(task.TTL) { 30 | result = ExecHTTP(task) 31 | debugvars.Add("http-tasks-done", 1) 32 | } else { 33 | result = TaskExpired(task) 34 | debugvars.Add("http-tasks-expired", 1) 35 | } 36 | task.ReplyTo <- result 37 | } 38 | } 39 | 40 | // TODO к реализации 41 | // Probe HTTP with additional checks for Widevine. 42 | // Really now only http-range check supported. 43 | func WidevineProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) { 44 | var result *Result 45 | 46 | defer func() { 47 | if r := recover(); r != nil { 48 | fmt.Println("trace dumped in Widevine prober:", r) 49 | } 50 | }() 51 | 52 | for { 53 | queueCount := debugvars.Get("wv-tasks-queue") 54 | queueCount.(*expvar.Int).Set(int64(len(tasks))) 55 | task := <-tasks 56 | if time.Now().Before(task.TTL) { 57 | result = ExecHTTP(task) 58 | debugvars.Add("wv-tasks-done", 1) 59 | } else { 60 | result = TaskExpired(task) 61 | debugvars.Add("wv-tasks-expired", 1) 62 | } 63 | task.ReplyTo <- result 64 | } 65 | } 66 | 67 | // HTTP Live Streaming support. 68 | // Parse and probe M3U8 playlists (multi- and single bitrate) 69 | // and report time statistics and errors 70 | func CupertinoProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) { 71 | var result *Result 72 | 73 | defer func() { 74 | if r := recover(); r != nil { 75 | fmt.Println("trace dumped in HLS prober:", r) 76 | } 77 | }() 78 | 79 | queueCount := debugvars.Get("hls-tasks-queue") 80 | queueCount.(*expvar.Int).Set(int64(len(tasks))) 81 | for { 82 | task := <-tasks 83 | if time.Now().Before(task.TTL) { 84 | result = ExecHTTP(task) 85 | if result.ErrType < ERROR_LEVEL && result.HTTPCode < 400 && result.ContentLength > 0 { 86 | playlist, listType, err := m3u8.Decode(result.Body, true) 87 | if err != nil { 88 | result.ErrType = BADFORMAT 89 | } else { 90 | switch listType { 91 | case m3u8.MASTER: 92 | m := playlist.(*m3u8.MasterPlaylist) 93 | subresult := make(chan *Result, 24) 94 | mainuri, err := url.Parse(task.URI) 95 | if err != nil { 96 | result.ErrType = UNKERR 97 | goto End 98 | } 99 | for _, variant := range m.Variants { 100 | uri, err := url.Parse(variant.URI) 101 | if err != nil { 102 | subresult <- &Result{Task: &Task{Tid: task.Tid, Stream: Stream{variant.URI, HLS, task.Name, task.Title, task.Group}}, ErrType: BADURI, Started: time.Now()} 103 | continue 104 | } 105 | var suburi string 106 | if uri.IsAbs() { // absolute URI 107 | suburi = variant.URI 108 | } else { // relative URI 109 | if variant.URI[0] == '/' { // from the root 110 | suburi = fmt.Sprintf("%s://%s%s", mainuri.Scheme, mainuri.Host, variant.URI) 111 | } else { // last element 112 | splitted := strings.Split(task.URI, "/") 113 | splitted[len(splitted)-1] = variant.URI 114 | suburi = strings.Join(splitted, "/") 115 | } 116 | } 117 | subtask := &Task{Tid: task.Tid, Stream: Stream{suburi, HLS, task.Name, task.Title, task.Group}, ReadBody: task.ReadBody, TTL: task.TTL} 118 | go func(subtask *Task) { 119 | subresult <- ExecHTTP(subtask) 120 | }(subtask) 121 | } 122 | taskCount := len(m.Variants) 123 | for taskCount > 0 { 124 | select { 125 | case data := <-subresult: 126 | result.SubResults = append(result.SubResults, data) 127 | case <-time.After(60 * time.Second): 128 | } 129 | taskCount-- 130 | } 131 | case m3u8.MEDIA: 132 | p := playlist.(*m3u8.MediaPlaylist) 133 | p.Encode().String() 134 | default: 135 | result.ErrType = BADFORMAT 136 | } 137 | } 138 | } 139 | debugvars.Add("hls-tasks-done", 1) 140 | } else { 141 | result = TaskExpired(task) 142 | debugvars.Add("hls-tasks-expired", 1) 143 | } 144 | End: 145 | task.ReplyTo <- result 146 | debugvars.Add("hls-tasks-done", 1) 147 | } 148 | } 149 | 150 | // HTTP Dynamic Streaming prober. 151 | // Parse and probe F4M playlists and report time statistics and errors. 152 | func SanjoseProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) { 153 | for { 154 | task := <-tasks 155 | result := ExecHTTP(task) 156 | task.ReplyTo <- result 157 | debugvars.Add("hds-tasks-done", 1) 158 | } 159 | } 160 | 161 | // Parse and probe media chunk 162 | // and report time statistics and errors 163 | func MediaProber(ctl *bcast.Group, streamType StreamType, taskq chan *Task, debugvars *expvar.Map) { 164 | for { 165 | time.Sleep(20 * time.Second) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /monitor.go: -------------------------------------------------------------------------------- 1 | // Stream parsers and keepers. 2 | package main 3 | 4 | import ( 5 | "expvar" 6 | "fmt" 7 | "github.com/grafov/bcast" 8 | "math/rand" 9 | "net/http" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // Run monitors for each stream. 15 | func StreamMonitor() { 16 | var debugvars = expvar.NewMap("streams") 17 | var requestedTasks = expvar.NewInt("requested-tasks") 18 | var queueSizeHLSTasks = expvar.NewInt("hls-tasks-queue") 19 | var executedHLSTasks = expvar.NewInt("hls-tasks-done") 20 | var expiredHLSTasks = expvar.NewInt("hls-tasks-expired") 21 | var queueSizeHDSTasks = expvar.NewInt("hds-tasks-queue") 22 | var executedHDSTasks = expvar.NewInt("hds-tasks-done") 23 | var expiredHDSTasks = expvar.NewInt("hds-tasks-expired") 24 | var queueSizeHTTPTasks = expvar.NewInt("http-tasks-queue") 25 | var executedHTTPTasks = expvar.NewInt("http-tasks-done") 26 | var expiredHTTPTasks = expvar.NewInt("http-tasks-expired") 27 | var queueSizeWVTasks = expvar.NewInt("wv-tasks-queue") 28 | var executedWVTasks = expvar.NewInt("wv-tasks-done") 29 | var expiredWVTasks = expvar.NewInt("wv-tasks-expired") 30 | var hlscount, hdscount, wvcount, httpcount int 31 | var hlsprobecount int 32 | 33 | debugvars.Set("requested-tasks", requestedTasks) 34 | debugvars.Set("hls-tasks-queue", queueSizeHLSTasks) 35 | debugvars.Set("hls-tasks-done", executedHLSTasks) 36 | debugvars.Set("hls-tasks-expired", expiredHLSTasks) 37 | debugvars.Set("hds-tasks-queue", queueSizeHDSTasks) 38 | debugvars.Set("hds-tasks-done", executedHDSTasks) 39 | debugvars.Set("hds-tasks-expired", expiredHDSTasks) 40 | debugvars.Set("http-tasks-queue", queueSizeHTTPTasks) 41 | debugvars.Set("http-tasks-done", executedHTTPTasks) 42 | debugvars.Set("http-tasks-expired", expiredHTTPTasks) 43 | debugvars.Set("wv-tasks-queue", queueSizeWVTasks) 44 | debugvars.Set("wv-tasks-done", executedWVTasks) 45 | debugvars.Set("wv-tasks-expired", expiredWVTasks) 46 | 47 | ctl := bcast.NewGroup() 48 | go ctl.Broadcasting(0) 49 | go Heartbeat(ctl) 50 | 51 | // запуск проберов и потоков 52 | for gname, gdata := range cfg.GroupParams { 53 | switch gdata.Type { 54 | case HLS: 55 | gtasks := make(chan *Task) 56 | for i := 0; i < gdata.Probers; i++ { 57 | go CupertinoProber(ctl, gtasks, debugvars) 58 | hlsprobecount++ 59 | } 60 | gchunktasks := make(chan *Task) 61 | for i := 0; i < gdata.MediaProbers; i++ { 62 | go MediaProber(ctl, HLS, gchunktasks, debugvars) 63 | } 64 | for _, stream := range *cfg.GroupStreams[gname] { 65 | go StreamBox(ctl, stream, HLS, gtasks, debugvars) 66 | hlscount++ 67 | } 68 | case HDS: 69 | gtasks := make(chan *Task) 70 | for i := 0; i < gdata.Probers; i++ { 71 | go SanjoseProber(ctl, gtasks, debugvars) 72 | } 73 | gchunktasks := make(chan *Task) 74 | for i := 0; i < gdata.MediaProbers; i++ { 75 | go MediaProber(ctl, HDS, gchunktasks, debugvars) 76 | } 77 | for _, stream := range *cfg.GroupStreams[gname] { 78 | go StreamBox(ctl, stream, HDS, gtasks, debugvars) 79 | hdscount++ 80 | } 81 | case HTTP: 82 | gtasks := make(chan *Task) 83 | for i := 0; i < gdata.Probers; i++ { 84 | go SimpleProber(ctl, gtasks, debugvars) 85 | } 86 | for _, stream := range *cfg.GroupStreams[gname] { 87 | go StreamBox(ctl, stream, HTTP, gtasks, debugvars) 88 | httpcount++ 89 | } 90 | case WV: 91 | gtasks := make(chan *Task) 92 | for i := 0; i < gdata.Probers; i++ { 93 | go WidevineProber(ctl, gtasks, debugvars) 94 | } 95 | for _, stream := range *cfg.GroupStreams[gname] { 96 | go StreamBox(ctl, stream, WV, gtasks, debugvars) 97 | wvcount++ 98 | } 99 | } 100 | } 101 | 102 | if hlsprobecount > 0 { 103 | fmt.Printf("%d HLS probers started.\n", hlsprobecount) 104 | } else { 105 | println("No HLS probers started.") 106 | } 107 | // if cfg.TotalProbersHDS > 0 { 108 | // fmt.Printf("%d HDS probers started.\n", cfg.TotalProbersHDS) 109 | // } else { 110 | // println("No HDS probers started.") 111 | // } 112 | // if cfg.TotalProbersHTTP > 0 { 113 | // fmt.Printf("%d HTTP probers started.\n", cfg.TotalProbersHTTP) 114 | // } else { 115 | // println("No HTTP probers started.") 116 | // } 117 | // if cfg.TotalProbersWV > 0 { 118 | // fmt.Printf("%d Widevine VOD probers started.\n", cfg.TotalProbersWV) 119 | // } else { 120 | // println("No Widevine probers started.") 121 | // } 122 | if hlscount > 0 { 123 | StatsGlobals.TotalHLSMonitoringPoints = hlscount 124 | fmt.Printf("%d HLS monitors started.\n", hlscount) 125 | } else { 126 | println("No HLS monitors started.") 127 | } 128 | if hdscount > 0 { 129 | StatsGlobals.TotalHDSMonitoringPoints = hdscount 130 | fmt.Printf("%d HDS monitors started.\n", hdscount) 131 | } else { 132 | println("No HDS monitors started.") 133 | } 134 | if httpcount > 0 { 135 | StatsGlobals.TotalHTTPMonitoringPoints = httpcount 136 | fmt.Printf("%d HTTP monitors started.\n", httpcount) 137 | } else { 138 | println("No HTTP monitors started.") 139 | } 140 | if wvcount > 0 { 141 | StatsGlobals.TotalWVMonitoringPoints = wvcount 142 | fmt.Printf("%d Widevine monitors started.\n", wvcount) 143 | } else { 144 | println("No Widevine monitors started.") 145 | } 146 | 147 | StatsGlobals.TotalMonitoringPoints = hlscount + hdscount + httpcount + wvcount 148 | } 149 | 150 | // Мониторинг и статистика групп потоков. 151 | func GroupBox(ctl *bcast.Group, group string, streamType StreamType, taskq chan *Task, debugvars *expvar.Map) { 152 | } 153 | 154 | // Container keep single stream properties and regulary make tasks for appropriate probers. 155 | func StreamBox(ctl *bcast.Group, stream Stream, streamType StreamType, taskq chan *Task, debugvars *expvar.Map) { 156 | var checkCount uint64 // число прошедших проверок 157 | var addSleepToBrokenStream time.Duration 158 | var tid int64 = time.Now().Unix() // got increasing offset on each program start 159 | var min, max int 160 | var command Command 161 | var online bool = false 162 | var stats Stats 163 | 164 | defer func() { 165 | if r := recover(); r != nil { 166 | fmt.Printf("Stream %s trace: %s\n", stream.Name, r) 167 | } 168 | }() 169 | 170 | task := &Task{Stream: stream, ReplyTo: make(chan *Result)} 171 | switch streamType { 172 | case HTTP: 173 | task.ReadBody = false 174 | case HLS: 175 | task.ReadBody = true 176 | case HDS: 177 | task.ReadBody = true 178 | case WV: 179 | task.ReadBody = false 180 | default: 181 | task.ReadBody = false 182 | } 183 | ctlrcv := ctl.Join() // управление мониторингом 184 | timer := time.Tick(3 * time.Second) 185 | 186 | for { 187 | select { 188 | case recv := <-ctlrcv.In: 189 | command = recv.(Command) 190 | switch command { 191 | case START_MON: 192 | online = true 193 | case STOP_MON: 194 | online = false 195 | } 196 | case <-timer: 197 | SaveStats(stream, stats) 198 | default: 199 | if !online { 200 | time.Sleep(1 * time.Second) 201 | continue 202 | } 203 | max = int(cfg.Params(stream.Group).TimeBetweenTasks) 204 | min = int(cfg.Params(stream.Group).TimeBetweenTasks / 4. * 3.) 205 | time.Sleep(time.Duration(rand.Intn(max-min)+min)*time.Second + addSleepToBrokenStream) // randomize streams order 206 | tid++ 207 | task.Tid = tid 208 | task.TTL = time.Now().Add(time.Duration(cfg.Params(stream.Group).TaskTTL * time.Second)) 209 | stats.Checks++ // TODO potentially overflow 210 | taskq <- task 211 | debugvars.Add("requested-tasks", 1) 212 | result := <-task.ReplyTo 213 | if result.ErrType == TTLEXPIRED { 214 | continue 215 | } else { 216 | checkCount++ 217 | if checkCount > 144 { 218 | fmt.Printf("Repeated %d times %s\n", checkCount, task.Name) 219 | } 220 | } 221 | 222 | for _, subres := range result.SubResults { 223 | subres.Pid = result 224 | go SaveResult(stream, *subres) 225 | } 226 | go SaveResult(stream, *result) 227 | 228 | max = int(cfg.Params(stream.Group).CheckBrokenTime) 229 | min = int(cfg.Params(stream.Group).CheckBrokenTime / 4. * 3.) 230 | 231 | switch { 232 | // permanent error, not a timeout: 233 | case result.ErrType > CRITICAL_LEVEL, result.ErrType == TTLEXPIRED: 234 | addSleepToBrokenStream = time.Duration(rand.Intn(max-min)+min) * time.Second 235 | // works ok: 236 | case result.ErrType == SUCCESS: 237 | addSleepToBrokenStream = 0 238 | default: 239 | addSleepToBrokenStream = 0 240 | } 241 | 242 | if result.ErrType != TTLEXPIRED { 243 | if result.ErrType >= WARNING_LEVEL { 244 | go Log(ERROR, stream, *result) 245 | } else { 246 | if result.Elapsed >= cfg.Params(stream.Group).VerySlowWarningTimeout*time.Second { 247 | result.ErrType = VERYSLOW 248 | go Log(WARNING, stream, *result) 249 | } else if result.Elapsed >= cfg.Params(stream.Group).SlowWarningTimeout*time.Second { 250 | result.ErrType = SLOW 251 | go Log(WARNING, stream, *result) 252 | } 253 | } 254 | } 255 | } 256 | } 257 | } 258 | 259 | // Check & report internet availability. Stop all probers when sample internet resources not available. 260 | // Refs to config option ``samples``. 261 | func Heartbeat(ctl *bcast.Group) { 262 | var previous bool 263 | 264 | ctlsnr := ctl.Join() 265 | 266 | time.Sleep(1 * time.Second) 267 | 268 | for { 269 | for _, uri := range cfg.Samples { 270 | client := NewTimeoutClient(12*time.Second, 6*time.Second) 271 | req, err := http.NewRequest("HEAD", uri, nil) 272 | if err != nil { 273 | fmt.Println("Internet not available. All checks stopped.") 274 | StatsGlobals.MonitoringState = false 275 | continue 276 | } 277 | _, err = client.Do(req) 278 | if err != nil { 279 | StatsGlobals.MonitoringState = false 280 | continue 281 | } 282 | StatsGlobals.MonitoringState = true 283 | } 284 | if previous != StatsGlobals.MonitoringState { 285 | if StatsGlobals.MonitoringState { 286 | ctlsnr.Send(START_MON) 287 | fmt.Println("Internet Ok. Monitoring started.") 288 | } else { 289 | ctlsnr.Send(STOP_MON) 290 | fmt.Println("Internet not available. Monitoring stopped.") 291 | } 292 | } 293 | previous = StatsGlobals.MonitoringState 294 | time.Sleep(4 * time.Second) 295 | } 296 | } 297 | 298 | // Helper for expired tasks. Return result with TTL Expired status. 299 | func TaskExpired(task *Task) *Result { 300 | result := &Result{Started: time.Now(), Elapsed: 0 * time.Second} 301 | result.ContentLength = -1 302 | result.ErrType = TTLEXPIRED 303 | return result 304 | } 305 | 306 | // Helper. Execute stream check task and return result with check status. 307 | func ExecHTTP(task *Task) *Result { 308 | defer func() { 309 | if r := recover(); r != nil { 310 | fmt.Println("trace dumped in ExecHTTP:", r) 311 | } 312 | }() 313 | 314 | result := &Result{Task: task, Started: time.Now(), Elapsed: 0 * time.Second} 315 | if !strings.HasPrefix(task.URI, "http://") && !strings.HasPrefix(task.URI, "https://") { 316 | result.ErrType = BADURI 317 | result.HTTPCode = 0 318 | result.HTTPStatus = "" 319 | result.ContentLength = -1 320 | return result 321 | } 322 | client := NewTimeoutClient(cfg.Params(task.Group).ConnectTimeout*time.Second, cfg.Params(task.Group).RWTimeout*time.Second) 323 | req, err := http.NewRequest("GET", task.URI, nil) // TODO разделить метод по проберам cfg.Params(task.Group).MethodHTTP 324 | if err != nil { 325 | fmt.Println(err) 326 | result.ErrType = BADURI 327 | result.HTTPCode = 0 328 | result.HTTPStatus = "" 329 | result.ContentLength = -1 330 | return result 331 | } 332 | req.Header.Set("User-Agent", UserAgent()) 333 | resp, err := client.Do(req) 334 | result.Elapsed = time.Since(result.Started) 335 | if err != nil { 336 | fmt.Printf("Connect timeout %s: %v\n", result.Elapsed, err) 337 | if result.Elapsed > cfg.Params(task.Group).ConnectTimeout*time.Second { 338 | result.ErrType = CTIMEOUT 339 | } else { 340 | result.ErrType = REFUSED 341 | } 342 | result.HTTPCode = 0 343 | result.HTTPStatus = "" 344 | result.ContentLength = -1 345 | return result 346 | } 347 | result.HTTPCode = resp.StatusCode 348 | if resp.StatusCode < 200 || resp.StatusCode >= 400 { 349 | result.ErrType = BADSTATUS 350 | } 351 | result.HTTPStatus = resp.Status 352 | result.ContentLength = resp.ContentLength 353 | result.Headers = resp.Header 354 | if task.ReadBody { 355 | result.RealContentLength, err = result.Body.ReadFrom(resp.Body) 356 | if err != nil { 357 | result.ErrType = BODYREAD 358 | } 359 | } 360 | resp.Body.Close() 361 | if result.RealContentLength > 0 && result.ContentLength != result.RealContentLength { 362 | result.ErrType = BADLENGTH 363 | } 364 | return result 365 | } 366 | 367 | // Helper. Verify HLS specific things. 368 | func verifyHLS(cfg *Config, task *Task, result *Result) { 369 | defer func() { 370 | if r := recover(); r != nil { 371 | fmt.Println("trace dumped in HLS parser:", r) 372 | result.ErrType = HLSPARSER 373 | } 374 | }() 375 | } 376 | 377 | // Ограничивать число запросов в ед.времени на ip 378 | // func RateLimiter() { 379 | 380 | // } 381 | -------------------------------------------------------------------------------- /pics/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafov/streamsurfer/da3ff432f6da1b01b77856ca877c9ef3741d2b23/pics/favicon.ico -------------------------------------------------------------------------------- /pics/logo-64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafov/streamsurfer/da3ff432f6da1b01b77856ca877c9ef3741d2b23/pics/logo-64.png -------------------------------------------------------------------------------- /reports.go: -------------------------------------------------------------------------------- 1 | // OBSOLETED by webui-report.go 2 | // Web reports generator 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "github.com/hoisie/mustache" // TODO migrate back to native golang templates 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | type ReportData struct { 13 | Vars map[string]string 14 | TableData []map[string]string // array of table rows 15 | } 16 | 17 | func ReportMainPage() []byte { 18 | return []byte(ReportMainPageTemplate) 19 | } 20 | 21 | // Errors for 3 last hours for all groups 22 | // TODO number of hours make variable 23 | func Report3Hours(vars map[string]string) []byte { 24 | var values []map[string]string 25 | var page string 26 | 27 | ErrHistory.RLock() 28 | defer ErrHistory.RUnlock() 29 | 30 | now := time.Now() 31 | curhour := now.Format("06010215") 32 | // h2ago := now.Add(-2 * time.Hour).Format("06010215") 33 | //h3ago := now.Add(-3 * time.Hour).Format("06010215") 34 | tmptbl := make(map[string]map[string]map[string]string) 35 | 36 | for key, val := range ErrHistory.count { 37 | errtype := "" 38 | if key.Curhour == curhour { 39 | switch key.ErrType { 40 | case SLOW, VERYSLOW: 41 | errtype = "sw" 42 | case BADSTATUS: 43 | errtype = "bs" 44 | case BADURI: 45 | errtype = "bu" 46 | case RTIMEOUT: 47 | errtype = "rt" 48 | case CTIMEOUT: 49 | errtype = "ct" 50 | case REFUSED: 51 | errtype = "cr" 52 | default: 53 | continue // count only errors listed above 54 | } 55 | if _, exists := tmptbl[key.Group]; !exists { 56 | tmptbl[key.Group] = make(map[string]map[string]string) 57 | } 58 | if _, exists := tmptbl[key.Group][key.Name]; !exists { 59 | tmptbl[key.Group][key.Name] = make(map[string]string) 60 | } 61 | tmptbl[key.Group][key.Name][errtype] = strconv.FormatInt(int64(val), 10) 62 | tmptbl[key.Group][key.Name]["group"] = key.Group 63 | tmptbl[key.Group][key.Name]["name"] = key.Name 64 | tmptbl[key.Group][key.Name]["uri"] = key.URI 65 | switch { // how much errors per hour? 66 | case val == 1: 67 | tmptbl[key.Group][key.Name][fmt.Sprintf("%s-severity", errtype)] = "info" 68 | case val > 1 && val <= 6: 69 | tmptbl[key.Group][key.Name][fmt.Sprintf("%s-severity", errtype)] = "warning" 70 | case val > 6: 71 | tmptbl[key.Group][key.Name][fmt.Sprintf("%s-severity", errtype)] = "error" 72 | default: 73 | tmptbl[key.Group][key.Name][fmt.Sprintf("%s-severity", errtype)] = "success" 74 | } 75 | } 76 | } 77 | 78 | for _, val := range tmptbl { 79 | for _, counter := range val { 80 | values = append(values, counter) 81 | } 82 | } 83 | page = mustache.Render(Report3HoursTemplate, ReportData{TableData: values}) 84 | 85 | return []byte(page) 86 | } 87 | 88 | // Report last state of reported streams. Only failures counted. 89 | func ReportLast(vars map[string]string, critical bool) []byte { 90 | var values []map[string]string 91 | var page string 92 | 93 | ReportedStreams.RLock() 94 | defer ReportedStreams.RUnlock() 95 | 96 | if _, exists := vars["group"]; exists { // report for selected group 97 | for _, value := range ReportedStreams.data[vars["group"]] { 98 | rprtLastAddRow(&values, value, critical) 99 | } 100 | page = mustache.Render(ReportGroupLastTemplate, ReportData{Vars: vars, TableData: values}) 101 | } else { // report for all groups 102 | for _, group := range ReportedStreams.data { 103 | for _, value := range group { 104 | rprtLastAddRow(&values, value, critical) 105 | } 106 | } 107 | page = mustache.Render(ReportLastTemplate, ReportData{TableData: values}) 108 | } 109 | 110 | return []byte(page) 111 | } 112 | 113 | // Helper. For "last*" report. 114 | func rprtLastAddRow(values *[]map[string]string, value StreamStats, critical bool) { 115 | var severity, report string 116 | 117 | switch { 118 | case value.Last.ErrType > WARNING_LEVEL && value.Last.ErrType < ERROR_LEVEL: 119 | if critical { 120 | return 121 | } 122 | severity = "warning" 123 | case value.Last.ErrType > ERROR_LEVEL && value.Last.ErrType < CRITICAL_LEVEL: 124 | if critical { 125 | return 126 | } 127 | severity = "error" 128 | case value.Last.ErrType > CRITICAL_LEVEL: 129 | severity = "critical" 130 | default: 131 | return 132 | } 133 | if critical { 134 | report = "last-critical" 135 | } else { 136 | report = "last" 137 | } 138 | *values = append(*values, map[string]string{ 139 | "uri": value.Stream.URI, 140 | "name": value.Stream.Name, 141 | "group": value.Stream.Group, 142 | "status": value.Last.HTTPStatus, 143 | "contentlength": strconv.FormatInt(value.Last.ContentLength, 10), 144 | "started": value.Last.Started.Format(TimeFormat), 145 | "elapsed": strconv.FormatFloat(value.Last.Elapsed.Seconds(), 'f', 3, 64), 146 | "error": StreamErrText(value.Last.ErrType), 147 | "totalerrs": strconv.FormatUint(uint64(value.Last.TotalErrs), 10), 148 | "severity": severity, 149 | "report": report, // report name 150 | }) 151 | } 152 | -------------------------------------------------------------------------------- /sample-config: -------------------------------------------------------------------------------- 1 | # -*-mode:yaml-*- 2 | http-api-listen: 0.0.0.0:8888 3 | http-api-user: root 4 | http-api-pass: $base64-hash-of-password-here 5 | user-agents: # http://www.enterpriseios.com/wiki/UserAgent http://www.useragentstring.com/pages/Mobile%20Browserlist/ 6 | - Mozilla/5.0 (Linux; U; Android 2.3; en-us) AppleWebKit/999+ (KHTML, like Gecko) Safari/999.9 7 | - Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 8 | - Mozilla/5.0 (Linux; U; Android 4.0.3; ru-ru; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30 9 | - Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 10 | - Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 11 | - Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3 12 | stubs: 13 | name: "HLS monitor" 14 | zabbix: 15 | discovery-path: /var/lib/streamsurfer/discovery.json 16 | name-template: "{{.Name}}" 17 | title-template: "{{.Title}}" 18 | unmortal: 19 | - http://google.com 20 | - http://ya.ru 21 | db-expired: 24 # hours 22 | defaults: 23 | probers: 2 24 | media-probers: 4 25 | time-between-tasks: 6 # sec 26 | check-broken-time: 40 # sec 27 | connect-timeout: 15 # sec 28 | rw-timeout: 20 # sec 29 | slow-warning-timeout: 6 # sec 30 | very-slow-warning-timeout: 12 # sec 31 | task-ttl: 300 # sec 32 | error-log: /var/log/streamsurfer/error.log 33 | http-method: get 34 | one-segment: true 35 | groups: 36 | our-new-vod: 37 | type: hls 38 | streams: 39 | - http:///30/42/sd_2013_podpolnaja_imperia_04_02_film/vod.m3u8 40 | - http:///56/95/sd_2013_podpolnaja_imperia_04_06_film/vod.m3u8 41 | - http:///46/73/sd_2013_podpolnaja_imperia_04_01_film/vod.m3u8 42 | - http:///03/66/sd_2013_podpolnaja_imperia_04_03_film/vod.m3u8 43 | - http:///71/51/sd_2013_podpolnaja_imperia_04_05_film/vod.m3u8 44 | - http:///75/54/sd_2013_podpolnaja_imperia_04_07_film/vod.m3u8 45 | - http:///15/46/sd_2014_game_of_thrones_04_02_film/vod.m3u8 46 | probers: 3 47 | media-probers: 2 48 | time-between-tasks: 20 # sec 49 | check-broken-time: 30 # sec 50 | connect-timeout: 15 # sec 51 | rw-timeout: 20 # sec 52 | slow-warning-timeout: 6 # sec 53 | very-slow-warning-timeout: 12 # sec 54 | task-ttl: 300 # sec 55 | error-log: /var/log/streamsurfer/error.log 56 | http-method: get 57 | one-segment: true 58 | parse-method: /[0-9]+/[0-9]+/([-_a-zA-Z0-9.]+)/vod.m3u8 59 | f451-live: 60 | type: hls 61 | streams-uri: http:///content_monitoring/?content=live 62 | probers: 4 63 | media-probers: 3 64 | time-between-tasks: 6 # sec 65 | check-broken-time: 40 # sec 66 | connect-timeout: 35 # sec 67 | rw-timeout: 60 # sec 68 | slow-warning-timeout: 9 # sec 69 | very-slow-warning-timeout: 18 # sec 70 | task-ttl: 300 # sec 71 | error-log: /var/log/streamsurfer/error.log 72 | http-method: get 73 | one-segment: true 74 | parse-method: /smil:([-_a-zA-Z0-9.]+)/playlist.m3u8 75 | -------------------------------------------------------------------------------- /source-loader.go: -------------------------------------------------------------------------------- 1 | // Load configuration file. 2 | package main 3 | 4 | import ( 5 | "bufio" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "launchpad.net/goyaml" 10 | "net/http" 11 | "regexp" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | // Exported config structure 18 | type Config struct { 19 | StreamsHLS []Stream 20 | StreamsHDS []Stream 21 | StreamsHTTP []Stream 22 | StreamsWV []Stream 23 | GroupsHLS map[string]string // map[group]group 24 | GroupsHDS map[string]string // map[group]group 25 | GroupsHTTP map[string]string // map[group]group 26 | GroupsWV map[string]string // map[group]group 27 | Groups map[Group][]string 28 | Samples []string 29 | Params Params 30 | GroupParams map[string]Params 31 | m sync.Mutex 32 | } 33 | 34 | // Internal config structure parsed from YAML 35 | type configYAML struct { 36 | StreamsHLS map[string][]string `yaml:"hls-streams,omitempty"` // HLS parsing 37 | StreamsHDS map[string][]string `yaml:"hds-streams,omitempty"` // HDS parsing 38 | StreamsHTTP map[string][]string `yaml:"http-streams,omitempty"` // plain HTTP checks 39 | StreamsWV map[string][]string `yaml:"wv-streams,omitempty"` // HTTP checks with additional WV VOD checks 40 | GetStreamsHLS []string `yaml:"get-hls-streams,omitempty"` // load remote HLS checks configuration 41 | GetStreamsHDS []string `yaml:"get-hds-streams,omitempty"` // load remote HLS checks configuration 42 | GetStreamsHTTP []string `yaml:"get-http-streams,omitempty"` // load remote HTTP checks configuration 43 | GetStreamsWV []string `yaml:"get-wv-streams,omitempty"` // load remote WV VOD checks configuration 44 | Samples []string `yaml:"samples"` 45 | Params Params `yaml:"params"` 46 | GroupParams map[string]Params `yaml:"group-params,omitempty"` // parameters per group 47 | Stubs StubValues `yaml:"stubs"` 48 | } 49 | 50 | type Params struct { 51 | ProbersHTTP uint `yaml:"http-probers,omitempty"` // num of 52 | ProbersHLS uint `yaml:"hls-probers,omitempty"` // num of 53 | ProbersHDS uint `yaml:"hds-probers,omitempty"` // num of 54 | ProbersWV uint `yaml:"wv-probers,omitempty"` // num of 55 | MediaProbers uint `yaml:"media-probers,omitempty"` // num of 56 | CheckBrokenTime uint `yaml:"check-broken-time"` // ms 57 | ConnectTimeout time.Duration `yaml:"connect-timeout,omitempty"` // sec 58 | RWTimeout time.Duration `yaml:"rw-timeout,omitempty"` // sec 59 | SlowWarningTimeout time.Duration `yaml:"slow-warning-timeout,omitempty"` // sec 60 | VerySlowWarningTimeout time.Duration `yaml:"very-slow-warning-timeout,omitempty"` // sec 61 | TimeBetweenTasks time.Duration `yaml:"time-between-tasks,omitempty"` // sec 62 | TaskTTL time.Duration `yaml:"task-ttl,omitempty"` // sec 63 | TryOneSegment bool `yaml:"one-segment,omitempty"` 64 | MethodHTTP string `yaml:"http-method,omitempty"` // GET, HEAD 65 | ListenHTTP string `yaml:"http-api-listen,omitempty"` 66 | ErrorLog string `yaml:"error-log,omitempty"` 67 | Zabbix Zabbix `yaml:"zabbix,omitempty"` 68 | ParseName string `yaml:"parse-name,omitempty"` // regexp for alternative method of title/name parsing from the URL 69 | User string `yaml:"user,omitempty"` 70 | Pass string `yaml:"pass,omitempty"` 71 | } 72 | 73 | type Zabbix struct { 74 | DiscoveryPath string `yaml:"discovery-path,omitempty"` 75 | DiscoveryGroups []string `yaml:"discovery-groups,omitempty"` 76 | StreamTemplate string `yaml:"stream-template,omitempty"` 77 | } 78 | 79 | // custom values for HTML-templates and reports 80 | type StubValues struct { 81 | Name string `yaml:"name,omitempty"` 82 | } 83 | 84 | // глобальный конфиг 85 | var cfg *Config 86 | 87 | // XXX выпилить 88 | //var NameParseMode string // regexp for parse name/title from the stream URL, by default string splitted by space and http:// part becomes URL other part becomes stream name 89 | 90 | func ReadConfig(confile string) *Config { 91 | var cfg = &configYAML{} 92 | 93 | // Hardcoded defaults: 94 | cfg.Stubs = StubValues{Name: "Stream Surfer"} 95 | // Final config: 96 | Cfg := new(Config) 97 | if confile == "" { 98 | confile = "/etc/streamsurfer/default.yaml" 99 | } 100 | data, e := ioutil.ReadFile(FullPath(confile)) 101 | if e == nil { 102 | e = goyaml.Unmarshal(data, &cfg) 103 | if e != nil { 104 | print("Config file parsing failed. Hardcoded defaults used.\n") 105 | } 106 | Cfg.Groups = make(map[Group][]string) // TODO свести использование к Cfg.Groups, убрать GroupsHLS, GroupsHDS, GroupsHTTP 107 | Cfg.GroupsHLS = make(map[string]string) 108 | Cfg.GroupsHDS = make(map[string]string) 109 | Cfg.GroupsHTTP = make(map[string]string) 110 | Cfg.GroupsWV = make(map[string]string) 111 | Cfg.Params = cfg.Params 112 | Cfg.Samples = cfg.Samples 113 | Cfg.Params.MethodHTTP = strings.ToUpper(cfg.Params.MethodHTTP) 114 | Stubs = &cfg.Stubs 115 | 116 | Cfg.GroupParams = make(map[string]Params) 117 | Cfg.GroupParams = cfg.GroupParams 118 | 119 | // for group, params := range cfg.GroupParams { 120 | // if 121 | // Cfg.GroupParams[group] = cfg.Params 122 | // fmt.Printf("%s %v\n", group, params) 123 | // // if params.ProbersHTTP != 0 { 124 | // // Cfg.GroupParams[group].ProbersHTTP = params.ProbersHTTP 125 | // // } 126 | // // if params.ProbersHLS != 0 { 127 | // // Cfg.GroupParams[group].ProbersHLS = params.ProbersHLS 128 | // // } 129 | // // if cfg.GroupParams[group].ProbersHDS != 0 { 130 | // // Cfg.GroupParams[group].ProbersHDS = cfg.GroupParams[group].ProbersHDS 131 | // // } 132 | // // if cfg.GroupParams[group].ProbersWV != 0 { 133 | // // Cfg.GroupParams[group].ProbersWV = cfg.GroupParams[group].ProbersWV 134 | // // } 135 | // if params.ParseName != "" { 136 | // Cfg.GroupParams[group].ParseName = params.ParseName 137 | // } 138 | // // if cfg.GroupParams[group].User != "" { 139 | // // Cfg.GroupParams[group].User = cfg.GroupParams[group].User 140 | // // Cfg.GroupParams[group].Pass = cfg.GroupParams[group].Pass 141 | // // } 142 | // } 143 | 144 | for groupName, streamList := range cfg.StreamsHLS { 145 | nameList := addLocalConfig(&Cfg.StreamsHLS, HLS, groupName, streamList) 146 | Cfg.GroupsHLS[groupName] = groupName 147 | Cfg.Groups[Group{HLS, groupName}] = nameList 148 | } 149 | for groupName, streamList := range cfg.StreamsHDS { 150 | nameList := addLocalConfig(&Cfg.StreamsHDS, HDS, groupName, streamList) 151 | Cfg.GroupsHDS[groupName] = groupName 152 | Cfg.Groups[Group{HDS, groupName}] = nameList 153 | } 154 | for groupName, streamList := range cfg.StreamsHTTP { 155 | nameList := addLocalConfig(&Cfg.StreamsHTTP, HTTP, groupName, streamList) 156 | Cfg.GroupsHTTP[groupName] = groupName 157 | Cfg.Groups[Group{HTTP, groupName}] = nameList 158 | } 159 | 160 | for groupName, streamList := range cfg.StreamsWV { 161 | nameList := addLocalConfig(&Cfg.StreamsWV, WV, groupName, streamList) 162 | Cfg.GroupsWV[groupName] = groupName 163 | Cfg.Groups[Group{WV, groupName}] = nameList 164 | } 165 | 166 | if cfg.GetStreamsHLS != nil { 167 | for _, source := range cfg.GetStreamsHLS { 168 | groupURI, groupName := splitName("", source) 169 | remoteUser := "" 170 | remotePass := "" 171 | if _, exists := cfg.GroupParams[groupName]; exists { 172 | remoteUser = cfg.GroupParams[groupName].User 173 | remotePass = cfg.GroupParams[groupName].Pass 174 | } 175 | nameList, err := addRemoteConfig(&Cfg.StreamsHLS, HLS, groupName, groupURI, remoteUser, remotePass) 176 | if err != nil { 177 | fmt.Printf("Load remote config for group \"%s\" (HLS) failed.\n", groupName) 178 | } else { 179 | Cfg.GroupsHLS[groupName] = groupName 180 | Cfg.Groups[Group{HLS, groupName}] = nameList 181 | } 182 | } 183 | } 184 | fmt.Printf("%+v\n", Cfg.GroupParams) 185 | if cfg.GetStreamsHDS != nil { 186 | for _, source := range cfg.GetStreamsHDS { 187 | groupURI, groupName := splitName("", source) 188 | remoteUser := "" 189 | remotePass := "" 190 | if _, exists := cfg.GroupParams[groupName]; exists { 191 | remoteUser = cfg.GroupParams[groupName].User 192 | remotePass = cfg.GroupParams[groupName].Pass 193 | } 194 | nameList, err := addRemoteConfig(&Cfg.StreamsHDS, HDS, groupName, groupURI, remoteUser, remotePass) 195 | if err != nil { 196 | fmt.Printf("Load remote config for group \"%s\" (HDS) failed.\n", groupName) 197 | } else { 198 | Cfg.GroupsHDS[groupName] = groupName 199 | Cfg.Groups[Group{HDS, groupName}] = nameList 200 | } 201 | } 202 | } 203 | if cfg.GetStreamsHTTP != nil { 204 | for _, source := range cfg.GetStreamsHTTP { 205 | groupURI, groupName := splitName("", source) 206 | remoteUser := "" 207 | remotePass := "" 208 | if _, exists := cfg.GroupParams[groupName]; exists { 209 | remoteUser = cfg.GroupParams[groupName].User 210 | remotePass = cfg.GroupParams[groupName].Pass 211 | } 212 | nameList, err := addRemoteConfig(&Cfg.StreamsHTTP, HTTP, groupName, groupURI, remoteUser, remotePass) 213 | if err != nil { 214 | fmt.Printf("Load remote config for group \"%s\" (HTTP) failed.\n", groupName) 215 | } else { 216 | Cfg.GroupsHTTP[groupName] = groupName 217 | Cfg.Groups[Group{HTTP, groupName}] = nameList 218 | } 219 | } 220 | } 221 | if cfg.GetStreamsWV != nil { 222 | for _, source := range cfg.GetStreamsWV { 223 | groupURI, groupName := splitName("", source) 224 | remoteUser := "" 225 | remotePass := "" 226 | if _, exists := cfg.GroupParams[groupName]; exists { 227 | remoteUser = cfg.GroupParams[groupName].User 228 | remotePass = cfg.GroupParams[groupName].Pass 229 | } 230 | nameList, err := addRemoteConfig(&Cfg.StreamsWV, WV, groupName, groupURI, remoteUser, remotePass) 231 | if err != nil { 232 | fmt.Printf("Load remote config for group \"%s\" (WV) failed.\n", groupName) 233 | } else { 234 | Cfg.GroupsWV[groupName] = groupName 235 | Cfg.Groups[Group{WV, groupName}] = nameList 236 | } 237 | } 238 | } 239 | 240 | for group, groupParams := range cfg.GroupParams { 241 | if groupParams.ProbersHLS == 0 { 242 | groupParams.ProbersHLS = cfg.Params.ProbersHLS 243 | } 244 | if groupParams.ProbersHDS == 0 { 245 | groupParams.ProbersHDS = cfg.Params.ProbersHDS 246 | } 247 | if groupParams.ProbersHTTP == 0 { 248 | groupParams.ProbersHTTP = cfg.Params.ProbersHTTP 249 | } 250 | if groupParams.ProbersWV == 0 { 251 | groupParams.ProbersWV = cfg.Params.ProbersWV 252 | } 253 | if groupParams.MediaProbers == 0 { 254 | groupParams.MediaProbers = cfg.Params.MediaProbers 255 | } 256 | if groupParams.CheckBrokenTime == 0 { 257 | groupParams.CheckBrokenTime = cfg.Params.CheckBrokenTime 258 | } 259 | if groupParams.ConnectTimeout == 0 { 260 | groupParams.ConnectTimeout = cfg.Params.ConnectTimeout 261 | } 262 | if groupParams.RWTimeout == 0 { 263 | groupParams.RWTimeout = cfg.Params.RWTimeout 264 | } 265 | if groupParams.SlowWarningTimeout == 0 { 266 | groupParams.SlowWarningTimeout = cfg.Params.SlowWarningTimeout 267 | } 268 | if groupParams.VerySlowWarningTimeout == 0 { 269 | groupParams.VerySlowWarningTimeout = cfg.Params.VerySlowWarningTimeout 270 | } 271 | if groupParams.TimeBetweenTasks == 0 { 272 | groupParams.TimeBetweenTasks = cfg.Params.TimeBetweenTasks 273 | } 274 | if groupParams.TaskTTL == 0 { 275 | groupParams.TaskTTL = cfg.Params.TaskTTL 276 | } 277 | if groupParams.TryOneSegment { 278 | groupParams.TryOneSegment = cfg.Params.TryOneSegment 279 | } 280 | if groupParams.ListenHTTP == "" { 281 | groupParams.ListenHTTP = cfg.Params.ListenHTTP 282 | } 283 | if groupParams.ErrorLog == "" { 284 | groupParams.ErrorLog = cfg.Params.ErrorLog 285 | } 286 | Cfg.GroupParams[group] = groupParams 287 | } 288 | } else { 289 | print("Config file not found. Hardcoded defaults used.\n") 290 | } 291 | // fmt.Printf("HLS: %+v\n\n", Cfg.StreamsHLS) 292 | //fmt.Printf("HTTP: %+v\n\n", Cfg.GroupsHTTP) 293 | 294 | return Cfg 295 | } 296 | 297 | // Helper. Split stream link to URI and Name parts. 298 | // Supported both cases: titleuri and urititle 299 | // URI must be prepended by http:// or https:// 300 | func splitName(group, source string) (uri string, name string) { 301 | source = strings.TrimSpace(source) 302 | sep := regexp.MustCompile("htt(p|ps)://") 303 | loc := sep.FindStringIndex(source) 304 | if loc != nil { 305 | if loc[0] == 0 { // uri title 306 | splitted := strings.SplitN(source, " ", 2) 307 | if len(splitted) > 1 { 308 | name = strings.TrimSpace(splitted[1]) 309 | } 310 | uri = strings.TrimSpace(splitted[0]) 311 | } else { // title uri 312 | name = strings.TrimSpace(source[0:loc[0]]) 313 | uri = source[loc[0]:] 314 | } 315 | if name == "" { 316 | name = uri 317 | } 318 | } 319 | if group == "" { 320 | return // для парсинга имён групп в YAML-конфиге не применяется парсинг по регвырам 321 | } 322 | if params, err := cfg.Params4(group); err == nil && params.ParseName != "" { 323 | re := regexp.MustCompile(params.ParseName) 324 | vals := re.FindStringSubmatch(uri) 325 | if len(vals) > 1 { 326 | name = vals[1] 327 | } 328 | } 329 | return 330 | } 331 | 332 | // Helper. Parse config of 333 | func addLocalConfig(dest *[]Stream, streamType StreamType, group string, sources []string) []string { 334 | var nameList []string 335 | 336 | for _, source := range sources { 337 | uri, name := splitName(group, source) 338 | *dest = append(*dest, Stream{URI: uri, Type: streamType, Name: name, Group: group}) 339 | nameList = append(nameList, name) 340 | } 341 | 342 | return nameList 343 | } 344 | 345 | // Helper. Get remote list of streams. 346 | func addRemoteConfig(dest *[]Stream, streamType StreamType, group string, uri, remoteUser, remotePass string) ([]string, error) { 347 | var nameList []string 348 | 349 | defer func() error { 350 | if r := recover(); r != nil { 351 | return errors.New(fmt.Sprintf("Can't get remote config for (%s) %s %s", streamType, group, uri)) 352 | } 353 | return nil 354 | }() 355 | 356 | client := NewTimeoutClient(10*time.Second, 10*time.Second) 357 | req, err := http.NewRequest("GET", uri, nil) 358 | if err != nil { 359 | return nil, err 360 | } 361 | result, err := client.Do(req) 362 | if err == nil { 363 | body := bufio.NewReader(result.Body) 364 | for { 365 | line, err := body.ReadString('\n') 366 | if err != nil { 367 | break 368 | } 369 | uri, name := splitName(group, line) 370 | nameList = append(nameList, name) 371 | *dest = append(*dest, Stream{URI: uri, Type: streamType, Name: name, Group: group}) 372 | } 373 | } 374 | return nameList, err 375 | } 376 | 377 | // TODO Dynamic configuration without program restart. 378 | // Elder. 379 | func ConfigKeeper(confile string) { 380 | cfg = new(Config) 381 | cfg = ReadConfig(confile) 382 | select {} // TODO reload config by query 383 | } 384 | 385 | func (cfg *Config) Params4(group string) (*Params, error) { 386 | cfg.m.Lock() 387 | defer cfg.m.Unlock() 388 | if val, ok := cfg.GroupParams[group]; !ok { 389 | return &cfg.Params, nil 390 | } else { 391 | return &val, nil 392 | } 393 | } 394 | -------------------------------------------------------------------------------- /stats.go: -------------------------------------------------------------------------------- 1 | // The code keeps streams statistics and program internal statistics. 2 | // Statistics output to files and to JSON HTTP API. 3 | package main 4 | 5 | import ( 6 | "errors" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var ( 12 | logq chan LogMessage 13 | //statIn chan Stats 14 | ) 15 | 16 | var StatsGlobals = struct { 17 | TotalMonitoringPoints int 18 | TotalHTTPMonitoringPoints int 19 | TotalWVMonitoringPoints int 20 | TotalHLSMonitoringPoints int 21 | TotalHDSMonitoringPoints int 22 | MonitoringState bool // is inet available? 23 | }{} 24 | 25 | // Обмен данными с хранителем статистики. 26 | var ( 27 | statIn chan StatInQuery 28 | statOut chan StatOutQuery 29 | resultIn chan ResultInQuery 30 | resultOut chan ResultOutQuery 31 | errorsOut chan OutQuery 32 | ) 33 | 34 | // // Streams statistics 35 | // var ReportedStreams = struct { 36 | // sync.RWMutex 37 | // data map[string]map[string]StreamStats // map [group][stream]stream-state 38 | // }{data: make(map[string]map[string]StreamStats)} 39 | 40 | var ErrHistory = struct { 41 | sync.RWMutex 42 | count map[ErrHistoryKey]uint 43 | }{count: make(map[ErrHistoryKey]uint)} 44 | 45 | var ErrTotalHistory = struct { 46 | sync.RWMutex 47 | count map[ErrTotalHistoryKey]uint 48 | }{count: make(map[ErrTotalHistoryKey]uint)} 49 | 50 | type Incident struct { 51 | Id uint64 52 | Key 53 | Error *Result 54 | } 55 | 56 | // Elder 57 | // Gather results history and statistics from all streamboxes 58 | func StatKeeper() { 59 | //var results map[Key][]Result = make(map[Key][]Result) 60 | var stats map[Key]Stats = make(map[Key]Stats) 61 | // var errors map[Key]map[time.Time]ErrType = make(map[Key]map[time.Time]ErrType) 62 | 63 | statIn = make(chan StatInQuery, 8192) // receive stats 64 | statOut = make(chan StatOutQuery, 8) // send stats 65 | resultIn = make(chan ResultInQuery, 4096) 66 | resultOut = make(chan ResultOutQuery, 8) 67 | errorsOut = make(chan OutQuery, 8) 68 | 69 | // storage maintainance period 70 | ///timer := time.Tick(12 * time.Second) 71 | 72 | for { 73 | select { 74 | case state := <-statIn: // receive new statitics data for saving 75 | stats[Key{state.Stream.Group, state.Stream.Name}] = state.Last 76 | 77 | case key := <-statOut: 78 | if val, ok := stats[key.Key]; ok { 79 | key.ReplyTo <- val 80 | } else { 81 | key.ReplyTo <- Stats{} 82 | } 83 | 84 | case state := <-resultIn: // incoming results from streamboxes 85 | //results[Key{state.Stream.Group, state.Stream.Name}] = append(results[Key{state.Stream.Group, state.Stream.Name}], state.Last) 86 | RedKeepResult(Key{state.Stream.Group, state.Stream.Name}, state.Last.Started, state.Last) 87 | if state.Last.ErrType > WARNING_LEVEL { 88 | RedKeepError(Key{state.Stream.Group, state.Stream.Name}, state.Last.Started, state.Last.ErrType) 89 | } 90 | // delete(errors, Key{state.Stream.Group, state.Stream.Name}) 91 | 92 | case key := <-resultOut: 93 | data, err := RedLoadResults(key.Key, time.Now().Add(-6*time.Hour), time.Now()) 94 | if err != nil { 95 | key.ReplyTo <- nil 96 | } else { 97 | key.ReplyTo <- data 98 | } 99 | 100 | case key := <-errorsOut: // get error list by streams 101 | // result := make(map[time.Time]ErrType) 102 | 103 | data, err := RedLoadErrors(key.Key, key.From, key.To) 104 | if err != nil { 105 | key.ReplyTo <- nil 106 | } else { 107 | key.ReplyTo <- data 108 | } 109 | 110 | default: // expired keys cleanup 111 | RemoveExpiredErrors(cfg.ExpireDurationDB) 112 | RemoveExpiredResults(cfg.ExpireDurationDB) 113 | } 114 | } 115 | } 116 | 117 | // Put result of probe task to statistics. 118 | func SaveStats(stream Stream, last Stats) { 119 | statIn <- StatInQuery{Stream: stream, Last: last} 120 | } 121 | 122 | // Получить состояние по последней проверке. 123 | func LoadStats(key Key) Stats { 124 | result := make(chan Stats) 125 | statOut <- StatOutQuery{Key: key, ReplyTo: result} 126 | data := <-result 127 | return data 128 | } 129 | 130 | // Put result of probe task to statistics. 131 | func SaveResult(stream Stream, last Result) { 132 | resultIn <- ResultInQuery{Stream: stream, Last: last} 133 | } 134 | 135 | // Получить состояние по последней проверке. 136 | func LoadLastResult(key Key) (KeepedResult, error) { 137 | result := make(chan []KeepedResult) 138 | resultOut <- ResultOutQuery{Key: key, ReplyTo: result} 139 | data := <-result 140 | if data != nil { 141 | return data[len(data)-1], nil 142 | } else { 143 | return KeepedResult{}, errors.New("result not found") 144 | } 145 | } 146 | 147 | // Получить статистику по каналу за всё время наблюдения. 148 | // TODO добавить фильтр по времени (начиная с) 149 | func LoadHistoryResults(key Key) ([]KeepedResult, error) { 150 | result := make(chan []KeepedResult) 151 | resultOut <- ResultOutQuery{Key: key, ReplyTo: result} 152 | data := <-result 153 | if data != nil { 154 | return data, nil 155 | } else { 156 | return nil, errors.New("result not found") 157 | } 158 | } 159 | 160 | func LoadHistoryErrors(key Key, from time.Duration) (map[time.Time]ErrType, error) { 161 | result := make(chan interface{}) 162 | errorsOut <- OutQuery{Key: key, From: time.Now().Add(-from), To: time.Now(), ReplyTo: result} 163 | data := <-result 164 | if data != nil { 165 | return data.(map[time.Time]ErrType), nil 166 | } else { 167 | return nil, errors.New("result not found") 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /streamsurfer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Stream Surfer is prober and monitor for HTTP video streaming. 3 | This file defines main entry points of the program. 4 | 5 | Copyleft 2013-2014 Alexander I.Grafov aka Axel 6 | 7 | This program is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | 20 | ॐ तारे तुत्तारे तुरे स्व 21 | */ 22 | 23 | package main 24 | 25 | /* TODO 26 | 27 | 1. Для ассетов выдавать общий список одной страницей (плюс по группам). Ассет, график статусов за последнее время. 28 | 2. Страница инфы по отдельному ассету. 29 | 30 | */ 31 | 32 | import ( 33 | "flag" 34 | "fmt" 35 | "io/ioutil" 36 | "os" 37 | "os/signal" 38 | ) 39 | 40 | const ( 41 | SURFER = "Stream Surfer" 42 | VERSION = "0.3-dev" 43 | ) 44 | 45 | var build_date string 46 | var Stubs = &configStub{} 47 | 48 | func main() { 49 | var confname = flag.String("config", "", "alternative configuration file") 50 | var verbose = flag.Bool("verbose", true, "verbose output of logs") 51 | 52 | defer func() { 53 | if r := recover(); r != nil { 54 | fmt.Println("Stream Surfer trace dumped:", r) 55 | if err := ioutil.WriteFile(FullPath("~/streamsurfer.trace"), r.([]byte), 0644); err != nil { 56 | fmt.Println("Can't write trace file!") 57 | } 58 | } 59 | }() 60 | 61 | fmt.Printf("Stream Surfer vers. %s (build %s)\n", VERSION, build_date) 62 | flag.Parse() 63 | 64 | InitConfig(*confname) 65 | 66 | go ConfigKeeper() 67 | go InitStorage() 68 | go LogKeeper(*verbose) // collect program logs and write them to file 69 | go StatKeeper() // collect probe statistics for report builders 70 | go StreamMonitor() // probe logic 71 | go HttpAPI() // control API 72 | go ProblemAnalyzer() // analyze problems related to groups of channels 73 | go ProblemReporter() // report problems to email 74 | 75 | terminate := make(chan os.Signal) 76 | signal.Notify(terminate, os.Interrupt) 77 | <-terminate 78 | fmt.Println("...probe service interrupted.") 79 | } 80 | -------------------------------------------------------------------------------- /streamsurfer_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestSplitName(t *testing.T) { 9 | name, uri := splitName("the title http://uri") 10 | fmt.Printf("%s | %s\n", name, uri) 11 | name, uri = splitName("the title https://uri uri2part") 12 | fmt.Printf("%s | %s\n", name, uri) 13 | name, uri = splitName("http://uri the title") 14 | fmt.Printf("%s | %s\n", name, uri) 15 | name, uri = splitName("incorrect uri") 16 | fmt.Printf("%s | %s\n", name, uri) 17 | } 18 | -------------------------------------------------------------------------------- /structure.go: -------------------------------------------------------------------------------- 1 | // Data structures related to whole program 2 | package main 3 | 4 | import ( 5 | "bytes" 6 | "fmt" 7 | "github.com/grafov/m3u8" 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | const ( 13 | SERVER = "Stream Surfer" 14 | ) 15 | 16 | // Kinds of streams 17 | // Must be in consistence with StreamType2String() and String2StreamType() 18 | const ( 19 | UNKSTREAM StreamType = iota // хрень какая-то 20 | SAMPLE // internet resources for monitor self checks 21 | HTTP // "plain" HTTP 22 | HLS // Apple HTTP Live Streaming 23 | HDS // Adobe HTTP Dynamic Streaming 24 | WV // Widevine VOD 25 | ) 26 | 27 | const ( 28 | INFO Severity = iota 29 | WARNING 30 | ERROR 31 | CRITICAL 32 | ) 33 | 34 | // Error codes (ordered by errors importance). 35 | // If several errors detected then only one with the heaviest weight reported. 36 | // Must be in consistence with String2StreamErr() and StreamErr2String() 37 | const ( 38 | SUCCESS ErrType = iota 39 | DEBUG_LEVEL // Internal debug messages follow below: 40 | TTLEXPIRED // Task was not executed because task TTL expired. StreamSurfer too busy. 41 | HLSPARSER // HLS parser error (debug) 42 | BADREQUEST // Request failed (internal client error) 43 | WARNING_LEVEL // Warnings follow below: 44 | SLOW // SlowWarning threshold on reading server response 45 | VERYSLOW // VerySlowWarning threshold on reading server response 46 | ERROR_LEVEL // Errors follow below: 47 | CTIMEOUT // Timeout on connect 48 | RTIMEOUT // Timeout on read 49 | BADLENGTH // ContentLength value not equal real content length 50 | BODYREAD // Response body read error 51 | CRITICAL_LEVEL // Permanent errors level 52 | REFUSED // Connection refused 53 | BADSTATUS // HTTP Status >= 400 54 | BADURI // Incorret URI format 55 | LISTEMPTY // HLS specific (by m3u8 lib) 56 | BADFORMAT // HLS specific (by m3u8 lib) 57 | UNKERR // хрень какая-то 58 | ) 59 | 60 | // Commands for probers. 61 | const ( 62 | STOP_MON Command = iota 63 | START_MON 64 | RELOAD_CONFIG // TODO made dynamic stream loading and unloading 65 | LOAD_GROUP 66 | LOAD_STREAM 67 | DROP_GROUP 68 | DROP_STREAM 69 | ) 70 | 71 | type Severity uint 72 | type StreamType uint // Type of checked streams 73 | type ErrType uint 74 | type Command uint 75 | 76 | type Group struct { 77 | Type StreamType 78 | Name string 79 | } 80 | 81 | type Stream struct { 82 | URI string 83 | Type StreamType 84 | Name string 85 | Title string // опциональный заголовок = name по умолчанию 86 | Group string 87 | } 88 | 89 | // Stream checking task 90 | type Task struct { 91 | Stream 92 | ReadBody bool 93 | ReplyTo chan *Result 94 | TTL time.Time // valid until the time 95 | Tid int64 // task id (unique for each stream box) 96 | } 97 | 98 | type VariantTask struct { 99 | Task 100 | } 101 | 102 | type ChunkTask struct { 103 | Task 104 | } 105 | 106 | // Stream group 107 | type GroupTask struct { 108 | Type StreamType 109 | Name string 110 | Tasks *Task 111 | ReplyTo chan Result 112 | } 113 | 114 | // Result of task of check streaming 115 | type Result struct { 116 | Task *Task 117 | ErrType ErrType 118 | HTTPCode int // HTTP status code 119 | HTTPStatus string // HTTP status string 120 | ContentLength int64 121 | RealContentLength int64 122 | Headers http.Header 123 | Body bytes.Buffer 124 | Started time.Time // начало исполнения проверки 125 | Elapsed time.Duration // понадобилось времени на задачу 126 | TotalErrs uint 127 | //Meta interface{} // Reference to metainformation about result data (playlist type etc.) 128 | Pid *Result // link to parent check (is nil for top level URLs) 129 | SubResults []*Result // Результаты вложенных проверок (i.e. media playlists for different bitrate of master playlists) 130 | } 131 | 132 | // Results persistently keeped in Redis 133 | type KeepedResult struct { 134 | Tid int64 // all subresults have same task id 135 | Stream 136 | Master bool // is master result? 137 | ErrType ErrType 138 | HTTPCode int // HTTP status code 139 | HTTPStatus string // HTTP status string 140 | ContentLength int64 141 | RealContentLength int64 142 | Headers http.Header 143 | Body []byte 144 | Started time.Time // начало исполнения проверки 145 | Elapsed time.Duration // понадобилось времени на задачу 146 | TotalErrs uint 147 | } 148 | 149 | // StreamBox statistics 150 | type Stats struct { 151 | Checks int64 // checks made 152 | Errors int64 // total errors found 153 | Errors6min int // errors in last 3 min 154 | Errors6hours int // 155 | LastCheck time.Time // last check was at the time 156 | NextCheck time.Time // next check will be at this time 157 | // TODO результаты анализа потока в streambox (анализ перезапускать регулярно по таймеру) 158 | // TODO вынести инфу о потоке потом в отдельную структуру 159 | } 160 | 161 | type MetaHLS struct { 162 | ListType m3u8.ListType // type of analyzed playlist 163 | DeepLinks []string // sublists for analysis 164 | } 165 | 166 | type MetaHDS struct { 167 | ListType m3u8.ListType // XXX type of analyzed playlist 168 | DeepLinks []string // sublists for analysis 169 | } 170 | 171 | // ключ для статистики 172 | type Key struct { 173 | Group string 174 | Name string 175 | } 176 | 177 | // serialised key 178 | func (k *Key) String() string { 179 | return fmt.Sprintf("%s/%s", k.Group, k.Name) 180 | } 181 | 182 | // запросppы на сохранение статистики 183 | type StatInQuery struct { 184 | Stream Stream 185 | Last Stats 186 | } 187 | 188 | // запросы на получение статистики 189 | type StatOutQuery struct { 190 | Key Key 191 | ReplyTo chan Stats 192 | } 193 | 194 | // запросы на сохранение статистики 195 | type ResultInQuery struct { 196 | Stream Stream 197 | Last Result 198 | } 199 | 200 | // запросы на получение статистики 201 | type ResultOutQuery struct { 202 | Key Key 203 | ReplyTo chan []KeepedResult 204 | } 205 | 206 | // common type for queries 207 | type OutQuery struct { 208 | Key Key 209 | From time.Time 210 | To time.Time 211 | ReplyTo chan interface{} 212 | } 213 | 214 | type ErrHistoryKey struct { 215 | Curhour string 216 | ErrType ErrType 217 | Group string 218 | Name string 219 | URI string 220 | } 221 | 222 | type ErrTotalHistoryKey struct { 223 | Curhour string 224 | Group string 225 | Name string 226 | } 227 | 228 | // Values for the webpage 229 | type PageValues struct { 230 | Title string 231 | Data interface{} 232 | } 233 | 234 | // Report about the occured errors 235 | type Report struct { 236 | Error ErrType 237 | Severity Severity 238 | Title string 239 | Body string 240 | RelatedGroups []*Group 241 | RelatedStreams []*Stream 242 | RelatedTasks []*Task 243 | Generated time.Time 244 | } 245 | 246 | // Notes last checked point for the stream. 247 | type CheckPoint struct { 248 | Tid int64 249 | Occured time.Time 250 | OpenedRange *ErrRange // last non closed error range 251 | } 252 | 253 | // Range of errors in the stream. For usage in reports. 254 | type ErrRange struct { 255 | FromTid int64 256 | ToTid int64 257 | Occured time.Time 258 | Discontinued time.Time 259 | Err ErrType 260 | } 261 | 262 | /* 263 | 264 | task1 change tid → isTaskOK 265 | mas isCheckOK = true 266 | med1 isCheckOK = true 267 | *med2 isCheckOK=false isTaskOK=false make forSave, start range 268 | med3 isCheckOK=true isTaskOK=false 269 | 270 | !isTaskOK → continue range 271 | 272 | task2 isTaskOK = true 273 | mas isCheckOK=true 274 | med1 isCheckOK=true 275 | *med2 isCheckOK=false isTaskOK=false make forSave 276 | med3 isCheckOK=true 277 | 278 | !isTaskOK → continue range 279 | 280 | task2 isTaskOK = true 281 | mas isCheckOK=true 282 | med1 isCheckOK=true 283 | *med2 isCheckOK=false isTaskOK=false make forSave 284 | med3 isCheckOK=true 285 | 286 | isTaskOK && forSave → append range 287 | 288 | 289 | 290 | ------------------------------------ 291 | 292 | if prevTid > 0 && prevTid != curTid 293 | if taskOK && forSave 294 | save range 295 | else 296 | taskOK=true 297 | prevTid = curTid 298 | 299 | if prevTid == 0 300 | prevTid = curTid 301 | 302 | if ERR 303 | taskOK=false 304 | prpare forSave (start, stop, tid0, tid1) 305 | 306 | 307 | 308 | */ 309 | -------------------------------------------------------------------------------- /templates.go: -------------------------------------------------------------------------------- 1 | // OBSOLETED by templates/*.tmpl 2 | // Templates for webui 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "html/template" 8 | ) 9 | 10 | const menu = ` 11 |
    12 |
    13 | Stream Surfer 14 | 20 |
    21 |
    22 | ` 23 | 24 | type TestMe struct { 25 | Test int 26 | } 27 | 28 | var ( 29 | tmpltReportStreamInfo = ` 30 | 31 | 32 | 33 | Stream Surfer Reports 34 | 35 | 36 | 37 | 38 | 39 |

    Reports are:

    40 |
      41 | {{range $i, $val := .}} 42 |
    • {{$val}}
      • 43 | {{end}} 44 |
    45 | 46 |
    47 | 48 | ` 49 | 50 | tmpltReportStreamHistory = ` 51 | 52 | 53 | 54 | Stream Surfer Reports 55 | 56 | 57 | 58 | 59 | 60 |

    Errors per 3 hours for all groups

    61 | 62 | 63 | 64 |
    65 | 66 |
    67 | 68 | ` 69 | ) 70 | 71 | /// ниже устаревшие шаблоны /////////////////////////////////////////////////////////////////////////////// 72 | /// vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv 73 | 74 | var ( 75 | ReportMainPageTemplate = fmt.Sprintf(` 76 | 77 | 78 | 79 | Stream Surfer Reports 80 | 81 | 82 | 83 | 84 | 85 | %s 86 |

    Reports are:

    87 | 92 | 93 |
    94 | 95 | `, menu) 96 | 97 | Report3HoursTemplate = fmt.Sprintf(` 98 | 99 | 100 | 101 | Errors per 3 hours for all groups :: Stream Surfer 102 | 103 | 104 | 105 | 106 | 107 | 108 | %s 109 |

    Errors per 3 hours for all groups

    110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | {{#TableData}} 140 | 141 | 142 | 143 | {{sw}} 144 | {{bs}} 145 | {{bu}} 146 | {{rt}} 147 | {{ct}} 148 | {{cr}} 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | {{/TableData}} 163 | 164 |
    GroupNameLast hourTwo hours agoThree hours ago
    SRBSBURTCTSRBSBURTCTSRBSBURTCTCR
    {{group}}{{name}}{{sw2}}{{bs2}}{{bs2}}{{rt2}}{{ct2}}{{cr2}}{{sw3}}{{bs3}}{{bu3}}{{rt3}}{{ct3}}{{cr3}}
    165 |
    166 | Generated by Stream Surfer 167 | 168 | 169 | `, menu) 170 | 171 | ReportLastTemplate = fmt.Sprintf(` 172 | 173 | 174 | 175 | All groups last errors :: Stream Surfer 176 | 177 | 178 | 179 | 180 | 181 | 182 | %s 183 |

    Last errors

    184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | {{#TableData}} 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | {{/TableData}} 210 | 211 |
    GroupNameStatusLengthRequest DurationLast CheckedErrorLast Hour
    {{group}}{{name}}{{status}}{{contentlength}}{{elapsed}}{{started}}{{error}}{{totalerrs}}
    212 |
    213 | Generated by Stream Surfer 214 | 215 | 216 | `, menu) 217 | 218 | ReportGroupLastTemplate = fmt.Sprintf(` 219 | 220 | 221 | 222 | {{#Vars}}{{group}} :: {{/Vars}}Stream Surfer 223 | 224 | 225 | 226 | 227 | 228 | 229 | %s 230 | {{#Vars}}

    {{group}} last errors

    {{/Vars}} 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | {{#TableData}} 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | {{/TableData}} 255 | 256 |
    NameStatusLengthRequest DurationLast CheckedErrorLast Hour
    {{name}}{{status}}{{contentlength}}{{elapsed}}{{started}}{{error}}{{totalerrs}}
    257 |
    258 | Generated by Stream Surfer 259 | 260 | `, menu) 261 | ) 262 | 263 | var IndexPageTemplate = template.Must(template.ParseFiles("html/index.html")) 264 | -------------------------------------------------------------------------------- /templates/activity-index.tmpl: -------------------------------------------------------------------------------- 1 | {{define "activity-index"}} 2 | {{template "page-header" .}} 3 |

    {{.title}}

    4 | 5 | 6 | 7 | 8 | {{range $i, $val := .thead}} 9 | 12 | {{end}} 13 | 14 | 15 | 16 | {{range $i, $row := .tbody}} 17 | {{range $j, $col := $row}} 18 | {{if $j}}{{/* окраска строк по уровню ошибок */}} 19 | {{else}} 20 | {{end}} 21 | {{end}} 22 | 23 | {{end}} 24 | 25 |
    10 | {{$val}} 11 |
    {{$col}}
    26 | {{template "page-footer" .}} 27 | {{end}} -------------------------------------------------------------------------------- /templates/index.tmpl: -------------------------------------------------------------------------------- 1 | {{define "index"}} 2 | 3 | 4 | 5 | {{.title}} :: Stream Surfer 6 | 7 | 8 | 9 | 10 | 11 | 12 |
    13 |
    14 |

    {{.title}}

    15 |

    Monitor is {{if .monState}}running{{else}}stopped{{end}}. {{.totalMonPoints}} streams monitored ({{.totalHLSMonPoints}} HLS, {{.totalHDSMonPoints}} HDS, {{.totalHTTPMonPoints}} HTTP).

    16 |

    Show streams activity

    17 |
    18 | 19 |
    20 |
    21 |

    Control

    22 |

    Donec id elit non mi porta gravida at eget metus. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus. Etiam porta sem malesuada magna mollis euismod. Donec sed odio dui.

    23 |

    Control panel

    24 |
    25 |
    26 |

    Reports

    27 |

    Last reports:

    28 | 33 |

    Reports

    34 |
    35 |
    36 |

    Documentation (online)

    37 |

    Donec sed odio dui. Cras justo odio, dapibus ac facilisis in, egestas eget quam. Vestibulum id ligula porta felis euismod semper. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus.

    38 |

    Open site

    39 |
    40 |
    41 |
    42 |
    43 | {{template "page-footer" .}} 44 | {{end}} -------------------------------------------------------------------------------- /templates/page-footer.tmpl: -------------------------------------------------------------------------------- 1 | {{define "page-footer"}} 2 |
    3 |
    4 | Generated by Stream Surfer. 5 |
    6 | 7 | 8 | {{end}} -------------------------------------------------------------------------------- /templates/page-header.tmpl: -------------------------------------------------------------------------------- 1 | {{define "page-header"}} 2 | 3 | 4 | 5 | {{.title}} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {{template "top-page-menu" .}} 14 | {{end}} -------------------------------------------------------------------------------- /templates/report-index.tmpl: -------------------------------------------------------------------------------- 1 | {{define "report-index"}} 2 | {{template "page-header" .}} 3 |

    Last reports

    4 | 5 | 10 | 11 | 12 | 13 | 14 | 15 | {{range $i, $val := .thead}} 16 | 19 | {{end}} 20 | 21 | 22 | 23 | {{range $i, $row := .tbody}} 24 | 25 | {{range $j, $col := $row}} 26 | 27 | {{end}} 28 | 29 | {{end}} 30 | 31 |
    17 | {{$val}} 18 |
    {{$col}}
    32 | 33 | 34 | 35 | {{template "page-footer" .}} 36 | {{end}} -------------------------------------------------------------------------------- /templates/report-stream-history.tmpl: -------------------------------------------------------------------------------- 1 | {{define "report-stream-history"}} 2 | {{template "page-header" .}} 3 |

    {{.title}}

    4 | 5 | stream info 6 | {{if .history}}show full history{{end}} 7 | {{if .errorsonly}}show errors only{{end}} 8 | 9 |

    History for last 30 minutes

    10 | 11 | 12 | 13 | {{range $i, $val := .thead}} 14 | 17 | {{end}} 18 | 19 | 20 | 21 | {{range $i, $row := .tbody}} 22 | {{range $j, $col := $row}} 23 | {{if $j}}{{/* окраска строк по уровню ошибок */}} 24 | {{else}} 25 | {{end}} 26 | {{end}} 27 | 28 | {{end}} 29 | 30 |
    15 | {{$val}} 16 |
    {{$col}}
    31 | {{template "page-footer" .}} 32 | {{end}} -------------------------------------------------------------------------------- /templates/report-stream-info.tmpl: -------------------------------------------------------------------------------- 1 | {{define "report-stream-info"}} 2 | {{template "page-header" .}} 3 |

    {{.title}}

    4 | 5 | show checks history 6 | show errors only 7 | 8 |

    Stream information

    9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
    Top level URL{{.url}}
    Playlist typeLIVE
    Profiles in master playlist3
    Target duration in a media playlists6s
    Occured chunks duration (min/max)6.000s - 6.000s
    18 | 19 |

    Problem statistics

    20 | For the last 24 hours. 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |
    Slow responses{{.slowcount}}
    Timeouts{{.timeoutcount}}
    HTTP connection errors{{.httpcount}}
    Playlist errors{{.formatcount}}
    29 | 30 |

    Check configuration

    31 | 32 | 33 | 34 | 35 | 36 | 37 |
    Template for checksHLS
    Time between checks15s
    Used probers for the group1
    38 | 39 |

    Attached reports

    40 | Attached all reports where the stream "{{.stream}}" involved. 41 |
      42 |
    • 2014-04-23 15:00 Stream not available up to 30min.
      • 43 |
    • 2014-04-23 14:30 Media playlists not available up to 20min.
      • 44 |
    • 2014-04-23 11:30 Slow responses up to 20min.
      • 45 |
    • 2014-04-22 15:00 Stream not available up to 30min.
      • 46 |
    • 2014-04-22 14:30 Media playlists not available up to 20min.
      • 47 |
    • 2014-04-22 11:30 Slow responses up to 20min.2014-04-21 15:00 Stream not available up to 30min.
      • 49 |
    • 2014-04-21 14:30 Media playlists not available up to 20min.
      • 50 |
    • 2014-04-21 11:30 Slow responses up to 20min. 52 | 53 | {{template "page-footer" .}} 54 | {{end}} -------------------------------------------------------------------------------- /templates/top-page-menu.tmpl: -------------------------------------------------------------------------------- 1 | {{define "top-page-menu"}} 2 |
      3 |
      4 | Stream Surfer 5 | 11 |
      12 |
      13 | {{end}} -------------------------------------------------------------------------------- /webui-report.go: -------------------------------------------------------------------------------- 1 | // Web UI. Reports generator 2 | package main 3 | 4 | import ( 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | var ( // TODO refactore it 13 | кешОшибок1ч map[time.Time]ErrType // быстрое решение для ускорения загрузки кучи данных из редиса 14 | кешТаймер time.Time 15 | ) 16 | 17 | // Initialize reporting subsystem. 18 | func InitReports() { 19 | кешОшибок1ч = make(map[time.Time]ErrType) 20 | } 21 | 22 | // TODO выводить среднее время ответов 23 | func ActivityIndex(res http.ResponseWriter, req *http.Request, vars map[string]string) { 24 | var tbody [][]string 25 | 26 | data := make(map[string]interface{}) 27 | if vars["group"] != "" { 28 | data["title"] = fmt.Sprintf("List of streams for %s", vars["group"]) 29 | } else { 30 | data["title"] = "List of streams" 31 | } 32 | if vars["group"] != "" { 33 | data["thead"] = []string{"Name", "Checks", "Avg. resp.time", "Problems (3 min)", "Problems (last 15 min)", "Problems (last 1 hour)"} 34 | } else { 35 | data["thead"] = []string{"Group", "Name", "Checks", "Avg. resp.time", "Problems (last 3 min)", "Problems (last 15 min)", "Problems (last 1 hour)"} 36 | } 37 | data["isactivity"] = true 38 | for gname := range cfg.GroupParams { 39 | if vars["group"] != "" && gname != strings.ToLower(vars["group"]) { 40 | continue 41 | } 42 | for _, stream := range *cfg.GroupStreams[gname] { 43 | severity := "" 44 | stats := LoadStats(Key{gname, stream.Name}) 45 | hist, err := LoadHistoryErrors(Key{gname, stream.Name}, 1*time.Hour) 46 | errcountLong := 0 47 | if err == nil { 48 | for _, val := range hist { 49 | if val > WARNING_LEVEL { 50 | errcountLong++ 51 | } 52 | } 53 | } 54 | hist, err = LoadHistoryErrors(Key{gname, stream.Name}, 15*time.Minute) 55 | errcountMid := 0 56 | if err == nil { 57 | for _, val := range hist { 58 | if val > WARNING_LEVEL { 59 | errcountMid++ 60 | } 61 | } 62 | } 63 | errcountShort := 0 64 | if len(кешОшибок1ч) > 0 && time.Since(кешТаймер) <= 10*time.Minute { 65 | hist = кешОшибок1ч 66 | err = nil 67 | } else { 68 | hist, err = LoadHistoryErrors(Key{gname, stream.Name}, 3*time.Minute) 69 | } 70 | if err == nil { 71 | кешОшибок1ч = hist 72 | for _, val := range hist { 73 | if val > ERROR_LEVEL { 74 | severity = "error" 75 | } 76 | if val > WARNING_LEVEL { 77 | errcountShort++ 78 | } 79 | } 80 | } 81 | if severity == "" && errcountShort > 0 { 82 | severity = "warning" 83 | } 84 | if vars["group"] != "" { 85 | tbody = append(tbody, []string{ 86 | severity, 87 | href(fmt.Sprintf("/act/%s/%s", gname, stream.Name), stream.Name), 88 | strconv.FormatInt(stats.Checks, 10), 89 | "0", 90 | strconv.Itoa(errcountShort), 91 | strconv.Itoa(errcountMid), 92 | strconv.Itoa(errcountLong)}) 93 | } else { 94 | tbody = append(tbody, []string{ 95 | severity, 96 | href(fmt.Sprintf("/act/%s", gname), gname), 97 | href(fmt.Sprintf("/act/%s/%s", gname, stream.Name), stream.Name), 98 | strconv.FormatInt(stats.Checks, 10), 99 | "0", 100 | strconv.Itoa(errcountShort), 101 | strconv.Itoa(errcountMid), 102 | strconv.Itoa(errcountLong)}) 103 | } 104 | } 105 | } 106 | data["tbody"] = tbody 107 | Page.ExecuteTemplate(res, "activity-index", data) 108 | } 109 | 110 | func ActivityStreamInfo(res http.ResponseWriter, req *http.Request, vars map[string]string) { 111 | fmt.Printf("%+v\n", req) 112 | data := make(map[string]interface{}) 113 | data["title"] = fmt.Sprintf("%s/%s info", vars["group"], vars["stream"]) 114 | data["isactivity"] = true 115 | data["stream"] = vars["stream"] 116 | data["history"] = fmt.Sprintf("/act/%s/%s/history", vars["group"], vars["stream"]) 117 | data["errorsonly"] = fmt.Sprintf("/act/%s/%s/errors", vars["group"], vars["stream"]) 118 | last, err := LoadLastResult(Key{vars["group"], vars["stream"]}) 119 | if err == nil { 120 | data["url"] = last.URI 121 | } 122 | data["slowcount"] = 0 123 | data["timeoutcount"] = 0 124 | data["httpcount"] = 0 125 | data["formatcount"] = 0 126 | hist, err := LoadHistoryErrors(Key{vars["group"], vars["stream"]}, 24*time.Hour) 127 | if err == nil { 128 | for _, val := range hist { 129 | switch val { 130 | case SLOW, VERYSLOW: 131 | data["slowcount"] = data["slowcount"].(int) + 1 132 | case CTIMEOUT, RTIMEOUT: 133 | data["timeoutcount"] = data["timeoutcount"].(int) + 1 134 | case BADLENGTH, BODYREAD, REFUSED, BADSTATUS, BADURI: 135 | data["httpcount"] = data["httpcount"].(int) + 1 136 | case LISTEMPTY, BADFORMAT: 137 | data["formatcount"] = data["formatcount"].(int) + 1 138 | } 139 | } 140 | } 141 | Page.ExecuteTemplate(res, "report-stream-info", data) 142 | } 143 | 144 | func ActivityStreamHistory(res http.ResponseWriter, req *http.Request, vars map[string]string) { 145 | var severity, checktype string 146 | var tbody [][]string 147 | 148 | data := make(map[string]interface{}) 149 | hist, err := LoadHistoryResults(Key{vars["group"], vars["stream"]}) 150 | if err != nil { 151 | http.Error(res, "Stream not found or not tested yet.", http.StatusNotFound) 152 | return 153 | } 154 | if vars["stamp"] != "" { // отобразить подробности по ошибке 155 | for _, val := range hist { 156 | stamp, err := strconv.ParseInt(vars["stamp"], 10, 64) 157 | if err != nil { 158 | goto FullHistory 159 | } 160 | if val.Started == time.Unix(0, stamp) { 161 | if vars["idx"] == "" { 162 | res.Write([]byte(fmt.Sprintf("GET %s\n\n", val.URI))) 163 | val.Headers.Write(res) 164 | res.Write([]byte("\n")) 165 | res.Write(val.Body) 166 | } else { 167 | //idx, err := strconv.Atoi(vars["idx"]) 168 | if err != nil { 169 | goto FullHistory 170 | } 171 | // if len(val.SubResults) >= idx+1 { 172 | // sub := val.SubResults[idx] 173 | // res.Write([]byte(fmt.Sprintf("GET %s\n\n", sub.URI))) 174 | // sub.Headers.Write(res) 175 | // res.Write([]byte("\n")) 176 | // res.Write(sub.Body.Bytes()) 177 | // } 178 | } 179 | return 180 | } 181 | } 182 | } 183 | 184 | FullHistory: 185 | data["title"] = fmt.Sprintf("%s/%s checks history", vars["group"], vars["stream"]) 186 | data["isactivity"] = true 187 | data["stream"] = vars["stream"] 188 | data["thead"] = []string{"Check type", "Date/time", "Check result", "HTTP status", "Time elapsed", "Content length", "Raw result"} 189 | 190 | switch vars["mode"] { 191 | case "history": 192 | data["errorsonly"] = true // fmt.Sprintf("/act/%s/%s/errors", vars["group"], vars["stream"]) 193 | case "errors": 194 | data["history"] = true // fmt.Sprintf("/act/%s/%s/history", vars["group"], vars["stream"]) 195 | } 196 | for i := len(hist) - 1; i >= 0; i-- { 197 | val := (hist)[i] 198 | if vars["mode"] == "errors" && val.ErrType <= WARNING_LEVEL { 199 | continue 200 | } 201 | switch { 202 | case val.ErrType == SUCCESS: 203 | severity = "info" 204 | case val.ErrType <= WARNING_LEVEL: 205 | severity = "warning" 206 | case val.ErrType > WARNING_LEVEL: 207 | severity = "error" 208 | default: 209 | severity = "success" 210 | } 211 | if val.Master { // TODO пофиксить для HTTP/HDS-проверок 212 | checktype = "master" 213 | } else { 214 | checktype = "media" 215 | } 216 | tbody = append(tbody, 217 | []string{severity, 218 | span(checktype, "label"), 219 | val.Started.Format("2006-01-02 15:04:05 -0700"), 220 | StreamErr2String(val.ErrType), 221 | val.HTTPStatus, 222 | val.Elapsed.String(), 223 | strconv.FormatInt(val.ContentLength, 10), 224 | href(fmt.Sprintf("%d/raw", val.Started.UnixNano()), "show raw result")}) 225 | } 226 | data["tbody"] = tbody 227 | Page.ExecuteTemplate(res, "report-stream-history", data) 228 | } 229 | 230 | func ReportIndex(res http.ResponseWriter, req *http.Request, vars map[string]string) { 231 | var tbody [][]string 232 | var severity string 233 | 234 | data := make(map[string]interface{}) 235 | data["title"] = "Available reports" 236 | data["isreport"] = true 237 | data["thead"] = []string{"Date", "Severity", "Title"} 238 | reports := LoadReports() 239 | for _, report := range reports { 240 | switch report.Severity { 241 | case INFO: 242 | severity = span("info", "label label-info") 243 | case WARNING: 244 | severity = span("info", "label label-warning") 245 | case ERROR: 246 | severity = span("error", "label label-important") 247 | case CRITICAL: 248 | severity = span("critical", "label label-important") 249 | } 250 | tbody = append(tbody, 251 | []string{report.Generated.Format("2006-01-02 15:04:05 -0700"), 252 | severity, 253 | report.Title}) 254 | } 255 | data["tbody"] = tbody 256 | Page.ExecuteTemplate(res, "report-index", data) 257 | } 258 | 259 | func ReportStreamErrors(res http.ResponseWriter, req *http.Request, vars map[string]string) { 260 | var tbody [][]string 261 | var severity string 262 | 263 | data := make(map[string]interface{}) 264 | data["title"] = "Available reports" 265 | data["isreport"] = true 266 | data["thead"] = []string{"Date", "Severity", "Title"} 267 | reports := LoadReports() 268 | for _, report := range reports { 269 | switch report.Severity { 270 | case INFO: 271 | severity = span("info", "label label-info") 272 | case WARNING: 273 | severity = span("info", "label label-warning") 274 | case ERROR: 275 | severity = span("error", "label label-important") 276 | case CRITICAL: 277 | severity = span("critical", "label label-important") 278 | } 279 | tbody = append(tbody, 280 | []string{report.Generated.Format("2006-01-02 15:04:05 -0700"), 281 | severity, 282 | report.Title}) 283 | } 284 | data["tbody"] = tbody 285 | Page.ExecuteTemplate(res, "report-stream-info", data) 286 | } 287 | -------------------------------------------------------------------------------- /zabbix.go: -------------------------------------------------------------------------------- 1 | // Integration with Zabbix monitoring tool 2 | package main 3 | 4 | import ( 5 | "bytes" 6 | "encoding/json" 7 | "sort" 8 | "text/template" 9 | ) 10 | 11 | type ZabbixDiscoveryData struct { 12 | Data []map[string]string `json:"data"` 13 | } 14 | 15 | // Data for stream template 16 | type streamTemplateData struct { 17 | Stream 18 | Check string 19 | } 20 | 21 | func ZabbixDiscoveryWeb(vars map[string]string) []byte { 22 | var page []byte 23 | var tmplName, tmplTitle *template.Template 24 | var bufn *bytes.Buffer = new(bytes.Buffer) 25 | var buft *bytes.Buffer = new(bytes.Buffer) 26 | var err error 27 | 28 | sort.Strings(cfg.Zabbix.DiscoveryGroups) 29 | data := new(ZabbixDiscoveryData) 30 | 31 | if cfg.Zabbix.NameTemplate != "" { 32 | tmplName, err = template.New("name").Parse(cfg.Zabbix.NameTemplate) 33 | } 34 | if err != nil || cfg.Zabbix.NameTemplate == "" { 35 | tmplName, _ = template.New("name").Parse("{{.Group}}-{{.Name}}") 36 | 37 | } 38 | if cfg.Zabbix.TitleTemplate != "" { 39 | tmplTitle, err = template.New("title").Parse(cfg.Zabbix.TitleTemplate) 40 | } 41 | if err != nil || cfg.Zabbix.TitleTemplate == "" { 42 | tmplTitle, _ = template.New("title").Parse("{{.Title}}") 43 | 44 | } 45 | 46 | for _, streams := range cfg.GroupStreams { 47 | for _, stream := range *streams { 48 | bufn.Reset() 49 | buft.Reset() 50 | _ = tmplName.Execute(bufn, streamTemplateData{Stream: stream, Check: StreamType2String(stream.Type)}) 51 | _ = tmplTitle.Execute(buft, streamTemplateData{Stream: stream, Check: StreamType2String(stream.Type)}) 52 | if _, exists := vars["group"]; exists { // report for selected group 53 | if stream.Group == vars["group"] { 54 | data.Data = append(data.Data, map[string]string{"{#STREAM}": bufn.String(), "{#TITLE}": buft.String()}) 55 | } 56 | } else if len(cfg.Zabbix.DiscoveryGroups) > 0 { 57 | if sort.SearchStrings(cfg.Zabbix.DiscoveryGroups, stream.Group) == 0 { 58 | data.Data = append(data.Data, map[string]string{"{#STREAM}": bufn.String(), "{#TITLE}": buft.String()}) 59 | } 60 | } else { 61 | data.Data = append(data.Data, map[string]string{"{#STREAM}": bufn.String(), "{#TITLE}": buft.String()}) 62 | } 63 | } 64 | } 65 | 66 | page, _ = json.Marshal(data) 67 | 68 | return page 69 | } 70 | 71 | // // OBSOLETED by web handler 72 | // // Discover names of streams as CHECKTYPE-GROUP-NAME. 73 | // // Path to discovery file must be defined in config file in `zabbix-discovery` parameter. 74 | // // Accordingly with spec https://www.zabbix.com/documentation/2.2/manual/discovery/low_level_discovery 75 | // func ZabbixDiscoveryFile() { 76 | // var discoveryw *bufio.Writer 77 | // var tmpl *template.Template 78 | // var buf *bytes.Buffer = new(bytes.Buffer) 79 | // var err error 80 | 81 | // if cfg.Zabbix.DiscoveryPath == "" { 82 | // return 83 | // } else { 84 | // file, err := os.Create(cfg.Zabbix.DiscoveryPath) 85 | // if err != nil { 86 | // return 87 | // } 88 | // discoveryw = bufio.NewWriter(file) 89 | // defer discoveryw.Flush() 90 | // fmt.Printf("Zabbix discovery file prepared: %s\n", cfg.Zabbix.DiscoveryPath) 91 | // } 92 | // sort.Strings(cfg.Zabbix.DiscoveryGroups) 93 | // data := new(ZabbixDiscoveryData) 94 | // if cfg.Zabbix.StreamTemplate == "" { 95 | // cfg.Zabbix.StreamTemplate = "{{.Check}}-{{.Group}}-{{.Name}}" 96 | // } 97 | // tmpl, err = template.New("stream").Parse(cfg.Zabbix.StreamTemplate) 98 | // if err != nil { 99 | // tmpl, _ = template.New("stream").Parse("{{.Check}}-{{.Group}}-{{.Name}}") 100 | 101 | // } 102 | 103 | // for _, streams := range cfg.GroupStreams { 104 | // for _, stream := range *streams { 105 | // buf.Reset() 106 | // _ = tmpl.Execute(buf, streamTemplateData{Stream: stream, Check: StreamType2String(stream.Type)}) 107 | // if len(cfg.Zabbix.DiscoveryGroups) > 0 { 108 | // if sort.SearchStrings(cfg.Zabbix.DiscoveryGroups, stream.Group) == 0 { 109 | // data.Data = append(data.Data, map[string]string{"{#STREAM}": buf.String()}) 110 | // } 111 | // } else { 112 | // data.Data = append(data.Data, map[string]string{"{#STREAM}": buf.String()}) 113 | // } 114 | // } 115 | // } 116 | 117 | // output, _ := json.Marshal(data) 118 | // discoveryw.Write(output) 119 | // discoveryw.WriteRune('\n') 120 | // } 121 | 122 | // Log problem for Zabbix agent (duplicates error log but in another format) 123 | // func ZabbixStatus(vars map[string]string) []byte { 124 | // var page bytes.Buffer 125 | 126 | // ReportedStreams.RLock() 127 | // defer ReportedStreams.RUnlock() 128 | 129 | // if _, exists := vars["group"]; exists { // report for selected group 130 | // for _, value := range ReportedStreams.data[vars["group"]] { 131 | // page.WriteString(strconv.Quote(fmt.Sprintf("%s-%s-%s", StreamTypeText(value.Stream.Type), value.Stream.Group, value.Stream.Name))) 132 | // page.WriteRune(',') 133 | // page.WriteString(strconv.Quote(StreamTypeText(value.Stream.Type))) 134 | // page.WriteRune(',') 135 | // page.WriteString(strconv.Quote(value.Stream.Group)) 136 | // page.WriteRune(',') 137 | // page.WriteString(strconv.Quote(value.Stream.Name)) 138 | // page.WriteRune(',') 139 | // errnum, _ := fatalityLevel(value.Last.ErrType) 140 | // page.WriteString(strconv.Itoa(errnum)) 141 | // page.WriteRune('\n') 142 | // } 143 | // } else { // report for all groups 144 | // for _, group := range ReportedStreams.data { 145 | // for _, value := range group { 146 | // page.WriteString(strconv.Quote(fmt.Sprintf("%s-%s-%s", StreamTypeText(value.Stream.Type), value.Stream.Group, value.Stream.Name))) 147 | // page.WriteRune(',') 148 | // page.WriteString(strconv.Quote(StreamTypeText(value.Stream.Type))) 149 | // page.WriteRune(',') 150 | // page.WriteString(strconv.Quote(value.Stream.Group)) 151 | // page.WriteRune(',') 152 | // page.WriteString(strconv.Quote(value.Stream.Name)) 153 | // page.WriteRune(',') 154 | // errnum, _ := fatalityLevel(value.Last.ErrType) 155 | // page.WriteString(strconv.Itoa(errnum)) 156 | // page.WriteRune('\n') 157 | // } 158 | // } 159 | // } 160 | 161 | // return page.Bytes() 162 | // } 163 | 164 | func fatalityLevel(err ErrType) (int, string) { 165 | switch { 166 | case err == SUCCESS: 167 | return 0, "info" 168 | case err > SUCCESS && err < BADSTATUS: 169 | return 1, "warning" 170 | case err >= BADSTATUS: 171 | return 2, "error" 172 | } 173 | return -1, "unknown" 174 | } 175 | --------------------------------------------------------------------------------