├── .github └── workflows │ └── testpatterns.yml ├── .gitignore ├── Jenkinsfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── files ├── ceph.json ├── gpfs.json ├── grok-patterns.json ├── hpc-ugent-patterns.json ├── icinga.json ├── jube.json ├── keyvalue.json ├── lmod.json ├── modulecmd.json ├── nfs.json ├── opennebula.json ├── quattor.json ├── rsyslog.json ├── shorewall.json ├── snoopy.json └── ssh.json ├── runtest.py ├── setup.cfg ├── setup.py ├── tests ├── data │ ├── 00_first │ ├── ceph │ ├── general_syslog │ ├── jube │ ├── lmod │ ├── modulecmd │ ├── nfs │ ├── opennebula │ ├── quattor │ ├── shorewall │ ├── snoopy │ ├── ssh │ ├── su │ └── zz_last ├── lmod.toml ├── snoopy.toml └── vector.toml ├── tox.ini └── vsc-ci.ini /.github/workflows/testpatterns.yml: -------------------------------------------------------------------------------- 1 | name: test logstash patterns 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test_patterns: 6 | runs-on: ubuntu-latest 7 | strategy: 8 | matrix: 9 | python-version: 3.6 10 | steps: 11 | - name: add logstash repo 12 | uses: myci-actions/add-deb-repo@4 13 | with: 14 | repo: deb https://artifacts.elastic.co/packages/7.x/apt stable main 15 | repo-name: elastic-7.x 16 | keys: D27D666CD88E42B4 17 | - name: install logstash 18 | run: sudo apt-get install logstash 19 | - name: add logstash to path 20 | run: echo "/usr/share/logstash/bin" >> $GITHUB_PATH 21 | - name: Checkout code 22 | uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | pip install tox 31 | git remote add hpcugent https://github.com/hpcugent/vsc-install.git 32 | - name: run tests 33 | run: tox -v -c tox.ini 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## aptana/eclipse 2 | *.project 3 | *.pydevproject 4 | ## quattor build xml 5 | quattor/build 6 | ## python 7 | *.pyc 8 | *.pyo 9 | ## emacs 10 | *~ 11 | .#* 12 | *.orig 13 | *.bak 14 | ## Various build products 15 | *.rpm 16 | *.o 17 | *.old 18 | *.DS_Store 19 | # vim 20 | .*.swp 21 | .*.swo 22 | 23 | ## Python __init__.py files for distributed packages 24 | rpms/python-vsc-packages-*/vsc/__init__.py 25 | MANIFEST 26 | dist/ 27 | build/ 28 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | 3 | def VECTOR_VERSION = "0.37.1" 4 | 5 | node { 6 | stage('checkout git') { 7 | checkout scm 8 | // remove untracked files (*.pyc for example) 9 | sh 'git clean -fxd' 10 | } 11 | 12 | stage('install vector') { 13 | sh "wget -nv https://packages.timber.io/vector/${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz" 14 | sh "tar -xzf vector-${VECTOR_VERSION}-x86_64-unknown-linux-gnu.tar.gz" 15 | env.PATH = "${pwd()}/vector-x86_64-unknown-linux-gnu/bin/:${env.PATH}" 16 | } 17 | 18 | stage('test') { 19 | sh 'python3.6 -V' 20 | sh 'pip3 install --ignore-installed --prefix $PWD/.vsc-tox tox' 21 | sh 'export PATH=$PWD/.vsc-tox/bin:$PATH && export PYTHONPATH=$PWD/.vsc-tox/lib/python$(python3 -c "import sys; print(\\"%s.%s\\" % sys.version_info[:2])")/site-packages:$PYTHONPATH && tox -v -c tox.ini' 22 | sh 'rm -r $PWD/.vsc-tox' 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 2.1, February 1999 3 | 4 | Copyright (C) 1991, 1999 Free Software Foundation, Inc. 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | [This is the first released version of the Lesser GPL. It also counts 10 | as the successor of the GNU Library Public License, version 2, hence 11 | the version number 2.1.] 12 | 13 | Preamble 14 | 15 | The licenses for most software are designed to take away your 16 | freedom to share and change it. By contrast, the GNU General Public 17 | Licenses are intended to guarantee your freedom to share and change 18 | free software--to make sure the software is free for all its users. 19 | 20 | This license, the Lesser General Public License, applies to some 21 | specially designated software packages--typically libraries--of the 22 | Free Software Foundation and other authors who decide to use it. You 23 | can use it too, but we suggest you first think carefully about whether 24 | this license or the ordinary General Public License is the better 25 | strategy to use in any particular case, based on the explanations below. 26 | 27 | When we speak of free software, we are referring to freedom of use, 28 | not price. Our General Public Licenses are designed to make sure that 29 | you have the freedom to distribute copies of free software (and charge 30 | for this service if you wish); that you receive source code or can get 31 | it if you want it; that you can change the software and use pieces of 32 | it in new free programs; and that you are informed that you can do 33 | these things. 34 | 35 | To protect your rights, we need to make restrictions that forbid 36 | distributors to deny you these rights or to ask you to surrender these 37 | rights. These restrictions translate to certain responsibilities for 38 | you if you distribute copies of the library or if you modify it. 39 | 40 | For example, if you distribute copies of the library, whether gratis 41 | or for a fee, you must give the recipients all the rights that we gave 42 | you. You must make sure that they, too, receive or can get the source 43 | code. If you link other code with the library, you must provide 44 | complete object files to the recipients, so that they can relink them 45 | with the library after making changes to the library and recompiling 46 | it. And you must show them these terms so they know their rights. 47 | 48 | We protect your rights with a two-step method: (1) we copyright the 49 | library, and (2) we offer you this license, which gives you legal 50 | permission to copy, distribute and/or modify the library. 51 | 52 | To protect each distributor, we want to make it very clear that 53 | there is no warranty for the free library. Also, if the library is 54 | modified by someone else and passed on, the recipients should know 55 | that what they have is not the original version, so that the original 56 | author's reputation will not be affected by problems that might be 57 | introduced by others. 58 | 59 | Finally, software patents pose a constant threat to the existence of 60 | any free program. We wish to make sure that a company cannot 61 | effectively restrict the users of a free program by obtaining a 62 | restrictive license from a patent holder. Therefore, we insist that 63 | any patent license obtained for a version of the library must be 64 | consistent with the full freedom of use specified in this license. 65 | 66 | Most GNU software, including some libraries, is covered by the 67 | ordinary GNU General Public License. This license, the GNU Lesser 68 | General Public License, applies to certain designated libraries, and 69 | is quite different from the ordinary General Public License. We use 70 | this license for certain libraries in order to permit linking those 71 | libraries into non-free programs. 72 | 73 | When a program is linked with a library, whether statically or using 74 | a shared library, the combination of the two is legally speaking a 75 | combined work, a derivative of the original library. The ordinary 76 | General Public License therefore permits such linking only if the 77 | entire combination fits its criteria of freedom. The Lesser General 78 | Public License permits more lax criteria for linking other code with 79 | the library. 80 | 81 | We call this license the "Lesser" General Public License because it 82 | does Less to protect the user's freedom than the ordinary General 83 | Public License. It also provides other free software developers Less 84 | of an advantage over competing non-free programs. These disadvantages 85 | are the reason we use the ordinary General Public License for many 86 | libraries. However, the Lesser license provides advantages in certain 87 | special circumstances. 88 | 89 | For example, on rare occasions, there may be a special need to 90 | encourage the widest possible use of a certain library, so that it becomes 91 | a de-facto standard. To achieve this, non-free programs must be 92 | allowed to use the library. A more frequent case is that a free 93 | library does the same job as widely used non-free libraries. In this 94 | case, there is little to gain by limiting the free library to free 95 | software only, so we use the Lesser General Public License. 96 | 97 | In other cases, permission to use a particular library in non-free 98 | programs enables a greater number of people to use a large body of 99 | free software. For example, permission to use the GNU C Library in 100 | non-free programs enables many more people to use the whole GNU 101 | operating system, as well as its variant, the GNU/Linux operating 102 | system. 103 | 104 | Although the Lesser General Public License is Less protective of the 105 | users' freedom, it does ensure that the user of a program that is 106 | linked with the Library has the freedom and the wherewithal to run 107 | that program using a modified version of the Library. 108 | 109 | The precise terms and conditions for copying, distribution and 110 | modification follow. Pay close attention to the difference between a 111 | "work based on the library" and a "work that uses the library". The 112 | former contains code derived from the library, whereas the latter must 113 | be combined with the library in order to run. 114 | 115 | GNU LESSER GENERAL PUBLIC LICENSE 116 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 117 | 118 | 0. This License Agreement applies to any software library or other 119 | program which contains a notice placed by the copyright holder or 120 | other authorized party saying it may be distributed under the terms of 121 | this Lesser General Public License (also called "this License"). 122 | Each licensee is addressed as "you". 123 | 124 | A "library" means a collection of software functions and/or data 125 | prepared so as to be conveniently linked with application programs 126 | (which use some of those functions and data) to form executables. 127 | 128 | The "Library", below, refers to any such software library or work 129 | which has been distributed under these terms. A "work based on the 130 | Library" means either the Library or any derivative work under 131 | copyright law: that is to say, a work containing the Library or a 132 | portion of it, either verbatim or with modifications and/or translated 133 | straightforwardly into another language. (Hereinafter, translation is 134 | included without limitation in the term "modification".) 135 | 136 | "Source code" for a work means the preferred form of the work for 137 | making modifications to it. For a library, complete source code means 138 | all the source code for all modules it contains, plus any associated 139 | interface definition files, plus the scripts used to control compilation 140 | and installation of the library. 141 | 142 | Activities other than copying, distribution and modification are not 143 | covered by this License; they are outside its scope. The act of 144 | running a program using the Library is not restricted, and output from 145 | such a program is covered only if its contents constitute a work based 146 | on the Library (independent of the use of the Library in a tool for 147 | writing it). Whether that is true depends on what the Library does 148 | and what the program that uses the Library does. 149 | 150 | 1. You may copy and distribute verbatim copies of the Library's 151 | complete source code as you receive it, in any medium, provided that 152 | you conspicuously and appropriately publish on each copy an 153 | appropriate copyright notice and disclaimer of warranty; keep intact 154 | all the notices that refer to this License and to the absence of any 155 | warranty; and distribute a copy of this License along with the 156 | Library. 157 | 158 | You may charge a fee for the physical act of transferring a copy, 159 | and you may at your option offer warranty protection in exchange for a 160 | fee. 161 | 162 | 2. You may modify your copy or copies of the Library or any portion 163 | of it, thus forming a work based on the Library, and copy and 164 | distribute such modifications or work under the terms of Section 1 165 | above, provided that you also meet all of these conditions: 166 | 167 | a) The modified work must itself be a software library. 168 | 169 | b) You must cause the files modified to carry prominent notices 170 | stating that you changed the files and the date of any change. 171 | 172 | c) You must cause the whole of the work to be licensed at no 173 | charge to all third parties under the terms of this License. 174 | 175 | d) If a facility in the modified Library refers to a function or a 176 | table of data to be supplied by an application program that uses 177 | the facility, other than as an argument passed when the facility 178 | is invoked, then you must make a good faith effort to ensure that, 179 | in the event an application does not supply such function or 180 | table, the facility still operates, and performs whatever part of 181 | its purpose remains meaningful. 182 | 183 | (For example, a function in a library to compute square roots has 184 | a purpose that is entirely well-defined independent of the 185 | application. Therefore, Subsection 2d requires that any 186 | application-supplied function or table used by this function must 187 | be optional: if the application does not supply it, the square 188 | root function must still compute square roots.) 189 | 190 | These requirements apply to the modified work as a whole. If 191 | identifiable sections of that work are not derived from the Library, 192 | and can be reasonably considered independent and separate works in 193 | themselves, then this License, and its terms, do not apply to those 194 | sections when you distribute them as separate works. But when you 195 | distribute the same sections as part of a whole which is a work based 196 | on the Library, the distribution of the whole must be on the terms of 197 | this License, whose permissions for other licensees extend to the 198 | entire whole, and thus to each and every part regardless of who wrote 199 | it. 200 | 201 | Thus, it is not the intent of this section to claim rights or contest 202 | your rights to work written entirely by you; rather, the intent is to 203 | exercise the right to control the distribution of derivative or 204 | collective works based on the Library. 205 | 206 | In addition, mere aggregation of another work not based on the Library 207 | with the Library (or with a work based on the Library) on a volume of 208 | a storage or distribution medium does not bring the other work under 209 | the scope of this License. 210 | 211 | 3. You may opt to apply the terms of the ordinary GNU General Public 212 | License instead of this License to a given copy of the Library. To do 213 | this, you must alter all the notices that refer to this License, so 214 | that they refer to the ordinary GNU General Public License, version 2, 215 | instead of to this License. (If a newer version than version 2 of the 216 | ordinary GNU General Public License has appeared, then you can specify 217 | that version instead if you wish.) Do not make any other change in 218 | these notices. 219 | 220 | Once this change is made in a given copy, it is irreversible for 221 | that copy, so the ordinary GNU General Public License applies to all 222 | subsequent copies and derivative works made from that copy. 223 | 224 | This option is useful when you wish to copy part of the code of 225 | the Library into a program that is not a library. 226 | 227 | 4. You may copy and distribute the Library (or a portion or 228 | derivative of it, under Section 2) in object code or executable form 229 | under the terms of Sections 1 and 2 above provided that you accompany 230 | it with the complete corresponding machine-readable source code, which 231 | must be distributed under the terms of Sections 1 and 2 above on a 232 | medium customarily used for software interchange. 233 | 234 | If distribution of object code is made by offering access to copy 235 | from a designated place, then offering equivalent access to copy the 236 | source code from the same place satisfies the requirement to 237 | distribute the source code, even though third parties are not 238 | compelled to copy the source along with the object code. 239 | 240 | 5. A program that contains no derivative of any portion of the 241 | Library, but is designed to work with the Library by being compiled or 242 | linked with it, is called a "work that uses the Library". Such a 243 | work, in isolation, is not a derivative work of the Library, and 244 | therefore falls outside the scope of this License. 245 | 246 | However, linking a "work that uses the Library" with the Library 247 | creates an executable that is a derivative of the Library (because it 248 | contains portions of the Library), rather than a "work that uses the 249 | library". The executable is therefore covered by this License. 250 | Section 6 states terms for distribution of such executables. 251 | 252 | When a "work that uses the Library" uses material from a header file 253 | that is part of the Library, the object code for the work may be a 254 | derivative work of the Library even though the source code is not. 255 | Whether this is true is especially significant if the work can be 256 | linked without the Library, or if the work is itself a library. The 257 | threshold for this to be true is not precisely defined by law. 258 | 259 | If such an object file uses only numerical parameters, data 260 | structure layouts and accessors, and small macros and small inline 261 | functions (ten lines or less in length), then the use of the object 262 | file is unrestricted, regardless of whether it is legally a derivative 263 | work. (Executables containing this object code plus portions of the 264 | Library will still fall under Section 6.) 265 | 266 | Otherwise, if the work is a derivative of the Library, you may 267 | distribute the object code for the work under the terms of Section 6. 268 | Any executables containing that work also fall under Section 6, 269 | whether or not they are linked directly with the Library itself. 270 | 271 | 6. As an exception to the Sections above, you may also combine or 272 | link a "work that uses the Library" with the Library to produce a 273 | work containing portions of the Library, and distribute that work 274 | under terms of your choice, provided that the terms permit 275 | modification of the work for the customer's own use and reverse 276 | engineering for debugging such modifications. 277 | 278 | You must give prominent notice with each copy of the work that the 279 | Library is used in it and that the Library and its use are covered by 280 | this License. You must supply a copy of this License. If the work 281 | during execution displays copyright notices, you must include the 282 | copyright notice for the Library among them, as well as a reference 283 | directing the user to the copy of this License. Also, you must do one 284 | of these things: 285 | 286 | a) Accompany the work with the complete corresponding 287 | machine-readable source code for the Library including whatever 288 | changes were used in the work (which must be distributed under 289 | Sections 1 and 2 above); and, if the work is an executable linked 290 | with the Library, with the complete machine-readable "work that 291 | uses the Library", as object code and/or source code, so that the 292 | user can modify the Library and then relink to produce a modified 293 | executable containing the modified Library. (It is understood 294 | that the user who changes the contents of definitions files in the 295 | Library will not necessarily be able to recompile the application 296 | to use the modified definitions.) 297 | 298 | b) Use a suitable shared library mechanism for linking with the 299 | Library. A suitable mechanism is one that (1) uses at run time a 300 | copy of the library already present on the user's computer system, 301 | rather than copying library functions into the executable, and (2) 302 | will operate properly with a modified version of the library, if 303 | the user installs one, as long as the modified version is 304 | interface-compatible with the version that the work was made with. 305 | 306 | c) Accompany the work with a written offer, valid for at 307 | least three years, to give the same user the materials 308 | specified in Subsection 6a, above, for a charge no more 309 | than the cost of performing this distribution. 310 | 311 | d) If distribution of the work is made by offering access to copy 312 | from a designated place, offer equivalent access to copy the above 313 | specified materials from the same place. 314 | 315 | e) Verify that the user has already received a copy of these 316 | materials or that you have already sent this user a copy. 317 | 318 | For an executable, the required form of the "work that uses the 319 | Library" must include any data and utility programs needed for 320 | reproducing the executable from it. However, as a special exception, 321 | the materials to be distributed need not include anything that is 322 | normally distributed (in either source or binary form) with the major 323 | components (compiler, kernel, and so on) of the operating system on 324 | which the executable runs, unless that component itself accompanies 325 | the executable. 326 | 327 | It may happen that this requirement contradicts the license 328 | restrictions of other proprietary libraries that do not normally 329 | accompany the operating system. Such a contradiction means you cannot 330 | use both them and the Library together in an executable that you 331 | distribute. 332 | 333 | 7. You may place library facilities that are a work based on the 334 | Library side-by-side in a single library together with other library 335 | facilities not covered by this License, and distribute such a combined 336 | library, provided that the separate distribution of the work based on 337 | the Library and of the other library facilities is otherwise 338 | permitted, and provided that you do these two things: 339 | 340 | a) Accompany the combined library with a copy of the same work 341 | based on the Library, uncombined with any other library 342 | facilities. This must be distributed under the terms of the 343 | Sections above. 344 | 345 | b) Give prominent notice with the combined library of the fact 346 | that part of it is a work based on the Library, and explaining 347 | where to find the accompanying uncombined form of the same work. 348 | 349 | 8. You may not copy, modify, sublicense, link with, or distribute 350 | the Library except as expressly provided under this License. Any 351 | attempt otherwise to copy, modify, sublicense, link with, or 352 | distribute the Library is void, and will automatically terminate your 353 | rights under this License. However, parties who have received copies, 354 | or rights, from you under this License will not have their licenses 355 | terminated so long as such parties remain in full compliance. 356 | 357 | 9. You are not required to accept this License, since you have not 358 | signed it. However, nothing else grants you permission to modify or 359 | distribute the Library or its derivative works. These actions are 360 | prohibited by law if you do not accept this License. Therefore, by 361 | modifying or distributing the Library (or any work based on the 362 | Library), you indicate your acceptance of this License to do so, and 363 | all its terms and conditions for copying, distributing or modifying 364 | the Library or works based on it. 365 | 366 | 10. Each time you redistribute the Library (or any work based on the 367 | Library), the recipient automatically receives a license from the 368 | original licensor to copy, distribute, link with or modify the Library 369 | subject to these terms and conditions. You may not impose any further 370 | restrictions on the recipients' exercise of the rights granted herein. 371 | You are not responsible for enforcing compliance by third parties with 372 | this License. 373 | 374 | 11. If, as a consequence of a court judgment or allegation of patent 375 | infringement or for any other reason (not limited to patent issues), 376 | conditions are imposed on you (whether by court order, agreement or 377 | otherwise) that contradict the conditions of this License, they do not 378 | excuse you from the conditions of this License. If you cannot 379 | distribute so as to satisfy simultaneously your obligations under this 380 | License and any other pertinent obligations, then as a consequence you 381 | may not distribute the Library at all. For example, if a patent 382 | license would not permit royalty-free redistribution of the Library by 383 | all those who receive copies directly or indirectly through you, then 384 | the only way you could satisfy both it and this License would be to 385 | refrain entirely from distribution of the Library. 386 | 387 | If any portion of this section is held invalid or unenforceable under any 388 | particular circumstance, the balance of the section is intended to apply, 389 | and the section as a whole is intended to apply in other circumstances. 390 | 391 | It is not the purpose of this section to induce you to infringe any 392 | patents or other property right claims or to contest validity of any 393 | such claims; this section has the sole purpose of protecting the 394 | integrity of the free software distribution system which is 395 | implemented by public license practices. Many people have made 396 | generous contributions to the wide range of software distributed 397 | through that system in reliance on consistent application of that 398 | system; it is up to the author/donor to decide if he or she is willing 399 | to distribute software through any other system and a licensee cannot 400 | impose that choice. 401 | 402 | This section is intended to make thoroughly clear what is believed to 403 | be a consequence of the rest of this License. 404 | 405 | 12. If the distribution and/or use of the Library is restricted in 406 | certain countries either by patents or by copyrighted interfaces, the 407 | original copyright holder who places the Library under this License may add 408 | an explicit geographical distribution limitation excluding those countries, 409 | so that distribution is permitted only in or among countries not thus 410 | excluded. In such case, this License incorporates the limitation as if 411 | written in the body of this License. 412 | 413 | 13. The Free Software Foundation may publish revised and/or new 414 | versions of the Lesser General Public License from time to time. 415 | Such new versions will be similar in spirit to the present version, 416 | but may differ in detail to address new problems or concerns. 417 | 418 | Each version is given a distinguishing version number. If the Library 419 | specifies a version number of this License which applies to it and 420 | "any later version", you have the option of following the terms and 421 | conditions either of that version or of any later version published by 422 | the Free Software Foundation. If the Library does not specify a 423 | license version number, you may choose any version ever published by 424 | the Free Software Foundation. 425 | 426 | 14. If you wish to incorporate parts of the Library into other free 427 | programs whose distribution conditions are incompatible with these, 428 | write to the author to ask for permission. For software which is 429 | copyrighted by the Free Software Foundation, write to the Free 430 | Software Foundation; we sometimes make exceptions for this. Our 431 | decision will be guided by the two goals of preserving the free status 432 | of all derivatives of our free software and of promoting the sharing 433 | and reuse of software generally. 434 | 435 | NO WARRANTY 436 | 437 | 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 438 | WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 439 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR 440 | OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY 441 | KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE 442 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 443 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE 444 | LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME 445 | THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 446 | 447 | 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN 448 | WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY 449 | AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU 450 | FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR 451 | CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE 452 | LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING 453 | RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A 454 | FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF 455 | SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 456 | DAMAGES. 457 | 458 | END OF TERMS AND CONDITIONS 459 | 460 | How to Apply These Terms to Your New Libraries 461 | 462 | If you develop a new library, and you want it to be of the greatest 463 | possible use to the public, we recommend making it free software that 464 | everyone can redistribute and change. You can do so by permitting 465 | redistribution under these terms (or, alternatively, under the terms of the 466 | ordinary General Public License). 467 | 468 | To apply these terms, attach the following notices to the library. It is 469 | safest to attach them to the start of each source file to most effectively 470 | convey the exclusion of warranty; and each file should have at least the 471 | "copyright" line and a pointer to where the full notice is found. 472 | 473 | 474 | Copyright (C) 475 | 476 | This library is free software; you can redistribute it and/or 477 | modify it under the terms of the GNU Lesser General Public 478 | License as published by the Free Software Foundation; either 479 | version 2.1 of the License, or (at your option) any later version. 480 | 481 | This library is distributed in the hope that it will be useful, 482 | but WITHOUT ANY WARRANTY; without even the implied warranty of 483 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 484 | Lesser General Public License for more details. 485 | 486 | You should have received a copy of the GNU Lesser General Public 487 | License along with this library; if not, write to the Free Software 488 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 489 | 490 | Also add information on how to contact you by electronic and paper mail. 491 | 492 | You should also get your employer (if you work as a programmer) or your 493 | school, if any, to sign a "copyright disclaimer" for the library, if 494 | necessary. Here is a sample; alter the names: 495 | 496 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 497 | library `Frob' (a library for tweaking knobs) written by James Random Hacker. 498 | 499 | , 1 April 1990 500 | Ty Coon, President of Vice 501 | 502 | That's all there is to it! -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include files/* 2 | include logstash/filters/* 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grok patterns 2 | 3 | Patterns for parsing and structuring log messages for different 4 | services with [Vector](http://vector.dev). 5 | 6 | Use [this app](https://grokdebug.herokuapp.com/) for debugging your 7 | patterns! Be careful, there might be subtle differences! 8 | 9 | When issuing pull requests to this repository, don't forget to include an example of the messages your commits try to parse!! 10 | 11 | ## Adding pattern 12 | 13 | Develop a new PATTERN typically in its own file, and call the main new pattern `_MSG`. 14 | 15 | Then you can: 16 | * Add the new message pattern to the Vector config file in the section `[transforms.syslog]`. Typically, 17 | you will add it to the top-level list of patterns to try (second argument of parse_groks). 18 | * Include the definitions for this grok pattern in its own JSON file `.json`, in the format 19 | given by the existing JSON files. You will need to add this filename to the `vector.toml` config file 20 | under the parse_groks argument `alias_sources`. 21 | * Add tests for your pattern in `tests/.toml`. Specify the input, and compose the VRL program that 22 | asserts the expected structured output. 23 | -------------------------------------------------------------------------------- /files/ceph.json: -------------------------------------------------------------------------------- 1 | { 2 | "CEPH_BYTES": "%{NONNEGINT} (?:(?:k|M|G|T|P)?B)", 3 | "CEPH_MSG": "%{DATA} (?:%{CEPH_PGMAP}|%{CEPH_SLOW_REQUEST})", 4 | "CEPH_PGMAP": ".*? pgmap v(?[0-9]+): %{NONNEGINT:ceph.pgs} pgs: (?:%{CEPH_PGMAP_DETAIL})(?:; %{CEPH_PGMAP_VOLUME})?(?:; %{CEPH_PGMAP_ACT})?(?:; %{CEPH_PGMAP_OBJ_DEGRADED})?(?:; %{CEPH_PGMAP_OBJ_MISPLACED})?(?:; %{CEPH_PGMAP_OBJ_RECOV})?$", 5 | "CEPH_PGMAP_ACT": "(%{CEPH_BYTES:ceph.actread}/s rd, )?(%{CEPH_BYTES:ceph.actwrite}/s wr, )?%{NONNEGINT:ceph.actops} op/s", 6 | "CEPH_PGMAP_DETAIL": "(?:%{CEPH_PGMAP_DETAIL_PAT}(?:, %{CEPH_PGMAP_DETAIL_PAT})*)", 7 | "CEPH_PGMAP_DETAIL_ACTIVE_CLEAN": "%{NONNEGINT:ceph.active_clean} active\\+clean", 8 | "CEPH_PGMAP_DETAIL_PAT": "(?:%{CEPH_PGMAP_DETAIL_ACTIVE_CLEAN}|%{CEPH_PGMAP_DETAIL_REST})", 9 | "CEPH_PGMAP_DETAIL_REST": "%{NONNEGINT} (?:[^ ,;]+)", 10 | "CEPH_PGMAP_OBJ_DEGRADED": "%{NONNEGINT:ceph.objdegraded}/%{NONNEGINT:ceph.objtotal} objects degraded [^;]*", 11 | "CEPH_PGMAP_OBJ_MISPLACED": "%{NONNEGINT:ceph.objmisplaced}/%{NONNEGINT:ceph.objtotal} objects misplaced [^;]*", 12 | "CEPH_PGMAP_OBJ_RECOV": "%{CEPH_BYTES:ceph.objrecovthr}/s, %{NONNEGINT:ceph.objrecovhz} objects/s recovering", 13 | "CEPH_PGMAP_VOLUME": "%{CEPH_BYTES:ceph.volumedata} data, %{CEPH_BYTES:ceph.volumeused} used, %{CEPH_BYTES:ceph.volumeavail} / %{CEPH_BYTES:ceph.volumetotal} avail", 14 | "CEPH_SLOW_REQUEST": ".*? (?osd.%{NONNEGINT}) .*? (?:%{CEPH_SLOW_REQUEST_SINGLE}|%{CEPH_SLOW_REQUEST_OLDEST}) %{CEPH_SLOW_REQUEST_SECONDS} (?:secs|seconds old)", 15 | "CEPH_SLOW_REQUEST_OLDEST": "%{NONNEGINT:ceph.slowrequestnumber} slow requests, .*? oldest blocked for .*?", 16 | "CEPH_SLOW_REQUEST_SECONDS": "(?%{NONNEGINT})(?:\\.%{NONNEGINT})?", 17 | "CEPH_SLOW_REQUEST_SINGLE": "slow request" 18 | } 19 | -------------------------------------------------------------------------------- /files/gpfs.json: -------------------------------------------------------------------------------- 1 | { 2 | "GPFSDATA": "%{GREEDYDATA}", 3 | "GPFSDATE": "%{DAY}%{SPACE}%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{YEAR}", 4 | "GPFSLOG": "%{GPFSDATE:gpfs.timestamp}:%{SPACE}%{GPFSDATA:gpfs.realmsg}" 5 | } 6 | -------------------------------------------------------------------------------- /files/grok-patterns.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "USERNAME": "[a-zA-Z0-9._-]+", 4 | "USER": "%{USERNAME}", 5 | "EMAILLOCALPART": "[a-zA-Z][a-zA-Z0-9_.+-=:]+", 6 | "EMAILADDRESS": "%{EMAILLOCALPART}@%{HOSTNAME}", 7 | "HTTPDUSER": "%{EMAILADDRESS}|%{USER}", 8 | "INT": "(?:[+-]?(?:[0-9]+))", 9 | "BASE10NUM": "(?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))", 10 | "NUMBER": "(?:%{BASE10NUM})", 11 | "BASE16NUM": "(?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))", 22 | "UUID": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}", 23 | 24 | "MAC": "(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})", 25 | "CISCOMAC": "(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})", 26 | "WINDOWSMAC": "(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})", 27 | "COMMONMAC": "(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})", 28 | "IPV6": "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?", 29 | "IPV4": "(?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+", 39 | "URIPROTO": "[A-Za-z]+(\+[A-Za-z+]+)?", 40 | "URIHOST": "%{IPORHOST}(?::%{POSINT:port})?", 41 | "URIPATH": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+", 42 | "#URIPARAM": "\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?", 43 | "URIPARAM": "\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*", 44 | "URIPATHPARAM": "%{URIPATH}(?:%{URIPARAM})?", 45 | "URI": "%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?", 46 | 47 | "MONTH": "\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b", 48 | "MONTHNUM": "(?:0?[1-9]|1[0-2])", 49 | "MONTHNUM2": "(?:0[1-9]|1[0-2])", 50 | "MONTHDAY": "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])", 51 | 52 | "DAY": "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)", 53 | 54 | "YEAR": "(?>\d\d){1,2}", 55 | "HOUR": "(?:2[0123]|[01]?[0-9])", 56 | "MINUTE": "(?:[0-5][0-9])", 57 | "SECOND": "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)", 58 | "TIME": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])", 59 | "DATE_US": "%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}", 60 | "DATE_EU": "%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}", 61 | "ISO8601_TIMEZONE": "(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))", 62 | "ISO8601_SECOND": "(?:%{SECOND}|60)", 63 | "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?", 64 | "DATE": "%{DATE_US}|%{DATE_EU}", 65 | "DATESTAMP": "%{DATE}[- ]%{TIME}", 66 | "TZ": "(?:[PMCE][SD]T|UTC)", 67 | "DATESTAMP_RFC822": "%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}", 68 | "DATESTAMP_RFC2822": "%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}", 69 | "DATESTAMP_OTHER": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}", 70 | "DATESTAMP_EVENTLOG": "%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}", 71 | "HTTPDERROR_DATE": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}", 72 | 73 | "SYSLOGTIMESTAMP": "%{MONTH} +%{MONTHDAY} %{TIME}", 74 | "PROG": "[\x21-\x5a\x5c\x5e-\x7e]+", 75 | "SYSLOGPROG": "%{PROG:program}(?:\[%{POSINT:pid}\])?", 76 | "SYSLOGHOST": "%{IPORHOST}", 77 | "SYSLOGFACILITY": "<%{NONNEGINT:facility}.%{NONNEGINT:priority}>", 78 | "HTTPDATE": "%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}", 79 | 80 | "QS": "%{QUOTEDSTRING}", 81 | 82 | "SYSLOGBASE": "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:", 83 | "COMMONAPACHELOG": "%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)", 84 | "COMBINEDAPACHELOG": "%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}", 85 | "HTTPD20_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}", 86 | "HTTPD24_ERRORLOG": "\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}", 87 | "HTTPD_ERRORLOG": "%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}", 88 | 89 | 90 | "LOGLEVEL": "([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)" 91 | } 92 | -------------------------------------------------------------------------------- /files/hpc-ugent-patterns.json: -------------------------------------------------------------------------------- 1 | { 2 | "USERNAME_PAT": "%{USERNAME}" 3 | } 4 | -------------------------------------------------------------------------------- /files/icinga.json: -------------------------------------------------------------------------------- 1 | { 2 | "ICINGA_CLASS": "([^:]*?):", 3 | "ICINGA_FIELD": "([^;]+)", 4 | "ICINGA_HOST": "([-\\w.]+)", 5 | "ICINGA_HOST_BODY": "%{ICINGA_FIELD:icinga.host.status};%{ICINGA_FIELD:icinga.host.status_type};%{INT:icinga.host.count};%{GREEDYDATA:icinga.host.check_output}", 6 | "ICINGA_MSG": "\\[%{NONNEGINT:icinga.timestamp}\\]%{SPACE}%{ICINGA_CLASS:icinga.class}%{SPACE}%{ICINGA_HOST:icinga.host};(?:%{ICINGA_SERVICE_BODY}|%{ICINGA_HOST_BODY})", 7 | "ICINGA_SERVICE_BODY": "%{ICINGA_FIELD:icinga.service.check};%{ICINGA_FIELD:icinga.service.status};%{ICINGA_FIELD:icinga.service.status_type};%{INT:icinga.service.count};%{GREEDYDATA:icinga.service.check_output}" 8 | } 9 | -------------------------------------------------------------------------------- /files/jube.json: -------------------------------------------------------------------------------- 1 | { 2 | "EXEC_HOST": "[-a-zA-Z0-9/+.,]+", 3 | "JOB_INPUT": "[-a-zA-Z0-9/._]+", 4 | "MODULE_NAME": "[-a-zA-Z0-9/.]+", 5 | "REALWORD": "[\\w-]+" 6 | } 7 | -------------------------------------------------------------------------------- /files/keyvalue.json: -------------------------------------------------------------------------------- 1 | { 2 | "KEY_EQ_VALUE_MSG": "(?:\\s+(?:%{NOTSPACE}=['\"]?%{NOTSPACE}['\"]?))+\\s*$", 3 | "KEYVALUE_MSG": "%{KEY_EQ_VALUE_MSG:KEY_EQ_VALUEDATA}" 4 | } 5 | -------------------------------------------------------------------------------- /files/lmod.json: -------------------------------------------------------------------------------- 1 | { 2 | "LMOD": "%{LMOD_GENERAL}, %{LMOD_LOAD}", 3 | "LMOD_GENERAL": "username=%{USERNAME:lmod.username}, cluster=%{DATA:lmod.cluster}, jobid=%{DATA:lmod.jobid}", 4 | "LMOD_LOAD": "userload=%{LMOD_USERLOAD:lmod.userload}, module=%{LMOD_MODULE}, fn=%{DATA:lmod.module.filename}", 5 | "LMOD_MODULE": "%{DATA:lmod.module.name}/%{DATA:lmod.module.version}", 6 | "LMOD_USERLOAD": "(?:yes|no)" 7 | } 8 | -------------------------------------------------------------------------------- /files/modulecmd.json: -------------------------------------------------------------------------------- 1 | { 2 | "MODULECMD_MSG": "(?:- modulecmd:: )?U%{USERNAME_PAT} H%{DATA:modulecmd.host} VC%{DATA:modulecmd.cluster} VAL%{DATA:modlecmd.architecture} ID%{DATA:modulecmd.jobid} %{DATA:modulecmd.shell}_%{DATA:modulecmd.cmd}_%{MODULE_NAME_AND_OR_VERSION}", 3 | "MODULE_NAME_AND_OR_VERSION": "(?:%{DATA:modulecmd.name}@%{GREEDYDATA:modulecmd.version}|%{GREEDYDATA:modulecmd.data})" 4 | } 5 | -------------------------------------------------------------------------------- /files/nfs.json: -------------------------------------------------------------------------------- 1 | { 2 | "NFS_GENERAL": ".*? nfs: server %{IPORHOST:nfs.server} %{DATA:nfs.reason}$", 3 | "NFS_MSG": "(?:%{NFS_GENERAL}|%{NFS_NFS4_GENERAL})", 4 | "NFS_NFS4_GENERAL": ".*?: (?[^:]*) on NFSv4 server %{IPORHOST:nfs.server}" 5 | } 6 | -------------------------------------------------------------------------------- /files/opennebula.json: -------------------------------------------------------------------------------- 1 | { 2 | "OPENNEBULA_MSG": "(?:%{OPENNEBULA_ONED_MONITOR_VM}|%{OPENNEBULA_ONED_VM_STATUS}|%{OPENNEBULA_ONED_MONITOR_HYP}|%{OPENNEBULA_SCHED_STATUS}|%{OPENNEBULA_ONED_GENERIC_VM})", 3 | "OPENNEBULA_ONED_GENERIC_VM": "\\[VM %{INT:one.vm.vmid}\\].*?\\[%{WORD:one.vm.level}\\]: .*?", 4 | "OPENNEBULA_ONED_MONITOR_HYP": ".*?\\[%{WORD:one.mon.level}\\]: .*? monitoring Host %{WORD:one.mon.hypervisor} \\(%{INT:one.mon.hostid}\\)", 5 | "OPENNEBULA_ONED_MONITOR_VM": ".*?\\[%{WORD:one.mon.level}\\]: VM %{INT:one.mon.vmid} .*?: STATE\\=%{WORD:one.mon.vmstate} CPU\\=%{NUMBER:one.mon.vmucpu} MEMORY\\=%{INT:one.mon.vmmem} NETRX\\=%{INT:one.mon.vmnetrx} NETTX\\=%{INT:one.mon.vmnettx} .*?", 6 | "OPENNEBULA_ONED_VM_STATUS": "\\[VM %{INT:one.vm.status.vmid}\\].*?\\[%{WORD:one.vm.status.level}\\]: .*? state is %{WORD:one.vm.status.vmstate}", 7 | "OPENNEBULA_SCHED_STATUS": ".*?\\[%{WORD:one.sched.status.level}\\]: .*? VM %{INT:one.sched.status.vmid} .*? host %{INT:one.sched.status.hostid} .*? datastore %{INT:one.sched.status.datastoreid}" 8 | } 9 | -------------------------------------------------------------------------------- /files/quattor.json: -------------------------------------------------------------------------------- 1 | { 2 | "QUATTOR_MSG": "(?:%{QUATTOR_NCD_START}|%{QUATTOR_NCD_FINISH})", 3 | "QUATTOR_NCD_FINISH": ".*?%{INT:quattor.ncderrors} errors\\, %{INT:quattor.ncdwarnings} warnings executing configure", 4 | "QUATTOR_NCD_START": ".*?version %{DATA:quattor.ncdversion} started.*?" 5 | } 6 | -------------------------------------------------------------------------------- /files/rsyslog.json: -------------------------------------------------------------------------------- 1 | { 2 | "RSYSLOGMESSAGE": "(?:%{GREEDYDATA:syslog_message})", 3 | "RSYSLOGCUSTOMHEADER": "(?:(?:<%{NONNEGINT:syslog_abspri}>(?:%{NONNEGINT:syslog_version} )?)?(?%{TIMESTAMP_ISO8601:}|%{SYSLOGTIMESTAMP}))", 4 | "RSYSLOGPREFIX": " %{RSYSLOGCUSTOMHEADER} %{IPORHOST:syslog_hostname} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?", 5 | "RSYSLOGCUSTOM": "%{RSYSLOGPREFIX}%{RSYSLOGMESSAGE}" 6 | } 7 | -------------------------------------------------------------------------------- /files/shorewall.json: -------------------------------------------------------------------------------- 1 | { 2 | "SHOREWALL_ICMP": ".*?%{WORD:shorewall.icmp.fwrule}\\:%{WORD:shorewall.icmp.fwtarget}\\:IN\\=%{WORD:shorewall.icmp.fwin} OUT\\=%{WORD:shorewall.icmp.fwout} SRC\\=%{IPORHOST:shorewall.icmp.fwsrc} DST\\=%{IPORHOST:shorewall.icmp.fwdst} .*? PROTO\\=%{WORD:shorewall.icmp.fwproto} .*?", 3 | "SHOREWALL_MSG": "(?:%{SHOREWALL_TCP}|%{SHOREWALL_UDP}|%{SHOREWALL_ICMP})", 4 | "SHOREWALL_TCP": ".*?%{WORD:shorewall.tcp.fwrule}\\:%{WORD:shorewall.tcp.wtarget}\\:IN\\=%{WORD:shorewall.tcp.win} OUT\\=\\s*MAC\\=%{DATA:shorewall.tcp.fwmac} SRC\\=%{IPORHOST:shorewall.tcp.fwsrc} DST\\=%{IPORHOST:shorewall.tcp.fwdst} .*? PROTO\\=%{WORD:shorewall.tcp.fwproto} SPT\\=%{INT:shorewall.tcp.fwspt} DPT\\=%{INT:shorewall.tcp.fwdpt} .*?", 5 | "SHOREWALL_UDP": ".*?%{WORD:shorewall.udp.fwrule}\\:%{WORD:shorewall.udp.fwtarget}\\:IN\\=%{WORD:shorewall.udp.fwin} OUT\\=%{WORD:shorewall.udp.fwout}.*?SRC\\=%{IPORHOST:shorewall.udp.fwsrc} DST\\=%{IPORHOST:shorewall.udp.fwdst} .*? PROTO\\=%{WORD:shorewall.udp.fwproto} SPT\\=%{INT:shorewall.udp.fwspt} DPT\\=%{INT:shorewall.udp.fwdpt} .*?" 6 | } 7 | -------------------------------------------------------------------------------- /files/snoopy.json: -------------------------------------------------------------------------------- 1 | { 2 | "SNOOPY_MSG": "(?:- snoopy\\[%{INT}\\]::\\s+)?\\[uid:%{INT:snoopy.uid}\\s+(username:%{USERNAME_PAT:snoopy.username}\\s+)?sid:%{INT:snoopy.sid}\\s+tty:%{DATA:snoopy.tty}\\s+cwd:%{DATA:snoopy.cwd}\\s+filename:%{DATA:snoopy.executable}\\]:\\s+%{GREEDYDATA:snoopy.command}" 3 | } 4 | -------------------------------------------------------------------------------- /files/ssh.json: -------------------------------------------------------------------------------- 1 | { 2 | "SSH_AUTH_REFUSED": ".*?Authentication refused: bad ownership or modes for file %{PATH:ssh.ownerpath}", 3 | "SSH_DISCONNECT": ".*?Received disconnect from %{IPORHOST:ssh.ip}: %{NONNEGINT}: (?disconnected by user)$", 4 | "SSH_INPUT_USERAUTH": ".*?input_userauth_request: (?invalid user %{USERNAME_PAT:ssh.username})", 5 | "SSH_INVALID_USER": "Invalid user", 6 | "SSH_MESSAGE_CONNECTION": "%{SSH_STATUS:status}", 7 | "SSH_MESSAGE_INVALID": "%{SSH_INVALID_USER:status} %{USERNAME_PAT} from %{IP}", 8 | "SSH_MSG": "(?:%{SSH_VALID_INVALID}|%{SSH_PAM_SESSION}|%{SSH_DISCONNECT}|%{SSH_SUBSYSTEM}|%{SSH_INPUT_USERAUTH}|%{SSH_AUTH_REFUSED})", 9 | "SSH_PAM_SESSION": ".*?pam_unix\\(sshd:session\\): session %{WORD:ssh.session} for user %{USERNAME_PAT}:ssh.username", 10 | "SSH_STATUS": "(?:Accepted|Failed)", 11 | "SSH_SUBSYSTEM": ".*?subsystem request for (?.*)", 12 | "SSH_VALID_INVALID": ".*?(?Accepted|Failed|Invalid user)(?: %{WORD:ssh.method} for)? %{USERNAME_PAT:ssh.username} from %{IPORHOST:ssh.op}(?: port %{NONNEGINT:ssh.port})?" 13 | } 14 | -------------------------------------------------------------------------------- /runtest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: latin-1 -*- 3 | # 4 | # Copyright 2009-2022 Ghent University 5 | # 6 | # This file is part of logstash-patterns, 7 | # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), 8 | # with support of Ghent University (http://ugent.be/hpc), 9 | # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), 10 | # the Hercules foundation (http://www.herculesstichting.be/in_English) 11 | # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). 12 | # 13 | # http://github.com/hpcugent/logstash-patterns 14 | # 15 | # logstash-patterns is free software: you can redistribute it and/or modify 16 | # it under the terms of the GNU General Public License as published by 17 | # the Free Software Foundation v2. 18 | # 19 | # logstash-patterns is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU General Public License 25 | # along with logstash-patterns. If not, see . 26 | # 27 | 28 | import argparse 29 | import glob 30 | import logging 31 | import os 32 | import shutil 33 | import sys 34 | from vsc.utils.run import run_asyncloop 35 | 36 | """ 37 | Test the grok patterns for vector usage. 38 | 39 | To test a new expression, add a file with the proper contents in toml 40 | format. Add the expectation in vrl, see e.g., lmod.toml 41 | 42 | @author: Stijn De Weirdt (Ghent University) 43 | @author: Andy Georges (Ghent University) 44 | """ 45 | 46 | 47 | # Where we store the grok patterns in JSON format 48 | GROK_CONFIG_DIR = "/tmp/grok" 49 | 50 | VECTOR_COMMAND = ["vector", "test", "tests/vector.toml"] # this will be the file with the tests 51 | 52 | 53 | def prep_grok(): 54 | """Prepare the environment""" 55 | try: 56 | shutil.rmtree(GROK_CONFIG_DIR) 57 | except: 58 | pass 59 | shutil.copytree(os.path.join(os.getcwd(), "files"), GROK_CONFIG_DIR) 60 | 61 | 62 | def main(): 63 | """The main, duh.""" 64 | 65 | parser = argparse.ArgumentParser(description="Process some integers.") 66 | parser.add_argument("--names", dest="names", action="store", help="Names of the tests to run, comma-separated") 67 | 68 | args = parser.parse_args() 69 | 70 | cfg_name = "vector.toml" 71 | cfg_file = os.path.join(os.getcwd(), "tests", cfg_name) 72 | 73 | if not os.path.isfile(cfg_file): 74 | logging.error("Could not find vector configfile %s", cfg_file) 75 | sys.exit(1) 76 | 77 | # copy the grok patterns 78 | prep_grok() 79 | 80 | if args.names: 81 | tests = map(lambda s: os.path.join("tests", s.strip()), args.names.split(",")) 82 | else: 83 | tests = glob.glob("tests/*.toml") 84 | 85 | # filter out vector.toml 86 | tests = filter(lambda s: not s.endswith("vector.toml"), tests) 87 | 88 | for test in tests: 89 | print(f"Running test {test}") 90 | ec, stdout = run_asyncloop(cmd=VECTOR_COMMAND + [test]) 91 | if ec != 0: 92 | logging.error(f"Test {test} failed: {stdout}") 93 | 94 | 95 | if __name__ == "__main__": 96 | main() 97 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_rpm] 2 | requires=perl(Template) 3 | [install] 4 | install-data=/usr/share/templates/ 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: latin-1 -*- 3 | # 4 | # Copyright 2009-2022 Ghent University 5 | # 6 | # This file is part of logstash-patterns, 7 | # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), 8 | # with support of Ghent University (http://ugent.be/hpc), 9 | # the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), 10 | # the Hercules foundation (http://www.herculesstichting.be/in_English) 11 | # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). 12 | # 13 | # http://github.com/hpcugent/logstash-patterns 14 | # 15 | # logstash-patterns is free software: you can redistribute it and/or modify 16 | # it under the terms of the GNU General Public License as published by 17 | # the Free Software Foundation v2. 18 | # 19 | # logstash-patterns is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | # GNU General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU General Public License 25 | # along with logstash-patterns. If not, see . 26 | # 27 | """Basic setup.py for building the logstash grok patterns""" 28 | 29 | import sys 30 | import os 31 | from distutils.core import setup 32 | import glob 33 | 34 | setup( 35 | name="logstash-patterns", 36 | version="3.0.0", 37 | description="Grok patterns for vector (JSON formatted)", 38 | long_description="""Grok patterns for parsing log messages with vector. 39 | 40 | Can be debugged on https://grokdebug.herokuapp.com/ or via the runtest.py in tests subdirectory 41 | """, 42 | license="LGPL", 43 | author="HPC UGent", 44 | author_email="hpc-admin@lists.ugent.be", 45 | # keep MANIFEST.in in sync (EL6 packaging issue) 46 | data_files=[("/usr/share/vector", glob.glob("files/*"))], 47 | url="http://www.ugent.be/hpc", 48 | ) 49 | -------------------------------------------------------------------------------- /tests/data/00_first: -------------------------------------------------------------------------------- 1 | """ 2 | { 3 | "raw": None, 4 | "expected" : None, 5 | }, 6 | 7 | """ 8 | data = [ 9 | ] 10 | -------------------------------------------------------------------------------- /tests/data/ceph: -------------------------------------------------------------------------------- 1 | data = [ 2 | 3 | { 4 | "raw":"<30>Apr 25 16:31:56 ceph002 ceph-mon: 2014-04-25 16:31:55.419574 mon.0 10.141.8.180:6789/0 446576 : [INF] pgmap v630722: 1312 pgs: 41 active+remapped+wait_backfill+backfill_toofull, 24 active+recovery_wait+remapped, 1033 active+clean, 19 active+recovering+remapped, 4 active+degraded+remapped+backfilling, 1 active+degraded+remapped+wait_backfill, 4 active+recovery_wait, 11 active+recovering, 8 active+clean+scrubbing+deep, 91 active+remapped+backfilling, 62 active+remapped+wait_backfill, 3 active+degraded+remapped+wait_backfill+backfill_toofull, 10 active+remapped+backfill_toofull, 1 active+recovering+degraded+remapped; 36753 GB data, 101 TB used, 30931 GB / 131 TB avail; 0 B/s wr, 161 op/s; 12675482/110833141 objects degraded (11.437%); 6157 kB/s, 6 objects/s recovering", 5 | "expected": { 6 | "actwrite": 0, 7 | "actops": 161, 8 | "objdegraded": 12675482, 9 | "objtotal": 110833141, 10 | "objrecovthr": 6157000, 11 | "objrecovhz": 6, 12 | "volumedata": 36753000000000, 13 | "volumeused": 101000000000000, 14 | "volumeavail": 30931000000000, 15 | "volumetotal": 131000000000000, 16 | 'objdegraded': 12675482, 17 | 'objrecovhz': 6, 18 | 'objrecovthr': 6157000, 19 | 'objtotal': 110833141, 20 | 'pgmapv': '630722', 21 | 'pgs': 1312, 22 | 'program': 'ceph-mon', 23 | 'syslog_abspri': u'30', 24 | '@source_host': 'ceph002', 25 | 'active_clean': u'1033', 26 | } 27 | }, 28 | 29 | { 30 | "raw":"<33>Apr 12 20:57:56 mds01 ceph-mon: 2016-04-12 20:57:04.437911 mon.0 [INF] pgmap v2758229: 4160 pgs: 2 active+remapped+backfilling, 2 active+recovering+degraded, 101 active+remapped+wait_backfill, 1503 active+recovery_wait+degraded, 4 active+clean+scrubbing+deep, 2548 active+clean; 171 TB data, 242 TB used, 631 TB / 874 TB avail; 1529 B/s rd, 202 MB/s wr, 723 op/s; 865075/1400635626 objects degraded (0.062%); 81400/1400635626 objects misplaced (0.006%); 46084 kB/s, 20 objects/s recovering", 31 | "expected": { 32 | "actread": 1529, 33 | "actwrite": 202000000, 34 | "actops": 723, 35 | "objdegraded": 865075, 36 | "objtotal": 1400635626, 37 | "objmisplaced": 81400, 38 | "objrecovthr": 46084000, 39 | "objrecovhz": 20, 40 | "volumedata": 171000000000000, 41 | "volumeused": 242000000000000, 42 | "volumeavail": 631000000000000, 43 | "volumetotal": 874000000000000, 44 | 'pgmapv': '2758229', 45 | 'pgs': 4160, 46 | 'program': 'ceph-mon', 47 | 'syslog_abspri': u'33', 48 | '@source_host': 'mds01', 49 | 'active_clean': u'2548', 50 | } 51 | }, 52 | 53 | ] 54 | -------------------------------------------------------------------------------- /tests/data/general_syslog: -------------------------------------------------------------------------------- 1 | """ 2 | { 3 | "raw": "", 4 | "expected" : None, 5 | }, 6 | 7 | The cases here should NOT be matched by anything that we explicitely check for. 8 | 9 | """ 10 | data = [ 11 | { 12 | "raw": "<30>May 25 00:00:00 betelgeuse zaphod[30]: yup, producing a pan galactic gargle blaster", 13 | "expected": { 14 | "@message": "yup, producing a pan galactic gargle blaster" 15 | } 16 | } 17 | 18 | ] 19 | -------------------------------------------------------------------------------- /tests/data/jube: -------------------------------------------------------------------------------- 1 | """ 2 | { 3 | "raw": "", 4 | "expected" : None, 5 | }, 6 | 7 | """ 8 | data = [ 9 | { 10 | "raw": "<14>1 2015-10-27T19:32:37.664122+01:00 node711 jube[25037]: - jube[25037]:: JUBE cluster=swalot jube_id=29794 jube_label=None time=2015-10-27T19:28:45 seconds=1445970525 user=vsc40075 host=node711 module=STREAM/5.10-intel-2015a input=100M-OpenMP pbs_job_id=29794 pbs_nodes=1 pbs_ppn= exec_host=node711.dugtrio.gent.vsc/0-11 cputime=2174 walltime=186 compute_walltime= stream_copy=41843.1 stream_scale=40382.8 stream_add=41208.2 stream_triad=40869.2", 11 | "expected": { 12 | u'cputime': 2174, 13 | u'exec_host': u'node711.dugtrio.gent.vsc/0-11', 14 | u'input': u'100M-OpenMP', 15 | u'cluster': u'swalot', 16 | u'jube_id': u'29794', 17 | u'jube_label': u'None', 18 | u'module': u'STREAM/5.10-intel-2015a', 19 | u'pbs_job_id': u'29794', 20 | u'pbs_nodes': u'1', 21 | u'seconds': 1445970525, 22 | u'stream_add': 41208.2, 23 | u'stream_copy': 41843.1, 24 | u'stream_scale': 40382.8, 25 | u'stream_triad': 40869.2, 26 | u'time': u'2015-10-27T19:28:45', 27 | u'username': u'vsc40075', 28 | u'walltime': 186, 29 | } 30 | }, 31 | { 32 | "raw": "<14>1 2015-10-30T16:18:07.980567+01:00 node706 jube[32460]: - jube[32460]:: JUBE cluster=swalot jube_id=29833 jube_label=None time=2015-10-30T16:12:46 seconds=1446217966 user=vsc40075 host=node706 module=WRF/3.6.1-intel-2015a-dmpar input=conus_12_km pbs_job_id=29833 pbs_nodes=2 pbs_ppn= exec_host=node706.dugtrio.gent.vsc/0-11,node708.dugtrio.gent.vsc/0-11 cputime=2363 walltime=280 compute_walltime=252 wrf_success=true", 33 | "expected": { 34 | u'compute_walltime': 252, 35 | u'cputime': 2363, 36 | u'exec_host': u'node706.dugtrio.gent.vsc/0-11,node708.dugtrio.gent.vsc/0-11', 37 | u'input': u'conus_12_km', 38 | u'cluster': u'swalot', 39 | u'jube_id': u'29833', 40 | u'jube_label': u'None', 41 | u'module': u'WRF/3.6.1-intel-2015a-dmpar', 42 | u'pbs_job_id': u'29833', 43 | u'pbs_nodes': u'2', 44 | u'program': u'jube', 45 | u'seconds': 1446217966, 46 | u'time': u'2015-10-30T16:12:46', 47 | u'username': u'vsc40075', 48 | u'walltime': 280, 49 | u'success': True, 50 | } 51 | }, 52 | { 53 | "raw": "<14>1 2015-10-27T19:32:37.664122+01:00 node711 jube[25037]: - jube[25037]:: JUBE cluster=swalot jube_id=29794 jube_label=None time=2015-10-27T19:28:45 seconds=1445970525 user=vsc40075 host=node711 module=STREAM/5.10-intel-2015a input=100M-OpenMP pbs_job_id=29794 pbs_nodes=1 pbs_ppn= exec_host=node711.dugtrio.gent.vsc/0-11 cputime=2174 walltime=186 compute_walltime= wrf_success=true", 54 | "expected": { 55 | u'cputime': 2174, 56 | u'exec_host': u'node711.dugtrio.gent.vsc/0-11', 57 | u'input': u'100M-OpenMP', 58 | u'cluster': u'swalot', 59 | u'jube_id': u'29794', 60 | u'jube_label': u'None', 61 | u'module': u'STREAM/5.10-intel-2015a', 62 | u'pbs_job_id': u'29794', 63 | u'pbs_nodes': u'1', 64 | u'seconds': 1445970525, 65 | u'time': u'2015-10-27T19:28:45', 66 | u'username': u'vsc40075', 67 | u'walltime': 186, 68 | u'success': True 69 | }, 70 | }, 71 | { 72 | "raw": "<14>1 2015-11-26T17:39:27.190173+01:00 node2422 jube[512]: - jube[512]:: JUBE cluster=swalot jube_id=170028 jube_label=golett-tier2-2015q4 time=2015-11-26T17:25:34 seconds=1448555134 user=vsc40075 host=node2422 module=STREAM/5.10-intel-2015a input=1B-OpenMP pbs_job_id=170028 pbs_nodes=node2422.golett.gent.vsc pbs_ppn= exec_host=node2422.golett.gent.vsc/0-23 cputime=18598 walltime=784 compute_walltime= stream_copy=93683.4 stream_scale=96370.9 stream_add=103748.4 stream_triad=103493.3", 73 | "expected" : { 74 | u'cputime': 18598, 75 | u'exec_host': u'node2422.golett.gent.vsc/0-23', 76 | u'input': u'1B-OpenMP', 77 | u'cluster': u'swalot', 78 | u'jube_id': u'170028', 79 | u'jube_label': u'golett-tier2-2015q4', 80 | u'module': u'STREAM/5.10-intel-2015a', 81 | u'pbs_job_id': u'170028', 82 | u'pbs_nodes': u'node2422.golett.gent.vsc', 83 | u'program': u'jube', 84 | u'seconds': 1448555134, 85 | u'stream_add': 103748.4, 86 | u'stream_copy': 93683.4, 87 | u'stream_scale': 96370.9, 88 | u'stream_triad': 103493.3, 89 | u'time': u'2015-11-26T17:25:34', 90 | u'username': u'vsc40075', 91 | u'walltime': 784, 92 | }, 93 | }, 94 | { 95 | "raw": "<14>1 2016-08-01T11:48:30.071651+02:00 node2602 jube[89798]: JUBE cluster=swalot jube_id=21235 jube_label=None time=2016-08-01T11:45:23 seconds=1470044723 user=vsc40075 host=node2602 module=IMB/4.0.1-intel-2015a input=mpi1 pbs_job_id=21235 pbs_nodes=2 pbs_ppn= exec_host=node2602.swalot.gent.vsc/0-19,node2603.swalot.gent.vsc/0-19 cputime=751 walltime=176 compute_walltime= benchmark=Reduce bytes=1048576 iter=40 t_min_us=3576.84 t_max_us=3693.70 t_avg_us=3651.12 rate=", 96 | "expected": { 97 | u'cputime': 751, 98 | u'exec_host': u'node2602.swalot.gent.vsc/0-19,node2603.swalot.gent.vsc/0-19', 99 | u'input': u'mpi1', 100 | u'cluster': u'swalot', 101 | u'jube_id': u'21235', 102 | u'jube_label': u'None', 103 | u'module': u'IMB/4.0.1-intel-2015a', 104 | u'pbs_job_id': u'21235', 105 | u'pbs_nodes': u'2', 106 | u'seconds': 1470044723, 107 | u'benchmark': u'Reduce', 108 | u'bytes': 1048576, 109 | u'iter': 40, 110 | u't_min_us': 3576.84, 111 | u't_max_us': 3693.70, 112 | u't_avg_us': 3651.12, 113 | }, 114 | }, 115 | ] 116 | -------------------------------------------------------------------------------- /tests/data/lmod: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw": "<13>1 2016-06-07T17:50:27.866534+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, cmd=load, args=-GCC", 4 | "expected": { 5 | "@source_host": "node2159", 6 | "program": "lmod", 7 | "username": "vsc40307", 8 | "cluster": "delcatty", 9 | "jobid": "434.master16.delcatty.gent.vsc", 10 | "command": "load", 11 | "arguments": "-GCC", 12 | } 13 | }, 14 | { 15 | "raw": "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016a", 16 | "expected": { 17 | "program": "lmod", 18 | "@source_host": "node2159", 19 | "program": "lmod", 20 | "username": "vsc40307", 21 | "cluster": "delcatty", 22 | "jobid": "434.master16.delcatty.gent.vsc", 23 | "userload": "yes", 24 | "module_name": "intel", 25 | "module_version": "2016a", 26 | "module_path": "/apps/gent/SL6/sandybridge/modules/all/intel/2016a", 27 | } 28 | }, 29 | { 30 | "raw": "<13>1 2016-06-07T17:50:22.495571+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, userload=no, module=binutils/2.25-GCCcore-4.9.3, fn=/apps/gent/SL6/sandybridge/modules/all/binutils/2.25-GCCcore-4.9.3", 31 | "expected": { 32 | "@source_host": "node2159", 33 | "program": "lmod", 34 | "username": "vsc40307", 35 | "cluster": "delcatty", 36 | "jobid": "434.master16.delcatty.gent.vsc", 37 | "userload": "no", 38 | "module_name": "binutils", 39 | "module_version": "2.25-GCCcore-4.9.3", 40 | "module_path": "/apps/gent/SL6/sandybridge/modules/all/binutils/2.25-GCCcore-4.9.3", 41 | } 42 | }, 43 | { 44 | "raw": "<13>1 2016-06-07T17:50:13.951838+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, cmd=spider, args=^ba$", 45 | "expected": { 46 | "@source_host": "node2159", 47 | "program": "lmod", 48 | "username": "vsc40307", 49 | "cluster": "delcatty", 50 | "jobid": "434.master16.delcatty.gent.vsc", 51 | "command": "spider", 52 | "arguments": "^ba$", 53 | } 54 | }, 55 | { 56 | "raw": "<13>1 2016-06-07T17:29:12.762154+02:00 node2626 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=, cmd=list, args=", 57 | "expected": { 58 | "@source_host": "node2626", 59 | "program": "lmod", 60 | "username": "vsc40307", 61 | "cluster": "delcatty", 62 | "command": "list", 63 | } 64 | }, 65 | { 66 | "raw": "<13>1 2016-06-07T17:29:14.762154+02:00 node2626 lmod: - lmod:: username=vsc40307, cluster=, jobid=, cmd=list, args=", 67 | "expected": { 68 | "@source_host": "node2626", 69 | "program": "lmod", 70 | "username": "vsc40307", 71 | "command": "list", 72 | } 73 | }, 74 | { 75 | "raw": "<13>1 2016-06-08T09:20:24.111835+02:00 node2626 lmod: - lmod:: username=vsc40307, cluster=swalot, jobid=, cmd=swap, args=GCC/6.1.0-2.25 GCC/4.9.3-2.25", 76 | "expected": { 77 | "@source_host": "node2626", 78 | "program": "lmod", 79 | "username": "vsc40307", 80 | "cluster": "swalot", 81 | "command": "swap", 82 | "arguments": "GCC/6.1.0-2.25 GCC/4.9.3-2.25", 83 | } 84 | }, 85 | ] 86 | -------------------------------------------------------------------------------- /tests/data/modulecmd: -------------------------------------------------------------------------------- 1 | data = [ 2 | 3 | { 4 | "raw" : "<13>1 2014-04-25T10:56:34.361503+02:00 node2110 modulecmd: - modulecmd:: Uvsc41088 Hnode2110.delcatty.os VCdelcatty VALsandybridge ID347398.master15.delcatty.gent.vsc bash_load_ifort", 5 | "expected": { 6 | "program": "modulecmd", 7 | "username": "vsc41088", 8 | "cluster": "delcatty", 9 | "architecture": "sandybridge", 10 | "jobid": "347398.master15.delcatty.gent.vsc", 11 | "modulecmd_shell": "bash", 12 | "modulecmd_cmd": "load", 13 | "module_name": "ifort", 14 | "@source_host": "node2110", 15 | } 16 | }, 17 | 18 | { 19 | "raw" : "<13>Apr 28 16:13:54 node843 modulecmd: Uvsc40769 Hnode843.raichu.os VCraichu VALsandybridge ID463808.master13.raichu.gent.vsc bash_load_scikit-learn@0.14-ictce-5.5.0-Python-2.7.6", 20 | "expected" : { 21 | 'architecture': 'sandybridge', 22 | 'cluster': 'raichu', 23 | 'jobid': '463808.master13.raichu.gent.vsc', 24 | 'module_name': 'scikit-learn', 25 | 'module_version': '0.14-ictce-5.5.0-Python-2.7.6', 26 | 'modulecmd_cmd': 'load', 27 | 'modulecmd_shell': 'bash', 28 | 'program': 'modulecmd', 29 | 'syslog_abspri': '13', 30 | 'username': 'vsc40769', 31 | }, 32 | }, 33 | 34 | { 35 | "raw": "<13>Apr 28 16:14:14 node861 modulecmd: Uvsc40769 Hnode861.raichu.os VCraichu VALsandybridge ID bash_load_cluster@raichu", 36 | "expected" : { 37 | 'architecture': 'sandybridge', 38 | 'cluster': 'raichu', 39 | 'module_version': 'raichu', 40 | 'module_name': 'cluster', 41 | 'modulecmd_cmd': 'load', 42 | 'modulecmd_shell': 'bash', 43 | 'program': 'modulecmd', 44 | 'syslog_abspri': '13', 45 | 'username': 'vsc40769', 46 | }, 47 | }, 48 | 49 | 50 | ] 51 | 52 | -------------------------------------------------------------------------------- /tests/data/nfs: -------------------------------------------------------------------------------- 1 | data = [ 2 | 3 | { 4 | "raw": "<5>1 2014-04-14T03:47:01.011933+02:00 node2011 kernel: - kernel:: nfs: server 172.24.11.14 not responding, timed out", 5 | "expected" :{ 6 | '@source_host': 'node2011', 7 | 'nfsreason': 'not responding, timed out', 8 | 'nfsserver': '172.24.11.14', 9 | 'program': 'kernel', 10 | 'syslog_abspri': '5', 11 | 'syslog_version': '1', 12 | } 13 | }, 14 | { 15 | "raw": "<4>1 2014-03-28T16:04:31.916838+01:00 node001 kernel: - kernel:: Error: state recovery failed on NFSv4 server 172.24.11.14 with error 2", 16 | "expected" :{ 17 | 'nfsreason': 'state recovery failed', 18 | 'nfsserver': '172.24.11.14', 19 | 'program': 'kernel', 20 | 'syslog_abspri': '4', 21 | 'syslog_version': '1', 22 | } 23 | }, 24 | 25 | ] -------------------------------------------------------------------------------- /tests/data/opennebula: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw" : "2015-03-19T08:13:12.916717+01:00 one01 oned: [Z0][VMM][D]: VM 65 successfully monitored: STATE=a CPU=1.0 MEMORY=4381156 NETRX=131913740597 NETTX=25058980345 DISK_SIZE=[ ID=0, SIZE=10240 ] DISK_SIZE=[ ID=1, SIZE=0 ]", 4 | "expected" : { 5 | "level" : "D", 6 | "vmid" : 65, 7 | "vmstate" : "a", 8 | "vmucpu" : 1.0, 9 | "vmmem" : 4381156, 10 | "vmnetrx" : 131913740597, 11 | "vmnettx" : 25058980345, 12 | } 13 | }, 14 | { 15 | "raw": "2015-03-19T10:20:53.019214+01:00 one01 oned[16433]: [VM 149][Z0][LCM][I]: New VM state is RUNNING", 16 | "expected": { 17 | "level" : "I", 18 | "vmid" : u'149', 19 | "vmstate" : "RUNNING", 20 | } 21 | }, 22 | { 23 | "raw": "2015-03-19T13:49:29.857986+01:00 one01 oned[16433]: [Z0][ONE][E]: Error monitoring Host hyp101 (29): Error executing probes", 24 | "expected": { 25 | "level" : "E", 26 | "hypervisor" : "hyp101", 27 | "hostid" : 29, 28 | } 29 | }, 30 | { 31 | "raw": "2015-03-19T10:20:50.038092+01:00 one01 mm_sched[16436]: [Z0][VM][I]: Dispatching VM 149 to host 32 and datastore 0", 32 | "expected": { 33 | "level" : "I", 34 | "vmid" : u'149', 35 | "hostid" : u'32', 36 | "datastoreid" : 0, 37 | } 38 | }, 39 | { 40 | "raw": "2015-03-19T10:37:40.619650+01:00 one01 oned[16433]: [VM 149][Z0][VMM][W]: Ignored: LOG I 149 Successfully execute transfer manager driver operation: tm_delete.", 41 | "expected": { 42 | "level" : "W", 43 | "vmid" : u'149', 44 | } 45 | }, 46 | ] 47 | -------------------------------------------------------------------------------- /tests/data/quattor: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw" : "2015-06-08T11:27:43.319841+02:00 vulpix04 ncd[6581]: - ncd[6581]:: 5 errors, 4 warnings executing configure", 4 | "expected" : { 5 | "ncderrors" : 5, 6 | "ncdwarnings" : 4, 7 | } 8 | }, 9 | { 10 | "raw" : "2015-06-08T11:27:04.525448+02:00 vulpix04 ncd[6581]: - ncd[6581]:: NCM-NCD version 15.4.0-rc13-SNAPSHOT started by root at: Mon Jun 8 11:27:04 2015", 11 | "expected" : { 12 | "ncdversion" : "15.4.0-rc13-SNAPSHOT", 13 | } 14 | }, 15 | ] 16 | -------------------------------------------------------------------------------- /tests/data/shorewall: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw" : "2016-04-07T09:27:26.729986+02:00 gastly kernel: - kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421", 4 | "expected" : { 5 | "fwrule" : "ipmi2ext", 6 | "fwtarget" : "REJECT", 7 | "fwin" : "em4", 8 | "fwout" : "em3", 9 | "fwsrc" : "10.0.0.2", 10 | "fwdst" : "10.0.0.1", 11 | "fwproto" : "ICMP", 12 | } 13 | }, 14 | { 15 | "raw" : "2016-03-31T23:45:27.615225+02:00 gastly kernel: - kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37", 16 | "expected" : { 17 | "fwrule" : "ipmi2int", 18 | "fwtarget" : "REJECT", 19 | "fwin" : "em4", 20 | "fwout" : "em1", 21 | "fwsrc" : "10.0.0.2", 22 | "fwdst" : "10.0.0.1", 23 | "fwproto" : "UDP", 24 | u"fwspt" : u"57002", 25 | u"fwdpt" : u"53", 26 | } 27 | }, 28 | { 29 | "raw" : "2016-03-29T16:10:49.386951+02:00 gligar03 kernel: - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0", 30 | "expected" : { 31 | "fwrule" : "ext2fw", 32 | "fwtarget" : "REJECT", 33 | "fwin" : "em3", 34 | "fwsrc" : "78.0.0.1", 35 | "fwdst" : "150.0.0.1", 36 | "fwmac" : "aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00", 37 | "fwproto" : "TCP", 38 | "fwspt" : 60048, 39 | "fwdpt" : 22, 40 | } 41 | }, 42 | ] 43 | -------------------------------------------------------------------------------- /tests/data/snoopy: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw": "<86>Apr 26 09:00:55 master2 snoopy[1301]: [uid:110 sid:8322 tty: cwd:/ filename:/bin/cut]: cut -d ; -f 4 ", 4 | "expected" : { 5 | "program": "snoopy", 6 | "pid": "1301", 7 | "uid": 110, 8 | "sid": 8322, 9 | "command": "cut -d ; -f 4 ", 10 | "cwd": "/", 11 | "executable": "/bin/cut", 12 | "@source_host": "master2", 13 | } 14 | }, 15 | { 16 | "raw": "<86>1 2015-12-19T17:30:22.145124+01:00 gligar03 snoopy[27316]: - snoopy[27316]:: [uid:110 sid:9379 tty:(none) cwd:/ filename:/usr/lib64/nagios/plugins/hpc/check_ifutil.pl]: /usr/lib64/nagios/plugins/hpc/check_ifutil.pl -i em1.295 -w 90 -c 95 -p -b 10000m", 17 | "expected" : { 18 | "program": "snoopy", 19 | "pid": "27316", 20 | "uid": 110, 21 | "sid": 9379, 22 | "command": "/usr/lib64/nagios/plugins/hpc/check_ifutil.pl -i em1.295 -w 90 -c 95 -p -b 10000m", 23 | "cwd": "/", 24 | "executable": "/usr/lib64/nagios/plugins/hpc/check_ifutil.pl", 25 | "tty": "(none)", 26 | "@source_host": "gligar03", 27 | } 28 | }, 29 | { 30 | "raw": "<86>1 2015-12-20T09:59:40.844711+01:00 gligar03 snoopy[46513]: - snoopy[46513]:: [uid:2540337 sid:19403 tty:ERROR(ttyname_r->EUNKNOWN) cwd:/vscmnt/gent_vulpix/_/user/home/gent/vsc403/vsc40337/UCS_LABELLED_NEW/20000_to_30000 filename:/usr/bin/qsub]: qsub -l walltime=72:00:00 job7_21293_30000_doit", 31 | "expected" : { 32 | "program": "snoopy", 33 | "pid": "46513", 34 | "cwd": "/vscmnt/gent_vulpix/_/user/home/gent/vsc403/vsc40337/UCS_LABELLED_NEW/20000_to_30000", 35 | "uid": 2540337, 36 | "sid": 19403, 37 | "command": "qsub -l walltime=72:00:00 job7_21293_30000_doit", 38 | "executable": "/usr/bin/qsub", 39 | "tty": "ERROR(ttyname_r->EUNKNOWN)", 40 | "@source_host": "gligar03", 41 | } 42 | }, 43 | { 44 | "raw": "<86>1 2015-12-20T09:59:36.133039+01:00 master13 snoopy[36344]: [uid:0 sid:36288 tty: cwd:/ filename:/sbin/pidof]: pidof -c -o 36318 -o 36292 -o %PPID -x cdp-listend", 45 | "expected" : { 46 | "program": "snoopy", 47 | "pid": "36344", 48 | "cwd": "/", 49 | "uid": 0, 50 | "sid": 36288, 51 | "command": "pidof -c -o 36318 -o 36292 -o %PPID -x cdp-listend", 52 | "executable": "/sbin/pidof", 53 | "@source_host": "master13", 54 | } 55 | }, 56 | { 57 | "raw": "<86>1 2015-12-08T16:59:54.996042+01:00 gligar01 snoopy[30093]: - snoopy[30093]:: [uid:2540003 sid:29974 tty:/dev/pts/7 cwd:/vscmnt/gent_vulpix/_/user/home/gent/vsc400/vsc40003 filename:/user/home/gent/vsc400/vsc40003/easybuild_easyinstalled/bin/easy_install]: easy_install -U --prefix /user/home/gent/vsc400/vsc40003/easybuild_easyinstalled https://github.com/hpcugent/easybuild-framework/archive/develop.tar.gz", 58 | "expected" : { 59 | "program": "snoopy", 60 | "pid": "30093", 61 | "cwd": "/vscmnt/gent_vulpix/_/user/home/gent/vsc400/vsc40003", 62 | "uid": 2540003, 63 | "sid": 29974, 64 | "command": "easy_install -U --prefix /user/home/gent/vsc400/vsc40003/easybuild_easyinstalled https://github.com/hpcugent/easybuild-framework/archive/develop.tar.gz", 65 | "executable": "/user/home/gent/vsc400/vsc40003/easybuild_easyinstalled/bin/easy_install", 66 | "tty": "/dev/pts/7", 67 | "@source_host": "gligar01", 68 | } 69 | }, 70 | { 71 | "raw": "<86>1 2016-01-05T18:38:32.550186+01:00 master1 snoopy[12006]: [uid:2009 username:wpoelman sid:11944 tty:(none) cwd:/home/wpoelman filename:/usr/bin/tr]: tr [:lower:] [:upper:]", 72 | "expected" : { 73 | 'program':'snoopy', 74 | 'command':'tr [:lower:] [:upper:]', 75 | 'cwd':'/home/wpoelman', 76 | 'executable':'/usr/bin/tr', 77 | 'pid':'12006', 78 | 'sid': 11944, 79 | 'tty':'(none)', 80 | 'uid': 2009, 81 | 'username':'wpoelman', 82 | "@source_host": "master1", 83 | } 84 | }, 85 | { 86 | "raw": "<86>1 2016-01-07T16:08:52.407416+01:00 gligar03 snoopy[43645]: [uid:2541677 sid:39946 tty:(none) cwd:/vscmnt/gent_vulpix/_/user/home/gent/vsc416/vsc41677/project/AAN/HF_6_31++Gdp/NBO filename:/bin/cut]: cut -d. -f2", 87 | "expected" : { 88 | 'program':'snoopy', 89 | 'command':'cut -d. -f2', 90 | 'cwd':'/vscmnt/gent_vulpix/_/user/home/gent/vsc416/vsc41677/project/AAN/HF_6_31++Gdp/NBO', 91 | 'executable':'/bin/cut', 92 | 'pid':'43645', 93 | 'sid': 39946, 94 | 'tty':'(none)', 95 | 'uid': 2541677, 96 | "@source_host": "gligar03", 97 | } 98 | }, 99 | ] 100 | -------------------------------------------------------------------------------- /tests/data/ssh: -------------------------------------------------------------------------------- 1 | 2 | data = [ 3 | 4 | { 5 | "raw": "<86>Apr 26 11:25:44 node2104 sshd[44668]: pam_unix(sshd:session): session opened for user vsc40000 by (uid=0)", 6 | "expected": { 7 | 'program': 'sshd', 8 | 'sshsession': 'opened', 9 | 'syslog_abspri': '86', 10 | 'username': 'vsc40000', 11 | } 12 | }, 13 | { 14 | "raw": "<86>Apr 26 11:25:44 node2104 sshd[44556]: pam_unix(sshd:session): session closed for user vsc40000", 15 | "expected": { 16 | 'program': 'sshd', 17 | 'sshsession': 'closed', 18 | 'syslog_abspri': '86', 19 | 'username': 'vsc40000', 20 | } 21 | }, 22 | 23 | { 24 | "raw": "<86>Apr 26 11:25:44 node2104 sshd[44668]: Accepted publickey for vsc40000 from 172.24.13.2 port 43493 ssh2", 25 | "expected" : { 26 | 'method': 'publickey', 27 | 'pid': '44668', 28 | 'port': '43493', 29 | 'program': 'sshd', 30 | 'status': 'Accepted', 31 | 'syslog_abspri': '86', 32 | 'username': 'vsc40000', 33 | } 34 | }, 35 | 36 | { 37 | "raw": "<86>Apr 26 11:25:44 node2104 sshd[44558]: Received disconnect from 172.24.13.2: 11: disconnected by user", 38 | "expected" : { 39 | 'ip': '172.24.13.2', 40 | 'pid': '44558', 41 | 'program': 'sshd', 42 | 'reason': 'disconnected by user', 43 | }, 44 | }, 45 | 46 | { 47 | "raw": "<86>Apr 26 11:22:40 gligar03 sshd[9697]: subsystem request for sftp", 48 | "expected" : { 49 | 'program': 'sshd', 50 | 'subsystem': 'sftp', 51 | }, 52 | }, 53 | { 54 | "raw": "<86>Apr 26 11:17:51 gligar02 sshd[11243]: input_userauth_request: invalid user vsc40000", 55 | "expected" : { 56 | 'inputuserauth': 'invalid user vsc40000', 57 | 'pid': '11243', 58 | 'program': 'sshd', 59 | }, 60 | }, 61 | 62 | # updated syslog raw data 26/03/2015 63 | 64 | { 65 | "raw": "2015-03-18T14:09:15.962645+01:00 node2432 sshd[26270]: Authentication refused: bad ownership or modes for file /vscmnt/gent_vulpix/_/user/home/gent/vsc404/vsc40479/.ssh/authorized_keys", 66 | "expected" : { 67 | 'ownerpath': '/vscmnt/gent_vulpix/_/user/home/gent/vsc404/vsc40479/.ssh/authorized_keys', 68 | 'pid': '26270', 69 | 'program': 'sshd', 70 | }, 71 | }, 72 | { 73 | "raw": "<86>1 2015-03-24T15:16:09.252883+01:00 gligar01 sshd[48764]: - sshd[48764]:: input_userauth_request: invalid user vsc40001", 74 | "expected" : { 75 | 'inputuserauth': 'invalid user vsc40001', 76 | 'pid': '48764', 77 | 'program': 'sshd', 78 | }, 79 | }, 80 | { 81 | "raw": "<86>1 2015-03-20T11:00:59.922704+01:00 gligar01 sshd[9215]: - sshd[9215]:: subsystem request for sftp", 82 | "expected" : { 83 | 'program': 'sshd', 84 | 'pid': '9215', 85 | 'subsystem': 'sftp', 86 | }, 87 | }, 88 | { 89 | "raw": "<86>1 2015-03-19T10:53:18.185704+01:00 hyp105 sshd[5170]: - sshd[5170]:: Received disconnect from 10.141.2.11: 11: disconnected by user", 90 | "expected" : { 91 | 'ip': '10.141.2.11', 92 | 'pid': '5170', 93 | 'program': 'sshd', 94 | 'reason': 'disconnected by user', 95 | }, 96 | }, 97 | { 98 | "raw": "<86>1 2015-03-19T10:42:03.145881+01:00 hyp108 sshd[8245]: - sshd[8245]:: Accepted publickey for oneadmin from 10.141.2.11 port 55141 ssh2: DSA d0:a6:12:8f:48:50:c5:78:31:bd:5b:40:1b:78:fb:88", 99 | "expected" : { 100 | 'method': 'publickey', 101 | 'pid': '8245', 102 | 'port': '55141', 103 | 'program': 'sshd', 104 | 'status': 'Accepted', 105 | 'syslog_abspri': '86', 106 | 'username': 'oneadmin', 107 | } 108 | }, 109 | { 110 | "raw": "<86>1 2015-03-19T11:20:40.126248+01:00 gligar02 sshd[4513]: - sshd[4513]:: pam_unix(sshd:session): session closed for user vsc41450", 111 | "expected": { 112 | 'program': 'sshd', 113 | 'sshsession': 'closed', 114 | 'syslog_abspri': '86', 115 | 'username': 'vsc41450', 116 | } 117 | }, 118 | { 119 | "raw": "<86>1 2015-03-19T12:37:06.245894+01:00 one01 sshd[20017]: - sshd[20017]:: pam_unix(sshd:session): session opened for user oneadmin by (uid=0)", 120 | "expected": { 121 | 'program': 'sshd', 122 | 'sshsession': 'opened', 123 | 'syslog_abspri': '86', 124 | 'username': 'oneadmin', 125 | } 126 | }, 127 | ] 128 | -------------------------------------------------------------------------------- /tests/data/su: -------------------------------------------------------------------------------- 1 | data = [ 2 | { 3 | "raw": "<86>1 2015-12-22T10:49:47.644749+01:00 node2131 su: - su:: pam_unix(su-l:session): session closed for user vsc40035", 4 | "expected": { 5 | 'program': 'su', 6 | '@message': '- su:: pam_unix(su-l:session): session closed for user vsc40035', 7 | '@source_host': 'node2131', 8 | } 9 | }, 10 | ] 11 | -------------------------------------------------------------------------------- /tests/data/zz_last: -------------------------------------------------------------------------------- 1 | data = [ 2 | 3 | 4 | 5 | ] -------------------------------------------------------------------------------- /tests/lmod.toml: -------------------------------------------------------------------------------- 1 | [[tests]] 2 | name = "Test lmod grok patterns" 3 | 4 | [[tests.inputs]] 5 | insert_at = "syslog" 6 | type = "log" 7 | 8 | [tests.inputs.log_fields] 9 | message = "2023-01-01T20:22:03.886018+01:00 node2630 lmod: username=vsc43614, cluster=swalot, jobid=35121297[22], userload=no, module=intel-compilers/2021.4.0, fn=/apps/gent/RHEL8/haswell-ib/modules/all/intel-compilers/2021.4.0.lua" 10 | 11 | [[tests.outputs]] 12 | extract_from = "syslog" 13 | 14 | [[tests.outputs.conditions]] 15 | type = "vrl" 16 | source = ''' 17 | assert!(is_string(.message)) 18 | assert_eq!(.lmod.username, "vsc43614") 19 | assert!(is_string(.lmod.module.filename)) 20 | assert_eq!(.lmod.module.name, "intel-compilers") 21 | assert_eq!(.lmod.module.version, "2021.4.0") 22 | assert_eq!(.lmod.cluster, "swalot") 23 | assert_eq!(.lmod.jobid, "35121297[22]") 24 | ''' 25 | -------------------------------------------------------------------------------- /tests/snoopy.toml: -------------------------------------------------------------------------------- 1 | [[tests]] 2 | name = "Test snoopy grok pattern 1" 3 | 4 | [[tests.inputs]] 5 | insert_at = "syslog" 6 | type = "log" 7 | 8 | [tests.inputs.log_fields] 9 | message = "Apr 26 09:00:55 master2 snoopy[1301]: [uid:110 sid:8322 tty: cwd:/ filename:/bin/cut]: cut -d ; -f 4 " 10 | 11 | [[tests.outputs]] 12 | extract_from = "syslog" 13 | 14 | [[tests.outputs.conditions]] 15 | type = "vrl" 16 | source = ''' 17 | assert!(is_string(.message)) 18 | assert_eq!(.snoopy.uid, "110") 19 | assert_eq!(.snoopy.executable, "/bin/cut") 20 | assert_eq!(.snoopy.cwd, "/") 21 | assert_eq!(.snoopy.command, "cut -d ; -f 4") 22 | ''' 23 | 24 | -------------------------------------------------------------------------------- /tests/vector.toml: -------------------------------------------------------------------------------- 1 | data_dir = "/tmp/grok" 2 | 3 | [sources.stdin] 4 | type = "stdin" 5 | 6 | [sinks.console] 7 | type = "console" 8 | encoding.codec = "json" 9 | inputs = ["syslog"] 10 | 11 | [transforms.syslog] 12 | inputs = ["stdin"] 13 | type = "remap" 14 | source = """ 15 | . |= parse_syslog!(.message) 16 | structured = parse_groks( 17 | .message, 18 | [ "%{LMOD:lmod}", 19 | "%{SSH_MSG:ssh}", 20 | "%{MODULECMD_MSG:modulecmd}", 21 | "%{NFS_MSG:nfs}", 22 | "%{OPENNEBULA_MSG:one}", 23 | "%{QUATTOR_MSG:quattor}", 24 | "%{SNOOPY_MSG:snoopy}" 25 | ], 26 | alias_sources: [ 27 | "/tmp/grok/gpfs.json", 28 | "/tmp/grok/icinga.json", 29 | "/tmp/grok/lmod.json", 30 | "/tmp/grok/modulecmd.json", 31 | "/tmp/grok/nfs.json", 32 | "/tmp/grok/opennebula.json", 33 | "/tmp/grok/quattor.json", 34 | "/tmp/grok/shorewall.json", 35 | "/tmp/grok/snoopy.json", 36 | "/tmp/grok/ssh.json", 37 | "/tmp/grok/hpc-ugent-patterns.json", 38 | ] 39 | ) ?? { 40 | "message": .message, 41 | "grokparsefailure": true 42 | } 43 | . = merge(., structured) 44 | """ 45 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # tox.ini: configuration file for tox 2 | # This file was automatically generated using 'python -m vsc.install.ci' 3 | # DO NOT EDIT MANUALLY 4 | 5 | [tox] 6 | envlist = py36,py39 7 | skipsdist = true 8 | 9 | [testenv:py36] 10 | commands_pre = 11 | pip install 'setuptools<42.0' 12 | python -m easy_install -U vsc-install 13 | python -m easy_install -U vsc-base 14 | python -m easy_install -U vsc-install 15 | 16 | [testenv:py39] 17 | ignore_outcome = true 18 | setenv = SETUPTOOLS_USE_DISTUTILS=local 19 | commands_pre = 20 | pip install 'setuptools<54.0' wheel 21 | python -c "from setuptools import setup;setup(script_args=['-q', 'easy_install', '-v', '-U', 'vsc-install'])" 22 | python -c "from setuptools import setup;setup(script_args=['-q', 'easy_install', '-v', '-U', 'vsc-base'])" 23 | python -c "from setuptools import setup;setup(script_args=['-q', 'easy_install', '-v', '-U', 'vsc-utils'])" 24 | 25 | [testenv] 26 | commands = python runtest.py 27 | passenv = USER 28 | -------------------------------------------------------------------------------- /vsc-ci.ini: -------------------------------------------------------------------------------- 1 | [vsc-ci] 2 | pip3_install_tox=1 3 | py3_tests_must_pass=1 4 | py3_only=1 5 | --------------------------------------------------------------------------------