├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTORS ├── Gemfile ├── LICENSE ├── NOTICE.TXT ├── README.md ├── Rakefile ├── docs └── index.asciidoc ├── lib └── logstash │ └── filters │ └── mutate.rb ├── logstash-filter-mutate.gemspec └── spec └── filters ├── integration └── multi_stage_spec.rb └── mutate_spec.rb /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Logstash 2 | 3 | All contributions are welcome: ideas, patches, documentation, bug reports, 4 | complaints, etc! 5 | 6 | Programming is not a required skill, and there are many ways to help out! 7 | It is more important to us that you are able to contribute. 8 | 9 | That said, some basic guidelines, which you are free to ignore :) 10 | 11 | ## Want to learn? 12 | 13 | Want to lurk about and see what others are doing with Logstash? 14 | 15 | * The irc channel (#logstash on irc.freenode.org) is a good place for this 16 | * The [forum](https://discuss.elastic.co/c/logstash) is also 17 | great for learning from others. 18 | 19 | ## Got Questions? 20 | 21 | Have a problem you want Logstash to solve for you? 22 | 23 | * You can ask a question in the [forum](https://discuss.elastic.co/c/logstash) 24 | * Alternately, you are welcome to join the IRC channel #logstash on 25 | irc.freenode.org and ask for help there! 26 | 27 | ## Have an Idea or Feature Request? 28 | 29 | * File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask. 30 | 31 | ## Something Not Working? Found a Bug? 32 | 33 | If you think you found a bug, it probably is a bug. 34 | 35 | * If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues) 36 | * If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins) 37 | * or ask the [forum](https://discuss.elastic.co/c/logstash). 38 | 39 | # Contributing Documentation and Code Changes 40 | 41 | If you have a bugfix or new feature that you would like to contribute to 42 | logstash, and you think it will take more than a few minutes to produce the fix 43 | (ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc) 44 | Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help. 45 | 46 | ## Contributing to plugins 47 | 48 | Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy! 49 | 50 | ## Contribution Steps 51 | 52 | 1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite 53 | 2. Please make sure you have signed our [Contributor License 54 | Agreement](https://www.elastic.co/contributor-agreement/). We are not 55 | asking you to assign copyright to us, but to give us the right to distribute 56 | your code without restriction. We ask this of all contributors in order to 57 | assure our users of the origin and continuing existence of the code. You 58 | only need to sign the CLA once. 59 | 3. Send a pull request! Push your changes to your fork of the repository and 60 | [submit a pull 61 | request](https://help.github.com/articles/using-pull-requests). In the pull 62 | request, describe what your changes do and mention any bugs/issues related 63 | to the pull request. 64 | 65 | 66 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here. 2 | 3 | For all general issues, please provide the following details for fast resolution: 4 | 5 | - Version: 6 | - Operating System: 7 | - Config File (if you have sensitive info, please remove it): 8 | - Sample Data: 9 | - Steps to Reproduce: 10 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/ 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | Gemfile.lock 3 | .bundle 4 | vendor 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | import: 2 | - logstash-plugins/.ci:travis/travis.yml@1.x -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 3.5.8 2 | - Fix "Can't modify frozen string" error when converting boolean to `string` [#171](https://github.com/logstash-plugins/logstash-filter-mutate/pull/171) 3 | 4 | ## 3.5.7 5 | - Clarify that `split` and `join` also support strings [#164](https://github.com/logstash-plugins/logstash-filter-mutate/pull/164) 6 | 7 | ## 3.5.6 8 | - [DOC] Added info on maintaining precision between Ruby float and Elasticsearch float [#158](https://github.com/logstash-plugins/logstash-filter-mutate/pull/158) 9 | 10 | ## 3.5.5 11 | - Fix: removed code and documentation for already removed 'remove' option. [#161](https://github.com/logstash-plugins/logstash-filter-mutate/pull/161) 12 | 13 | ## 3.5.4 14 | - [DOC] In 'replace' documentation, mention 'add' behavior [#155](https://github.com/logstash-plugins/logstash-filter-mutate/pull/155) 15 | - [DOC] Note that each mutate must be in its own code block as noted in issue [#27](https://github.com/logstash-plugins/logstash-filter-mutate/issues/27). Doc fix [#101](https://github.com/logstash-plugins/logstash-filter-mutate/pull/101) 16 | 17 | ## 3.5.3 18 | - [DOC] Expand description and behaviors for `rename` option [#156](https://github.com/logstash-plugins/logstash-filter-mutate/pull/156) 19 | 20 | ## 3.5.2 21 | - Fix: ensure that when an error occurs during registration, we use the correct i18n key to propagate the error message in a useful manner [#154](https://github.com/logstash-plugins/logstash-filter-mutate/pull/154) 22 | 23 | ## 3.5.1 24 | - Fix: removed a minor optimization in case-conversion helpers that could result in a race condition in very rare and specific situations [#151](https://github.com/logstash-plugins/logstash-filter-mutate/pull/151) 25 | 26 | ## 3.5.0 27 | - Fix: eliminated possible pipeline crashes; when a failure occurs during the application of this mutate filter, the rest of 28 | the operations are now aborted and a configurable tag is added to the event [#136](https://github.com/logstash-plugins/logstash-filter-mutate/pull/136) 29 | 30 | ## 3.4.0 31 | - Added ability to directly convert from integer and float to boolean [#127](https://github.com/logstash-plugins/logstash-filter-mutate/pull/127) 32 | 33 | ## 3.3.4 34 | - [DOC] Changed documentation to clarify execution order and to provide workaround 35 | [#128](https://github.com/logstash-plugins/logstash-filter-mutate/pull/128) 36 | 37 | ## 3.3.3 38 | - Changed documentation to clarify use of `replace` config option [#125](https://github.com/logstash-plugins/logstash-filter-mutate/pull/125) 39 | 40 | ## 3.3.2 41 | - Fix: when converting to `float` and `float_eu`, explicitly support same range of inputs as their integer counterparts; eliminates a regression introduced in 3.3.1 in which support for non-string inputs was inadvertently removed. 42 | 43 | ## 3.3.1 44 | - Fix: Number strings using a **decimal comma** (e.g. 1,23), added convert support to specify integer_eu and float_eu. 45 | 46 | ## 3.3.0 47 | - feature: Added capitalize feature. 48 | 49 | ## 3.2.0 50 | - Support boolean to integer conversion #107 51 | 52 | ## 3.1.7 53 | - Update gemspec summary 54 | 55 | ## 3.1.6 56 | - Fix some documentation issues 57 | 58 | ## 3.1.4 59 | - feature: Allow to copy fields. 60 | 61 | ## 3.1.3 62 | - Don't create empty fields when lower/uppercasing a non-existant field 63 | 64 | ## 3.1.2 65 | - bugfix: split method was not working, #78 66 | 67 | ## 3.1.1 68 | - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 69 | 70 | ## 3.1.0 71 | - breaking,config: Remove deprecated config `remove`. Please use generic `remove_field` instead. 72 | 73 | ## 3.0.1 74 | - internal: Republish all the gems under jruby. 75 | 76 | ## 3.0.0 77 | - internal,deps: Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 78 | 79 | ## 2.0.6 80 | - internal,test: Temp fix for patterns path in tests 81 | 82 | ## 2.0.5 83 | - internal,deps: Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash 84 | 85 | ## 2.0.4 86 | - internal,deps: New dependency requirements for logstash-core for the 5.0 release 87 | 88 | ## 2.0.3 89 | - internal,cleanup: Code cleanups and fix field assignments 90 | 91 | ## 2.0.0 92 | - internal: Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 93 | instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 94 | - internal,deps: Dependency on logstash-core update to 2.0 95 | 96 | ## 1.0.2 97 | - bugfix: Fix for uppercase and lowercase fail when value is already desired case 98 | - internal,test: Modify tests to prove bug and verify fix. 99 | 100 | ## 1.0.1 101 | - bugfix: Fix for uppercase and lowercase malfunction 102 | - internal,test: Specific test to prove bug and fix. 103 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Andrew Rowson (growse) 6 | * Avishai Ish-Shalom (avishai-ish-shalom) 7 | * Christian S. (squiddle) 8 | * Colin Surprenant (colinsurprenant) 9 | * Danny Berger (dpb587) 10 | * Ehtesh Choudhury (shurane) 11 | * Guillaume ZITTA (gza) 12 | * John E. Vincent (lusis) 13 | * Jonathan Van Eenwyk (jdve) 14 | * Jordan Sissel (jordansissel) 15 | * Kesavan Rengarajan (cosmok) 16 | * Kurt Hurtado (kurtado) 17 | * Laust Rud Jacobsen (rud) 18 | * Nick Ethier (nickethier) 19 | * Pete Fritchman (fetep) 20 | * Philippe Weber (wiibaa) 21 | * Pier-Hugues Pellerin (ph) 22 | * Ralph Meijer (ralphm) 23 | * Richard Pijnenburg (electrical) 24 | * Suyog Rao (suyograo) 25 | * Tal Levy (talevy) 26 | * piavlo 27 | * Abdul Haseeb Hussain (AbdulHaseebHussain) 28 | 29 | Note: If you've sent us patches, bug reports, or otherwise contributed to 30 | Logstash, and you aren't on the list above and want to be, please let us know 31 | and we'll make sure you're here. Contributions from folks like you are what make 32 | open source awesome. 33 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" 6 | use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" 7 | 8 | if Dir.exist?(logstash_path) && use_logstash_source 9 | gem 'logstash-core', :path => "#{logstash_path}/logstash-core" 10 | gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" 11 | end 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2020 Elastic and contributors 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE.TXT: -------------------------------------------------------------------------------- 1 | Elasticsearch 2 | Copyright 2012-2015 Elasticsearch 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Plugin 2 | 3 | [![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-mutate.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-mutate) 4 | 5 | This is a plugin for [Logstash](https://github.com/elastic/logstash). 6 | 7 | It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. 8 | 9 | ## Documentation 10 | 11 | Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). 12 | 13 | - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive 14 | - For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide 15 | 16 | ## Need Help? 17 | 18 | Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum. 19 | 20 | ## Developing 21 | 22 | ### 1. Plugin Developement and Testing 23 | 24 | #### Code 25 | - To get started, you'll need JRuby with the Bundler gem installed. 26 | 27 | - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example). 28 | 29 | - Install dependencies 30 | ```sh 31 | bundle install 32 | ``` 33 | 34 | #### Test 35 | 36 | - Update your dependencies 37 | 38 | ```sh 39 | bundle install 40 | ``` 41 | 42 | - Run tests 43 | 44 | ```sh 45 | bundle exec rspec 46 | ``` 47 | 48 | ### 2. Running your unpublished Plugin in Logstash 49 | 50 | #### 2.1 Run in a local Logstash clone 51 | 52 | - Edit Logstash `Gemfile` and add the local plugin path, for example: 53 | ```ruby 54 | gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" 55 | ``` 56 | - Install plugin 57 | ```sh 58 | # Logstash 2.3 and higher 59 | bin/logstash-plugin install --no-verify 60 | 61 | # Prior to Logstash 2.3 62 | bin/plugin install --no-verify 63 | 64 | ``` 65 | - Run Logstash with your plugin 66 | ```sh 67 | bin/logstash -e 'filter {awesome {}}' 68 | ``` 69 | At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash. 70 | 71 | #### 2.2 Run in an installed Logstash 72 | 73 | You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using: 74 | 75 | - Build your plugin gem 76 | ```sh 77 | gem build logstash-filter-awesome.gemspec 78 | ``` 79 | - Install the plugin from the Logstash home 80 | ```sh 81 | # Logstash 2.3 and higher 82 | bin/logstash-plugin install --no-verify 83 | 84 | # Prior to Logstash 2.3 85 | bin/plugin install --no-verify 86 | 87 | ``` 88 | - Start Logstash and proceed to test the plugin 89 | 90 | ## Contributing 91 | 92 | All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. 93 | 94 | Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. 95 | 96 | It is more important to the community that you are able to contribute. 97 | 98 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | @files=[] 2 | 3 | task :default do 4 | system("rake -T") 5 | end 6 | 7 | require "logstash/devutils/rake" 8 | -------------------------------------------------------------------------------- /docs/index.asciidoc: -------------------------------------------------------------------------------- 1 | :plugin: mutate 2 | :type: filter 3 | 4 | /////////////////////////////////////////// 5 | START - GENERATED VARIABLES, DO NOT EDIT! 6 | /////////////////////////////////////////// 7 | :version: %VERSION% 8 | :release_date: %RELEASE_DATE% 9 | :changelog_url: %CHANGELOG_URL% 10 | :include_path: ../../../../logstash/docs/include 11 | /////////////////////////////////////////// 12 | END - GENERATED VARIABLES, DO NOT EDIT! 13 | /////////////////////////////////////////// 14 | 15 | [id="plugins-{type}s-{plugin}"] 16 | 17 | === Mutate filter plugin 18 | 19 | include::{include_path}/plugin_header.asciidoc[] 20 | 21 | ==== Description 22 | 23 | The mutate filter allows you to perform general mutations on fields. You 24 | can rename, replace, and modify fields in your events. 25 | 26 | [id="plugins-{type}s-{plugin}-proc_order"] 27 | ===== Processing order 28 | 29 | Mutations in a config file are executed in this order: 30 | 31 | * coerce 32 | * rename 33 | * update 34 | * replace 35 | * convert 36 | * gsub 37 | * uppercase 38 | * capitalize 39 | * lowercase 40 | * strip 41 | * split 42 | * join 43 | * merge 44 | * copy 45 | 46 | IMPORTANT: Each mutation must be in its own code block if the sequence of operations needs to be preserved. 47 | 48 | Example: 49 | [source,ruby] 50 | ----- 51 | filter { 52 | mutate { 53 | split => { "hostname" => "." } 54 | add_field => { "shortHostname" => "%{[hostname][0]}" } 55 | } 56 | 57 | mutate { 58 | rename => {"shortHostname" => "hostname"} 59 | } 60 | } 61 | ----- 62 | 63 | [id="plugins-{type}s-{plugin}-options"] 64 | ==== Mutate Filter Configuration Options 65 | 66 | This plugin supports the following configuration options plus the <> described later. 67 | 68 | [cols="<,<,<",options="header",] 69 | |======================================================================= 70 | |Setting |Input type|Required 71 | | <> |<>|No 72 | | <> |<>|No 73 | | <> |<>|No 74 | | <> |<>|No 75 | | <> |<>|No 76 | | <> |<>|No 77 | | <> |<>|No 78 | | <> |<>|No 79 | | <> |<>|No 80 | | <> |<>|No 81 | | <> |<>|No 82 | | <> |<>|No 83 | | <> |<>|No 84 | | <> |<>|No 85 | | <> |<>|No 86 | |======================================================================= 87 | 88 | Also see <> for a list of options supported by all 89 | filter plugins. 90 | 91 |   92 | 93 | [id="plugins-{type}s-{plugin}-convert"] 94 | ===== `convert` 95 | 96 | * Value type is <> 97 | * There is no default value for this setting. 98 | 99 | Convert a field's value to a different type, like turning a string to an 100 | integer. If the field value is an array, all members will be converted. 101 | If the field is a hash no action will be taken. 102 | 103 | .Conversion insights 104 | [NOTE] 105 | ================================================================================ 106 | The values are converted using Ruby semantics. 107 | Be aware that using `float` and `float_eu` converts the value to a double-precision 64-bit IEEE 754 floating point decimal number. 108 | In order to maintain precision due to the conversion, you should use a `double` in the Elasticsearch mappings. 109 | ================================================================================ 110 | 111 | Valid conversion targets, and their expected behaviour with different inputs are: 112 | 113 | * `integer`: 114 | - strings are parsed; comma-separators are supported (e.g., the string `"1,000"` produces an integer with value of one thousand); when strings have decimal parts, they are _truncated_. 115 | - floats and decimals are _truncated_ (e.g., `3.99` becomes `3`, `-2.7` becomes `-2`) 116 | - boolean true and boolean false are converted to `1` and `0` respectively 117 | * `integer_eu`: 118 | - same as `integer`, except string values support dot-separators and comma-decimals (e.g., `"1.000"` produces an integer with value of one thousand) 119 | * `float`: 120 | - integers are converted to floats 121 | - strings are parsed; comma-separators and dot-decimals are supported (e.g., `"1,000.5"` produces a float with value of one thousand and one half) 122 | - boolean true and boolean false are converted to `1.0` and `0.0` respectively 123 | * `float_eu`: 124 | - same as `float`, except string values support dot-separators and comma-decimals (e.g., `"1.000,5"` produces a float with value of one thousand and one half) 125 | * `string`: 126 | - all values are stringified and encoded with UTF-8 127 | * `boolean`: 128 | - integer 0 is converted to boolean `false` 129 | - integer 1 is converted to boolean `true` 130 | - float 0.0 is converted to boolean `false` 131 | - float 1.0 is converted to boolean `true` 132 | - strings `"true"`, `"t"`, `"yes"`, `"y"`, `"1"`and `"1.0"` are converted to boolean `true` 133 | - strings `"false"`, `"f"`, `"no"`, `"n"`, `"0"` and `"0.0"` are converted to boolean `false` 134 | - empty strings are converted to boolean `false` 135 | - all other values pass straight through without conversion and log a warning message 136 | - for arrays each value gets processed separately using rules above 137 | 138 | This plugin can convert multiple fields in the same document, see the example below. 139 | 140 | Example: 141 | [source,ruby] 142 | filter { 143 | mutate { 144 | convert => { 145 | "fieldname" => "integer" 146 | "booleanfield" => "boolean" 147 | } 148 | } 149 | } 150 | 151 | [id="plugins-{type}s-{plugin}-copy"] 152 | ===== `copy` 153 | 154 | * Value type is <> 155 | * There is no default value for this setting. 156 | 157 | Copy an existing field to another field. Existing target field will be overriden. 158 | 159 | Example: 160 | [source,ruby] 161 | filter { 162 | mutate { 163 | copy => { "source_field" => "dest_field" } 164 | } 165 | } 166 | 167 | [id="plugins-{type}s-{plugin}-gsub"] 168 | ===== `gsub` 169 | 170 | * Value type is <> 171 | * There is no default value for this setting. 172 | 173 | Match a regular expression against a field value and replace all matches 174 | with a replacement string. Only fields that are strings or arrays of 175 | strings are supported. For other kinds of fields no action will be taken. 176 | 177 | This configuration takes an array consisting of 3 elements per 178 | field/substitution. 179 | 180 | Be aware of escaping any backslash in the config file. 181 | 182 | Example: 183 | [source,ruby] 184 | filter { 185 | mutate { 186 | gsub => [ 187 | # replace all forward slashes with underscore 188 | "fieldname", "/", "_", 189 | # replace backslashes, question marks, hashes, and minuses 190 | # with a dot "." 191 | "fieldname2", "[\\?#-]", "." 192 | ] 193 | } 194 | } 195 | 196 | 197 | [id="plugins-{type}s-{plugin}-join"] 198 | ===== `join` 199 | 200 | * Value type is <> 201 | * There is no default value for this setting. 202 | 203 | Join an array with a separator character or string. 204 | Does nothing on non-array fields. 205 | 206 | Example: 207 | [source,ruby] 208 | filter { 209 | mutate { 210 | join => { "fieldname" => "," } 211 | } 212 | } 213 | 214 | [id="plugins-{type}s-{plugin}-lowercase"] 215 | ===== `lowercase` 216 | 217 | * Value type is <> 218 | * There is no default value for this setting. 219 | 220 | Convert a string to its lowercase equivalent. 221 | 222 | Example: 223 | [source,ruby] 224 | filter { 225 | mutate { 226 | lowercase => [ "fieldname" ] 227 | } 228 | } 229 | 230 | [id="plugins-{type}s-{plugin}-merge"] 231 | ===== `merge` 232 | 233 | * Value type is <> 234 | * There is no default value for this setting. 235 | 236 | Merge two fields of arrays or hashes. 237 | String fields will be automatically be converted into an array, so: 238 | ========================== 239 | `array` + `string` will work 240 | `string` + `string` will result in an 2 entry array in `dest_field` 241 | `array` and `hash` will not work 242 | ========================== 243 | Example: 244 | [source,ruby] 245 | filter { 246 | mutate { 247 | merge => { "dest_field" => "added_field" } 248 | } 249 | } 250 | 251 | [id="plugins-{type}s-{plugin}-coerce"] 252 | ===== `coerce` 253 | 254 | * Value type is <> 255 | * There is no default value for this setting. 256 | 257 | Set the default value of a field that exists but is null 258 | 259 | Example: 260 | [source,ruby] 261 | filter { 262 | mutate { 263 | # Sets the default value of the 'field1' field to 'default_value' 264 | coerce => { "field1" => "default_value" } 265 | } 266 | } 267 | 268 | [id="plugins-{type}s-{plugin}-rename"] 269 | ===== `rename` 270 | 271 | * Value type is <> 272 | * There is no default value for this setting. 273 | 274 | Rename one or more fields. 275 | 276 | If the destination field already exists, its value is replaced. 277 | 278 | If one of the source fields doesn't exist, no action is performed for that field. 279 | (This is not considered an error; the `tag_on_failure` tag is not applied.) 280 | 281 | When renaming multiple fields, the order of operations is not guaranteed. 282 | 283 | Example: 284 | [source,ruby] 285 | filter { 286 | mutate { 287 | # Renames the 'HOSTORIP' field to 'client_ip' 288 | rename => { "HOSTORIP" => "client_ip" } 289 | } 290 | } 291 | 292 | [id="plugins-{type}s-{plugin}-replace"] 293 | ===== `replace` 294 | 295 | * Value type is <> 296 | * There is no default value for this setting. 297 | 298 | Replace the value of a field with a new value, or add the field if it 299 | doesn't already exist. The new value can include `%{foo}` strings 300 | to help you build a new value from other parts of the event. 301 | 302 | Example: 303 | [source,ruby] 304 | filter { 305 | mutate { 306 | replace => { "message" => "%{source_host}: My new message" } 307 | } 308 | } 309 | 310 | [id="plugins-{type}s-{plugin}-split"] 311 | ===== `split` 312 | 313 | * Value type is <> 314 | * There is no default value for this setting. 315 | 316 | Split a field to an array using a separator character or string. 317 | Only works on string fields. 318 | 319 | Example: 320 | [source,ruby] 321 | filter { 322 | mutate { 323 | split => { "fieldname" => "," } 324 | } 325 | } 326 | 327 | [id="plugins-{type}s-{plugin}-strip"] 328 | ===== `strip` 329 | 330 | * Value type is <> 331 | * There is no default value for this setting. 332 | 333 | Strip whitespace from field. NOTE: this only works on leading and trailing whitespace. 334 | 335 | Example: 336 | [source,ruby] 337 | filter { 338 | mutate { 339 | strip => ["field1", "field2"] 340 | } 341 | } 342 | 343 | [id="plugins-{type}s-{plugin}-update"] 344 | ===== `update` 345 | 346 | * Value type is <> 347 | * There is no default value for this setting. 348 | 349 | Update an existing field with a new value. If the field does not exist, 350 | then no action will be taken. 351 | 352 | Example: 353 | [source,ruby] 354 | filter { 355 | mutate { 356 | update => { "sample" => "My new message" } 357 | } 358 | } 359 | 360 | [id="plugins-{type}s-{plugin}-uppercase"] 361 | ===== `uppercase` 362 | 363 | * Value type is <> 364 | * There is no default value for this setting. 365 | 366 | Convert a string to its uppercase equivalent. 367 | 368 | Example: 369 | [source,ruby] 370 | filter { 371 | mutate { 372 | uppercase => [ "fieldname" ] 373 | } 374 | } 375 | 376 | [id="plugins-{type}s-{plugin}-capitalize"] 377 | ===== `capitalize` 378 | 379 | * Value type is <> 380 | * There is no default value for this setting. 381 | 382 | Convert a string to its capitalized equivalent. 383 | 384 | Example: 385 | [source,ruby] 386 | filter { 387 | mutate { 388 | capitalize => [ "fieldname" ] 389 | } 390 | } 391 | 392 | [id="plugins-{type}s-{plugin}-tag_on_failure"] 393 | ===== `tag_on_failure` 394 | 395 | * Value type is <> 396 | * The default value for this setting is `_mutate_error` 397 | 398 | If a failure occurs during the application of this mutate filter, the rest of 399 | the operations are aborted and the provided tag is added to the event. 400 | 401 | [id="plugins-{type}s-{plugin}-common-options"] 402 | include::{include_path}/{type}.asciidoc[] 403 | -------------------------------------------------------------------------------- /lib/logstash/filters/mutate.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/filters/base" 3 | require "logstash/namespace" 4 | 5 | # The mutate filter allows you to perform general mutations on fields. You 6 | # can rename, replace, and modify fields in your events. 7 | class LogStash::Filters::Mutate < LogStash::Filters::Base 8 | config_name "mutate" 9 | 10 | # Sets a default value when the field exists but the value is null. 11 | # 12 | # Example: 13 | # [source,ruby] 14 | # filter { 15 | # mutate { 16 | # # Sets the default value of the 'field1' field to 'default_value' 17 | # coerce => { "field1" => "default_value" } 18 | # } 19 | # } 20 | config :coerce, :validate => :hash 21 | 22 | # Rename one or more fields. 23 | # 24 | # Example: 25 | # [source,ruby] 26 | # filter { 27 | # mutate { 28 | # # Renames the 'HOSTORIP' field to 'client_ip' 29 | # rename => { "HOSTORIP" => "client_ip" } 30 | # } 31 | # } 32 | config :rename, :validate => :hash 33 | 34 | # Replace a field with a new value. The new value can include `%{foo}` strings 35 | # to help you build a new value from other parts of the event. 36 | # 37 | # Example: 38 | # [source,ruby] 39 | # filter { 40 | # mutate { 41 | # replace => { "message" => "%{source_host}: My new message" } 42 | # } 43 | # } 44 | config :replace, :validate => :hash 45 | 46 | # Update an existing field with a new value. If the field does not exist, 47 | # then no action will be taken. 48 | # 49 | # Example: 50 | # [source,ruby] 51 | # filter { 52 | # mutate { 53 | # update => { "sample" => "My new message" } 54 | # } 55 | # } 56 | config :update, :validate => :hash 57 | 58 | # Convert a field's value to a different type, like turning a string to an 59 | # integer. If the field value is an array, all members will be converted. 60 | # If the field is a hash no action will be taken. 61 | # 62 | # If the conversion type is `boolean`, the acceptable values are: 63 | # 64 | # * **True:** `true`, `t`, `yes`, `y`, and `1` 65 | # * **False:** `false`, `f`, `no`, `n`, and `0` 66 | # 67 | # If a value other than these is provided, it will pass straight through 68 | # and log a warning message. 69 | # 70 | # If the conversion type is `integer` and the value is a boolean, it will be converted as: 71 | # * **True:** `1` 72 | # * **False:** `0` 73 | # 74 | # If you have numeric strings that have decimal commas (Europe and ex-colonies) 75 | # e.g. "1.234,56" or "2.340", by using conversion targets of integer_eu or float_eu 76 | # the convert function will treat "." as a group separator and "," as a decimal separator. 77 | # 78 | # Conversion targets of integer or float will now correctly handle "," as a group separator. 79 | # 80 | # Valid conversion targets are: integer, float, integer_eu, float_eu, string, and boolean. 81 | # 82 | # Example: 83 | # [source,ruby] 84 | # filter { 85 | # mutate { 86 | # convert => { "fieldname" => "integer" } 87 | # } 88 | # } 89 | config :convert, :validate => :hash 90 | 91 | # Match a regular expression against a field value and replace all matches 92 | # with another string. Only fields that are strings or arrays of strings are 93 | # supported. For other kinds of fields no action will be taken. 94 | # 95 | # This configuration takes an array consisting of 3 elements per 96 | # field/substitution. 97 | # 98 | # Be aware of escaping any backslash in the config file. 99 | # 100 | # Example: 101 | # [source,ruby] 102 | # filter { 103 | # mutate { 104 | # gsub => [ 105 | # # replace all forward slashes with underscore 106 | # "fieldname", "/", "_", 107 | # # replace backslashes, question marks, hashes, and minuses 108 | # # with a dot "." 109 | # "fieldname2", "[\\?#-]", "." 110 | # ] 111 | # } 112 | # } 113 | # 114 | config :gsub, :validate => :array 115 | 116 | # Convert a string to its uppercase equivalent. 117 | # 118 | # Example: 119 | # [source,ruby] 120 | # filter { 121 | # mutate { 122 | # uppercase => [ "fieldname" ] 123 | # } 124 | # } 125 | config :uppercase, :validate => :array 126 | 127 | # Convert a string to its lowercase equivalent. 128 | # 129 | # Example: 130 | # [source,ruby] 131 | # filter { 132 | # mutate { 133 | # lowercase => [ "fieldname" ] 134 | # } 135 | # } 136 | config :lowercase, :validate => :array 137 | 138 | # Convert a string to its capitalized equivalent. 139 | # 140 | # Example: 141 | # [source,ruby] 142 | # filter { 143 | # mutate { 144 | # capitalize => [ "fieldname" ] 145 | # } 146 | # } 147 | config :capitalize, :validate => :array 148 | 149 | # Split a field to an array using a separator character. Only works on string 150 | # fields. 151 | # 152 | # Example: 153 | # [source,ruby] 154 | # filter { 155 | # mutate { 156 | # split => { "fieldname" => "," } 157 | # } 158 | # } 159 | config :split, :validate => :hash 160 | 161 | # Join an array with a separator character. Does nothing on non-array fields. 162 | # 163 | # Example: 164 | # [source,ruby] 165 | # filter { 166 | # mutate { 167 | # join => { "fieldname" => "," } 168 | # } 169 | # } 170 | config :join, :validate => :hash 171 | 172 | # Strip whitespace from field. NOTE: this only works on leading and trailing whitespace. 173 | # 174 | # Example: 175 | # [source,ruby] 176 | # filter { 177 | # mutate { 178 | # strip => ["field1", "field2"] 179 | # } 180 | # } 181 | config :strip, :validate => :array 182 | 183 | # Merge two fields of arrays or hashes. 184 | # String fields will be automatically be converted into an array, so: 185 | # ========================== 186 | # `array` + `string` will work 187 | # `string` + `string` will result in an 2 entry array in `dest_field` 188 | # `array` and `hash` will not work 189 | # ========================== 190 | # Example: 191 | # [source,ruby] 192 | # filter { 193 | # mutate { 194 | # merge => { "dest_field" => "added_field" } 195 | # } 196 | # } 197 | config :merge, :validate => :hash 198 | 199 | # Copy an existing field to another field. Existing target field will be overriden. 200 | # ========================== 201 | # Example: 202 | # [source,ruby] 203 | # filter { 204 | # mutate { 205 | # copy => { "source_field" => "dest_field" } 206 | # } 207 | # } 208 | config :copy, :validate => :hash 209 | 210 | # Tag to apply if the operation errors 211 | config :tag_on_failure, :validate => :string, :default => '_mutate_error' 212 | 213 | TRUE_REGEX = (/^(true|t|yes|y|1|1.0)$/i).freeze 214 | FALSE_REGEX = (/^(false|f|no|n|0|0.0)$/i).freeze 215 | CONVERT_PREFIX = "convert_".freeze 216 | 217 | def register 218 | valid_conversions = %w(string integer float boolean integer_eu float_eu ) 219 | # TODO(sissel): Validate conversion requests if provided. 220 | @convert.nil? or @convert.each do |field, type| 221 | if !valid_conversions.include?(type) 222 | raise LogStash::ConfigurationError, I18n.t( 223 | "logstash.runner.configuration.invalid_plugin_register", 224 | :plugin => "filter", 225 | :type => "mutate", 226 | :error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'" 227 | ) 228 | end 229 | end 230 | 231 | @gsub_parsed = [] 232 | @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement| 233 | if [field, needle, replacement].any? {|n| n.nil?} 234 | raise LogStash::ConfigurationError, I18n.t( 235 | "logstash.runner.configuration.invalid_plugin_register", 236 | :plugin => "filter", 237 | :type => "mutate", 238 | :error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry" 239 | ) 240 | end 241 | 242 | @gsub_parsed << { 243 | :field => field, 244 | :needle => (needle.index("%{").nil?? Regexp.new(needle): needle), 245 | :replacement => replacement 246 | } 247 | end 248 | end 249 | 250 | def filter(event) 251 | coerce(event) if @coerce 252 | rename(event) if @rename 253 | update(event) if @update 254 | replace(event) if @replace 255 | convert(event) if @convert 256 | gsub(event) if @gsub 257 | uppercase(event) if @uppercase 258 | capitalize(event) if @capitalize 259 | lowercase(event) if @lowercase 260 | strip(event) if @strip 261 | split(event) if @split 262 | join(event) if @join 263 | merge(event) if @merge 264 | copy(event) if @copy 265 | 266 | filter_matched(event) 267 | rescue => ex 268 | meta = { :exception => ex.message } 269 | meta[:backtrace] = ex.backtrace if logger.debug? 270 | logger.warn('Exception caught while applying mutate filter', meta) 271 | event.tag(@tag_on_failure) 272 | end 273 | 274 | private 275 | 276 | def coerce(event) 277 | @coerce.each do |field, default_value| 278 | next unless event.include?(field) && event.get(field)==nil 279 | event.set(field, event.sprintf(default_value)) 280 | end 281 | end 282 | 283 | def rename(event) 284 | @rename.each do |old, new| 285 | old = event.sprintf(old) 286 | new = event.sprintf(new) 287 | next unless event.include?(old) 288 | event.set(new, event.remove(old)) 289 | end 290 | end 291 | 292 | def update(event) 293 | @update.each do |field, newvalue| 294 | next unless event.include?(field) 295 | event.set(field, event.sprintf(newvalue)) 296 | end 297 | end 298 | 299 | def replace(event) 300 | @replace.each do |field, newvalue| 301 | event.set(field, event.sprintf(newvalue)) 302 | end 303 | end 304 | 305 | def convert(event) 306 | @convert.each do |field, type| 307 | next unless event.include?(field) 308 | original = event.get(field) 309 | # calls convert_{string,integer,float,boolean} depending on type requested. 310 | converter = method(CONVERT_PREFIX + type) 311 | 312 | case original 313 | when Hash 314 | @logger.debug? && @logger.debug("I don't know how to type convert a hash, skipping", :field => field, :value => original) 315 | when Array 316 | event.set(field, original.map { |v| v.nil? ? v : converter.call(v) }) 317 | when NilClass 318 | # ignore 319 | else 320 | event.set(field, converter.call(original)) 321 | end 322 | end 323 | end 324 | 325 | def convert_string(value) 326 | # since this is a filter and all inputs should be already UTF-8 327 | # we wont check valid_encoding? but just force UTF-8 for 328 | # the Fixnum#to_s case which always result in US-ASCII 329 | # also not that force_encoding checks current encoding against the 330 | # target encoding and only change if necessary, so calling 331 | # valid_encoding? is redundant 332 | # see https://twitter.com/jordansissel/status/444613207143903232 333 | # use + since .to_s on nil/boolean returns a frozen string since ruby 2.7 334 | (+value.to_s).force_encoding(Encoding::UTF_8) 335 | end 336 | 337 | def convert_boolean(value) 338 | return true if value.to_s =~ TRUE_REGEX 339 | return false if value.to_s.empty? || value.to_s =~ FALSE_REGEX 340 | @logger.warn("Failed to convert #{value} into boolean.") 341 | value 342 | end 343 | 344 | def convert_integer(value) 345 | return 1 if value == true 346 | return 0 if value == false 347 | return value.to_i if !value.is_a?(String) 348 | value.tr(",", "").to_i 349 | end 350 | 351 | def convert_float(value) 352 | return 1.0 if value == true 353 | return 0.0 if value == false 354 | value = value.delete(",") if value.kind_of?(String) 355 | value.to_f 356 | end 357 | 358 | def convert_integer_eu(value) 359 | us_value = cnv_replace_eu(value) 360 | convert_integer(us_value) 361 | end 362 | 363 | def convert_float_eu(value) 364 | us_value = cnv_replace_eu(value) 365 | convert_float(us_value) 366 | end 367 | 368 | # When given a String, returns a new String whose contents have been converted from 369 | # EU-style comma-decimals and dot-separators to US-style dot-decimals and comma-separators. 370 | # 371 | # For all other values, returns value unmodified. 372 | def cnv_replace_eu(value) 373 | return value if !value.is_a?(String) 374 | value.tr(",.", ".,") 375 | end 376 | 377 | def gsub(event) 378 | @gsub_parsed.each do |config| 379 | field = config[:field] 380 | needle = config[:needle] 381 | replacement = config[:replacement] 382 | 383 | value = event.get(field) 384 | case value 385 | when Array 386 | result = value.map do |v| 387 | if v.is_a?(String) 388 | gsub_dynamic_fields(event, v, needle, replacement) 389 | else 390 | @logger.warn("gsub mutation is only applicable for strings and arrays of strings, skipping", :field => field, :value => v) 391 | v 392 | end 393 | end 394 | event.set(field, result) 395 | when String 396 | event.set(field, gsub_dynamic_fields(event, value, needle, replacement)) 397 | else 398 | @logger.debug? && @logger.debug("gsub mutation is only applicable for strings and arrays of strings, skipping", :field => field, :value => event.get(field)) 399 | end 400 | end 401 | end 402 | 403 | def gsub_dynamic_fields(event, original, needle, replacement) 404 | if needle.is_a?(Regexp) 405 | original.gsub(needle, event.sprintf(replacement)) 406 | else 407 | # we need to replace any dynamic fields 408 | original.gsub(Regexp.new(event.sprintf(needle)), event.sprintf(replacement)) 409 | end 410 | end 411 | 412 | def uppercase(event) 413 | @uppercase.each do |field| 414 | original = event.get(field) 415 | next if original.nil? 416 | # in certain cases JRuby returns a proxy wrapper of the event[field] value 417 | # therefore we can't assume that we are modifying the actual value behind 418 | # the key so read, modify and overwrite 419 | result = case original 420 | when Array 421 | # can't map upcase! as it replaces an already upcase value with nil 422 | # ["ABCDEF"].map(&:upcase!) => [nil] 423 | original.map do |elem| 424 | (elem.is_a?(String) ? elem.upcase : elem) 425 | end 426 | when String 427 | original.upcase 428 | else 429 | @logger.debug? && @logger.debug("Can't uppercase something that isn't a string", :field => field, :value => original) 430 | original 431 | end 432 | event.set(field, result) 433 | end 434 | end 435 | 436 | def lowercase(event) 437 | #see comments for #uppercase 438 | @lowercase.each do |field| 439 | original = event.get(field) 440 | next if original.nil? 441 | result = case original 442 | when Array 443 | original.map! do |elem| 444 | (elem.is_a?(String) ? elem.downcase : elem) 445 | end 446 | when String 447 | original.downcase 448 | else 449 | @logger.debug? && @logger.debug("Can't lowercase something that isn't a string", :field => field, :value => original) 450 | original 451 | end 452 | event.set(field, result) 453 | end 454 | end 455 | 456 | def capitalize(event) 457 | #see comments for #uppercase 458 | @capitalize.each do |field| 459 | original = event.get(field) 460 | next if original.nil? 461 | result = case original 462 | when Array 463 | original.map! do |elem| 464 | (elem.is_a?(String) ? elem.capitalize : elem) 465 | end 466 | when String 467 | original.capitalize 468 | else 469 | @logger.debug? && @logger.debug("Can't capitalize something that isn't a string", :field => field, :value => original) 470 | original 471 | end 472 | event.set(field, result) 473 | end 474 | end 475 | 476 | def split(event) 477 | @split.each do |field, separator| 478 | value = event.get(field) 479 | if value.is_a?(String) 480 | event.set(field, value.split(separator)) 481 | else 482 | @logger.debug? && @logger.debug("Can't split something that isn't a string", :field => field, :value => event.get(field)) 483 | end 484 | end 485 | end 486 | 487 | def join(event) 488 | @join.each do |field, separator| 489 | value = event.get(field) 490 | if value.is_a?(Array) 491 | event.set(field, value.join(separator)) 492 | end 493 | end 494 | end 495 | 496 | def strip(event) 497 | @strip.each do |field| 498 | value = event.get(field) 499 | case value 500 | when Array 501 | event.set(field, value.map{|s| s.strip }) 502 | when String 503 | event.set(field, value.strip) 504 | end 505 | end 506 | end 507 | 508 | def merge(event) 509 | @merge.each do |dest_field, added_fields| 510 | # When multiple calls, added_field is an array 511 | 512 | dest_field_value = event.get(dest_field) 513 | 514 | Array(added_fields).each do |added_field| 515 | added_field_value = event.get(added_field) 516 | 517 | if dest_field_value.is_a?(Hash) ^ added_field_value.is_a?(Hash) 518 | @logger.error("Not possible to merge an array and a hash: ", :dest_field => dest_field, :added_field => added_field ) 519 | next 520 | end 521 | 522 | # No need to test the other 523 | if dest_field_value.is_a?(Hash) 524 | # do not use event[dest_field].update because the returned object from event[dest_field] 525 | # can/will be a copy of the actual event data and directly updating it will not update 526 | # the Event internal data. The updated value must be reassigned in the Event. 527 | event.set(dest_field, dest_field_value.update(added_field_value)) 528 | else 529 | # do not use event[dest_field].concat because the returned object from event[dest_field] 530 | # can/will be a copy of the actual event data and directly updating it will not update 531 | # the Event internal data. The updated value must be reassigned in the Event. 532 | event.set(dest_field, Array(dest_field_value).concat(Array(added_field_value))) 533 | end 534 | end 535 | end 536 | end 537 | 538 | def copy(event) 539 | @copy.each do |src_field, dest_field| 540 | original = event.get(src_field) 541 | next if original.nil? 542 | event.set(dest_field,LogStash::Util.deep_clone(original)) 543 | end 544 | end 545 | end 546 | -------------------------------------------------------------------------------- /logstash-filter-mutate.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | 3 | s.name = 'logstash-filter-mutate' 4 | s.version = '3.5.8' 5 | s.licenses = ['Apache License (2.0)'] 6 | s.summary = "Performs mutations on fields" 7 | s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" 8 | s.authors = ["Elastic"] 9 | s.email = 'info@elastic.co' 10 | s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" 11 | s.require_paths = ["lib"] 12 | 13 | # Files 14 | s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] 15 | 16 | # Tests 17 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 18 | 19 | # Special flag to let us know this is actually a logstash plugin 20 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } 21 | 22 | # Gem dependencies 23 | s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" 24 | s.add_development_dependency "logstash-patterns-core" 25 | s.add_development_dependency "logstash-filter-grok" 26 | s.add_development_dependency "logstash-codec-plain" 27 | s.add_development_dependency "logstash-devutils" 28 | end 29 | -------------------------------------------------------------------------------- /spec/filters/integration/multi_stage_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/devutils/rspec/spec_helper" 3 | 4 | module LogStash::Environment 5 | # running mutate which depends on grok outside a logstash package means 6 | # LOGSTASH_HOME will not be defined, so let's set it here 7 | # before requiring the grok filter 8 | unless self.const_defined?(:LOGSTASH_HOME) 9 | LOGSTASH_HOME = File.expand_path("../../../", __FILE__) 10 | end 11 | 12 | # also :pattern_path method must exist (due grok filter) 13 | unless self.method_defined?(:pattern_path) 14 | def pattern_path(path) 15 | ::File.join(LOGSTASH_HOME, "patterns", path) 16 | end 17 | end 18 | end 19 | 20 | describe 'LogStash::Filters::Mutate' do 21 | 22 | context 'MUTATE-33: multi stage with json, grok and mutate, Case mutation' do 23 | let(:config) do 24 | <<-CONFIG 25 | filter { 26 | grok { 27 | match => { "message" => "(?:hello) %{WORD:bar}" } 28 | break_on_match => false 29 | } 30 | mutate { 31 | lowercase => [ "bar", "lower1", "lower2" ] 32 | } 33 | } 34 | CONFIG 35 | end 36 | 37 | sample({"message" => "hello WORLD", "lower1" => "PPQQRRSS", "lower2" => "pppqqq"}) do 38 | result = results.first 39 | expect(result.get("bar")).to eq('world') 40 | expect(result.get("lower1")).to eq("ppqqrrss") 41 | expect(result.get("lower2")).to eq("pppqqq") 42 | end 43 | 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/filters/mutate_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require "logstash/devutils/rspec/spec_helper" 4 | require "logstash/filters/mutate" 5 | 6 | # running mutate which depends on grok outside a logstash package means 7 | # LOGSTASH_HOME will not be defined, so let's set it here 8 | # before requiring the grok filter 9 | unless LogStash::Environment.const_defined?(:LOGSTASH_HOME) 10 | LogStash::Environment::LOGSTASH_HOME = File.expand_path("../../../", __FILE__) 11 | end 12 | 13 | # temporary fix to have the spec pass for an urgen mass-publish requirement. 14 | # cut & pasted from the same tmp fix in the grok spec 15 | # see https://github.com/logstash-plugins/logstash-filter-grok/issues/72 16 | # this needs to be refactored and properly fixed 17 | module LogStash::Environment 18 | # also :pattern_path method must exist so we define it too 19 | unless self.method_defined?(:pattern_path) 20 | def pattern_path(path) 21 | ::File.join(LOGSTASH_HOME, "patterns", path) 22 | end 23 | end 24 | end 25 | 26 | logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION) 27 | 28 | if (Gem::Requirement.create('~> 7.0').satisfied_by?(logstash_version) || 29 | (Gem::Requirement.create('~> 6.4').satisfied_by?(logstash_version) && LogStash::SETTINGS.get('config.field_reference.parser') == 'STRICT')) 30 | describe LogStash::Filters::Mutate do 31 | let(:config) { Hash.new } 32 | subject(:mutate_filter) { LogStash::Filters::Mutate.new(config) } 33 | 34 | before(:each) { mutate_filter.register } 35 | 36 | let(:event) { LogStash::Event.new(attrs) } 37 | 38 | context 'when operation would cause an error' do 39 | 40 | let(:invalid_field_name) { "[[][[[[]message" } 41 | let(:config) do 42 | super().merge("add_field" => {invalid_field_name => "nope"}) 43 | end 44 | 45 | shared_examples('catch and tag error') do 46 | let(:expected_tag) { '_mutate_error' } 47 | 48 | let(:event) { LogStash::Event.new({"message" => "foo"})} 49 | 50 | context 'when the event is filtered' do 51 | before(:each) { mutate_filter.filter(event) } 52 | it 'does not raise an exception' do 53 | # noop 54 | end 55 | 56 | it 'tags the event with the expected tag' do 57 | expect(event).to include('tags') 58 | expect(event.get('tags')).to include(expected_tag) 59 | end 60 | end 61 | end 62 | 63 | context 'when `tag_on_failure` is not provided' do 64 | include_examples 'catch and tag error' 65 | end 66 | 67 | context 'when `tag_on_failure` is provided' do 68 | include_examples 'catch and tag error' do 69 | let(:expected_tag) { 'my_custom_tag' } 70 | let(:config) { super().merge('tag_on_failure' => expected_tag) } 71 | end 72 | end 73 | end 74 | end 75 | end 76 | 77 | describe LogStash::Filters::Mutate do 78 | 79 | let(:config) { {} } 80 | subject { LogStash::Filters::Mutate.new(config) } 81 | 82 | let(:attrs) { { } } 83 | let(:event) { LogStash::Event.new(attrs) } 84 | 85 | before(:each) do 86 | subject.register 87 | end 88 | 89 | context "when doing uppercase of an array" do 90 | 91 | let(:config) do 92 | { "uppercase" => ["array_of"] } 93 | end 94 | 95 | let(:attrs) { { "array_of" => ["a", 2, "C"] } } 96 | 97 | it "should uppercase not raise an error" do 98 | expect { subject.filter(event) }.not_to raise_error 99 | end 100 | 101 | it "should convert only string elements" do 102 | subject.filter(event) 103 | expect(event.get("array_of")).to eq(["A", 2, "C"]) 104 | end 105 | end 106 | 107 | context "when doing capitalize of an array" do 108 | 109 | let(:config) do 110 | { "capitalize" => ["array_of"] } 111 | end 112 | 113 | let(:attrs) { { "array_of" => ["ab", 2, "CDE"] } } 114 | 115 | it "should capitalize not raise an error" do 116 | expect { subject.filter(event) }.not_to raise_error 117 | end 118 | 119 | it "should convert only string elements" do 120 | subject.filter(event) 121 | expect(event.get("array_of")).to eq(["Ab", 2, "Cde"]) 122 | end 123 | end 124 | 125 | context "when doing lowercase of an array" do 126 | 127 | let(:config) do 128 | { "lowercase" => ["array_of"] } 129 | end 130 | 131 | let(:attrs) { { "array_of" => ["a", 2, "C"] } } 132 | 133 | it "should lowercase all string elements" do 134 | expect { subject.filter(event) }.not_to raise_error 135 | end 136 | 137 | it "should convert only string elements" do 138 | subject.filter(event) 139 | expect(event.get("array_of")).to eq(["a", 2, "c"]) 140 | end 141 | end 142 | 143 | %w(lowercase uppercase capitalize).each do |operation| 144 | context "executing #{operation} a non-existant field" do 145 | let(:attrs) { } 146 | 147 | let(:config) do 148 | { operation => ["fake_field"] } 149 | end 150 | 151 | it "should not create that field" do 152 | subject.filter(event) 153 | expect(event).not_to include("fake_field") 154 | end 155 | end 156 | context "avoid mutating contents of field, as they may be shared" do 157 | let(:original_value) { "oRiGiNaL vAlUe".freeze } 158 | let(:shared_value) { original_value.dup } 159 | let(:attrs) { {"field" => shared_value } } 160 | let(:config) do 161 | { 162 | operation => "field" 163 | } 164 | end 165 | 166 | it 'should not mutate the value' do 167 | subject.filter(event) 168 | expect(shared_value).to eq(original_value) 169 | end 170 | end 171 | end 172 | end 173 | 174 | describe LogStash::Filters::Mutate do 175 | 176 | let(:config) { {} } 177 | subject { LogStash::Filters::Mutate.new(config) } 178 | 179 | let(:attrs) { { } } 180 | let(:event) { LogStash::Event.new(attrs) } 181 | 182 | before(:each) do 183 | subject.register 184 | end 185 | 186 | describe "#strip" do 187 | 188 | let(:config) do 189 | { "strip" => ["path"] } 190 | end 191 | 192 | let(:attrs) { { "path" => " /store.php " } } 193 | 194 | it "should cleam trailing spaces" do 195 | subject.filter(event) 196 | expect(event.get("path")).to eq("/store.php") 197 | end 198 | 199 | context "when converting multiple attributed at once" do 200 | 201 | let(:config) do 202 | { "strip" => ["foo", "bar"] } 203 | end 204 | 205 | let(:attrs) { { "foo" => " /bar.php ", "bar" => " foo" } } 206 | 207 | it "should cleam trailing spaces" do 208 | subject.filter(event) 209 | expect(event.get("foo")).to eq("/bar.php") 210 | expect(event.get("bar")).to eq("foo") 211 | end 212 | end 213 | end 214 | 215 | describe "#split" do 216 | 217 | let(:config) do 218 | { "split" => {"field" => "," } } 219 | end 220 | 221 | context "when source field is a string" do 222 | 223 | let(:attrs) { { "field" => "foo,bar,baz" } } 224 | 225 | it "should split string into array" do 226 | subject.filter(event) 227 | expect(event.get("field")).to eq(["foo","bar","baz"]) 228 | end 229 | 230 | it "should convert single field to array" do 231 | event.set("field","foo") 232 | subject.filter(event) 233 | expect(event.get("field")).to eq(["foo"]) 234 | 235 | event.set("field","foo,") 236 | subject.filter(event) 237 | expect(event.get("field")).to eq(["foo"]) 238 | end 239 | 240 | end 241 | 242 | context "when source field is not a string" do 243 | 244 | it "should not modify source field nil" do 245 | event.set("field",nil) 246 | subject.filter(event) 247 | expect(event.get("field")).to eq(nil) 248 | end 249 | 250 | it "should not modify source field array" do 251 | event.set("field",["foo","bar"]) 252 | subject.filter(event) 253 | expect(event.get("field")).to eq(["foo","bar"]) 254 | end 255 | 256 | it "should not modify source field hash" do 257 | event.set("field",{"foo" => "bar,baz"}) 258 | subject.filter(event) 259 | expect(event.get("field")).to eq({"foo" => "bar,baz"}) 260 | end 261 | end 262 | end 263 | 264 | describe "#copy" do 265 | 266 | let(:config) do 267 | { "copy" => {"field" => "target" } } 268 | end 269 | 270 | context "when source field is a string" do 271 | 272 | let(:attrs) { { "field" => "foobar" } } 273 | 274 | it "should deep copy the field" do 275 | subject.filter(event) 276 | expect(event.get("target")).to eq(event.get("field")) 277 | #fields should be independant 278 | event.set("field",nil); 279 | expect(event.get("target")).not_to eq(event.get("field")) 280 | end 281 | end 282 | 283 | context "when source field is an array" do 284 | 285 | let(:attrs) { { "field" => ["foo","bar"] } } 286 | 287 | it "should not modify source field nil" do 288 | subject.filter(event) 289 | expect(event.get("target")).to eq(event.get("field")) 290 | #fields should be independant 291 | event.set("field",event.get("field") << "baz") 292 | expect(event.get("target")).not_to eq(event.get("field")) 293 | end 294 | end 295 | 296 | context "when source field is a hash" do 297 | 298 | let(:attrs) { { "field" => { "foo" => "bar"} } } 299 | 300 | it "should not modify source field nil" do 301 | subject.filter(event) 302 | expect(event.get("target")).to eq(event.get("field")) 303 | #fields should be independant 304 | event.set("[field][foo]","baz") 305 | expect(event.get("[target][foo]")).not_to eq(event.get("[field][foo]")) 306 | end 307 | end 308 | end 309 | end 310 | 311 | describe LogStash::Filters::Mutate do 312 | 313 | context "config validation" do 314 | describe "invalid convert type should raise a configuration error" do 315 | config <<-CONFIG 316 | filter { 317 | mutate { 318 | convert => [ "message", "int"] #should be integer 319 | } 320 | } 321 | CONFIG 322 | 323 | sample "not_really_important" do 324 | expect {subject}.to raise_error(LogStash::ConfigurationError, /Invalid conversion type/) 325 | end 326 | end 327 | describe "invalid gsub triad should raise a configuration error" do 328 | config <<-CONFIG 329 | filter { 330 | mutate { 331 | gsub => [ "message", "toreplace"] 332 | } 333 | } 334 | CONFIG 335 | 336 | sample "not_really_important" do 337 | expect {subject}.to raise_error(LogStash::ConfigurationError, /Invalid gsub configuration/) 338 | end 339 | end 340 | end 341 | 342 | describe "basics" do 343 | config <<-CONFIG 344 | filter { 345 | mutate { 346 | lowercase => ["lowerme","Lowerme", "lowerMe"] 347 | uppercase => ["upperme", "Upperme", "upperMe"] 348 | capitalize => ["capitalizeme", "Capitalizeme", "capitalizeMe"] 349 | convert => [ "intme", "integer", "floatme", "float" ] 350 | rename => [ "rename1", "rename2" ] 351 | replace => [ "replaceme", "hello world" ] 352 | replace => [ "newfield", "newnew" ] 353 | update => [ "nosuchfield", "weee" ] 354 | update => [ "updateme", "updated" ] 355 | 356 | } 357 | } 358 | CONFIG 359 | 360 | event = { 361 | "lowerme" => "example", 362 | "upperme" => "EXAMPLE", 363 | "capitalizeme" => "Example", 364 | "Lowerme" => "ExAmPlE", 365 | "Upperme" => "ExAmPlE", 366 | "Capitalizeme" => "ExAmPlE", 367 | "lowerMe" => [ "ExAmPlE", "example" ], 368 | "upperMe" => [ "ExAmPlE", "EXAMPLE" ], 369 | "capitalizeMe" => [ "ExAmPlE", "Example" ], 370 | "intme" => [ "1234", "7890.4", "7.9" ], 371 | "floatme" => [ "1234.455" ], 372 | "rename1" => [ "hello world" ], 373 | "updateme" => [ "who cares" ], 374 | "replaceme" => [ "who cares" ] 375 | } 376 | 377 | sample event do 378 | expect(subject.get("lowerme")).to eq 'example' 379 | expect(subject.get("upperme")).to eq 'EXAMPLE' 380 | expect(subject.get("capitalizeme")).to eq 'Example' 381 | expect(subject.get("Lowerme")).to eq 'example' 382 | expect(subject.get("Upperme")).to eq 'EXAMPLE' 383 | expect(subject.get("Capitalizeme")).to eq 'Example' 384 | expect(subject.get("lowerMe")).to eq ['example', 'example'] 385 | expect(subject.get("upperMe")).to eq ['EXAMPLE', 'EXAMPLE'] 386 | expect(subject.get("capitalizeMe")).to eq ['Example', 'Example'] 387 | expect(subject.get("intme") ).to eq [1234, 7890, 7] 388 | expect(subject.get("floatme")).to eq [1234.455] 389 | expect(subject).not_to include("rename1") 390 | expect(subject.get("rename2")).to eq [ "hello world" ] 391 | 392 | expect(subject).to include("newfield") 393 | expect(subject.get("newfield")).to eq "newnew" 394 | expect(subject).not_to include("nosuchfield") 395 | expect(subject.get("updateme")).to eq "updated" 396 | end 397 | end 398 | 399 | describe "case handling of multibyte unicode strings will only change ASCII" do 400 | config <<-CONFIG 401 | filter { 402 | mutate { 403 | lowercase => ["lowerme"] 404 | uppercase => ["upperme"] 405 | capitalize => ["capitalizeme"] 406 | } 407 | } 408 | CONFIG 409 | 410 | event = { 411 | "lowerme" => [ "АБВГД\0MMM", "こにちわ", "XyZółć", "NÎcË GÛŸ"], 412 | "upperme" => [ "аБвгд\0mmm", "こにちわ", "xYzółć", "Nîcë gûÿ"], 413 | "capitalizeme" => ["АБВГД\0mmm", "こにちわ", "xyzółć", "nÎcË gÛŸ"], 414 | } 415 | 416 | sample event do 417 | expect(subject.get("lowerme")).to eq [ "абвгд\0mmm", "こにちわ", "xyzółć", "nîcë gûÿ"] 418 | expect(subject.get("upperme")).to eq [ "АБВГД\0MMM", "こにちわ", "XYZÓŁĆ", "NÎCË GÛŸ"] 419 | expect(subject.get("capitalizeme")).to eq [ "Абвгд\0mmm", "こにちわ", "Xyzółć", "Nîcë gûÿ"] 420 | end 421 | end 422 | 423 | describe "convert one field to string" do 424 | config ' 425 | filter { 426 | mutate { 427 | convert => [ "unicorns", "string" ] 428 | } 429 | }' 430 | 431 | sample({"unicorns" => 1234}) do 432 | expect(subject.get("unicorns")).to eq "1234" 433 | end 434 | end 435 | 436 | describe "convert strings to boolean values" do 437 | config <<-CONFIG 438 | filter { 439 | mutate { 440 | convert => { "true_field" => "boolean" } 441 | convert => { "false_field" => "boolean" } 442 | convert => { "true_upper" => "boolean" } 443 | convert => { "false_upper" => "boolean" } 444 | convert => { "true_one" => "boolean" } 445 | convert => { "false_zero" => "boolean" } 446 | convert => { "true_yes" => "boolean" } 447 | convert => { "false_no" => "boolean" } 448 | convert => { "true_y" => "boolean" } 449 | convert => { "false_n" => "boolean" } 450 | convert => { "wrong_field" => "boolean" } 451 | convert => { "integer_false" => "boolean" } 452 | convert => { "integer_true" => "boolean" } 453 | convert => { "integer_negative" => "boolean" } 454 | convert => { "integer_wrong" => "boolean" } 455 | convert => { "float_true" => "boolean" } 456 | convert => { "float_false" => "boolean" } 457 | convert => { "float_negative" => "boolean" } 458 | convert => { "float_wrong" => "boolean" } 459 | convert => { "float_wrong2" => "boolean" } 460 | convert => { "array" => "boolean" } 461 | convert => { "hash" => "boolean" } 462 | } 463 | } 464 | CONFIG 465 | event = { 466 | "true_field" => "true", 467 | "false_field" => "false", 468 | "true_upper" => "True", 469 | "false_upper" => "False", 470 | "true_one" => "1", 471 | "false_zero" => "0", 472 | "true_yes" => "yes", 473 | "false_no" => "no", 474 | "true_y" => "Y", 475 | "false_n" => "N", 476 | "wrong_field" => "none of the above", 477 | "integer_false" => 0, 478 | "integer_true" => 1, 479 | "integer_negative"=> -1, 480 | "integer_wrong" => 2, 481 | "float_true" => 1.0, 482 | "float_false" => 0.0, 483 | "float_negative" => -1.0, 484 | "float_wrong" => 1.0123, 485 | "float_wrong2" => 0.01, 486 | "array" => [ "1", "0", 0,1,2], 487 | "hash" => { "a" => 0 } 488 | } 489 | sample event do 490 | expect(subject.get("true_field") ).to eq(true) 491 | expect(subject.get("false_field") ).to eq(false) 492 | expect(subject.get("true_upper") ).to eq(true) 493 | expect(subject.get("false_upper") ).to eq(false) 494 | expect(subject.get("true_one") ).to eq(true) 495 | expect(subject.get("false_zero") ).to eq(false) 496 | expect(subject.get("true_yes") ).to eq(true) 497 | expect(subject.get("false_no") ).to eq(false) 498 | expect(subject.get("true_y") ).to eq(true) 499 | expect(subject.get("false_n") ).to eq(false) 500 | expect(subject.get("wrong_field") ).to eq("none of the above") 501 | expect(subject.get("integer_false") ).to eq(false) 502 | expect(subject.get("integer_true") ).to eq(true) 503 | expect(subject.get("integer_negative")).to eq(-1) 504 | expect(subject.get("integer_wrong") ).to eq(2) 505 | expect(subject.get("float_true") ).to eq(true) 506 | expect(subject.get("float_false") ).to eq(false) 507 | expect(subject.get("float_negative") ).to eq(-1.0) 508 | expect(subject.get("float_wrong") ).to eq(1.0123) 509 | expect(subject.get("float_wrong2") ).to eq(0.01) 510 | expect(subject.get("array") ).to eq([true, false, false, true,2]) 511 | expect(subject.get("hash") ).to eq({ "a" => 0 }) 512 | end 513 | end 514 | 515 | describe "convert to float" do 516 | 517 | config <<-CONFIG 518 | filter { 519 | mutate { 520 | convert => { 521 | "field" => "float" 522 | } 523 | } 524 | } 525 | CONFIG 526 | 527 | context 'when field is a string with no separator and dot decimal' do 528 | sample({'field' => '3141.5926'}) do 529 | expect(subject.get('field')).to be_within(0.0001).of(3141.5926) 530 | end 531 | end 532 | 533 | context 'when field is a string with a comma separator and dot decimal' do 534 | sample({'field' => '3,141.5926'}) do 535 | expect(subject.get('field')).to be_within(0.0001).of(3141.5926) 536 | end 537 | end 538 | 539 | context 'when field is a string comma separator and no decimal' do 540 | sample({'field' => '3,141'}) do 541 | expect(subject.get('field')).to be_within(0.0001).of(3141.0) 542 | end 543 | end 544 | 545 | context 'when field is a string no separator and no decimal' do 546 | sample({'field' => '3141'}) do 547 | expect(subject.get('field')).to be_within(0.0001).of(3141.0) 548 | end 549 | end 550 | 551 | context 'when field is a float' do 552 | sample({'field' => 3.1415926}) do 553 | expect(subject.get('field')).to be_within(0.000001).of(3.1415926) 554 | end 555 | end 556 | 557 | context 'when field is an integer' do 558 | sample({'field' => 3}) do 559 | expect(subject.get('field')).to be_within(0.000001).of(3) 560 | end 561 | end 562 | 563 | context 'when field is the true value' do 564 | sample({'field' => true}) do 565 | expect(subject.get('field')).to eq(1.0) 566 | end 567 | end 568 | 569 | context 'when field is the false value' do 570 | sample({'field' => false}) do 571 | expect(subject.get('field')).to eq(0.0) 572 | end 573 | end 574 | 575 | context 'when field is nil' do 576 | sample({'field' => nil}) do 577 | expect(subject.get('field')).to be_nil 578 | end 579 | end 580 | 581 | context 'when field is not set' do 582 | sample({'field' => nil}) do 583 | expect(subject.get('field')).to be_nil 584 | end 585 | end 586 | end 587 | 588 | 589 | describe "convert to float_eu" do 590 | config <<-CONFIG 591 | filter { 592 | mutate { 593 | convert => { 594 | "field" => "float_eu" 595 | } 596 | } 597 | } 598 | CONFIG 599 | 600 | context 'when field is a string with no separator and comma decimal' do 601 | sample({'field' => '3141,5926'}) do 602 | expect(subject.get('field')).to be_within(0.0001).of(3141.5926) 603 | end 604 | end 605 | 606 | context 'when field is a string with a dot separator and comma decimal' do 607 | sample({'field' => '3.141,5926'}) do 608 | expect(subject.get('field')).to be_within(0.0001).of(3141.5926) 609 | end 610 | end 611 | 612 | context 'when field is a string dot separator and no decimal' do 613 | sample({'field' => '3.141'}) do 614 | expect(subject.get('field')).to be_within(0.0001).of(3141.0) 615 | end 616 | end 617 | 618 | context 'when field is a string no separator and no decimal' do 619 | sample({'field' => '3141'}) do 620 | expect(subject.get('field')).to be_within(0.0001).of(3141.0) 621 | end 622 | end 623 | 624 | context 'when field is a float' do 625 | sample({'field' => 3.1415926}) do 626 | expect(subject.get('field')).to be_within(0.000001).of(3.1415926) 627 | end 628 | end 629 | 630 | context 'when field is an integer' do 631 | sample({'field' => 3}) do 632 | expect(subject.get('field')).to be_within(0.000001).of(3) 633 | end 634 | end 635 | 636 | context 'when field is the true value' do 637 | sample({'field' => true}) do 638 | expect(subject.get('field')).to eq(1.0) 639 | end 640 | end 641 | 642 | context 'when field is the false value' do 643 | sample({'field' => false}) do 644 | expect(subject.get('field')).to eq(0.0) 645 | end 646 | end 647 | 648 | context 'when field is nil' do 649 | sample({'field' => nil}) do 650 | expect(subject.get('field')).to be_nil 651 | end 652 | end 653 | 654 | context 'when field is not set' do 655 | sample({'field' => nil}) do 656 | expect(subject.get('field')).to be_nil 657 | end 658 | end 659 | end 660 | 661 | describe "gsub on a String" do 662 | config ' 663 | filter { 664 | mutate { 665 | gsub => [ "unicorns", "but extinct", "and common" ] 666 | } 667 | }' 668 | 669 | sample({"unicorns" => "Magnificient, but extinct, animals"}) do 670 | expect(subject.get("unicorns")).to eq "Magnificient, and common, animals" 671 | end 672 | end 673 | 674 | describe "gsub on an Array of Strings" do 675 | config ' 676 | filter { 677 | mutate { 678 | gsub => [ "unicorns", "extinct", "common" ] 679 | } 680 | }' 681 | 682 | sample({"unicorns" => [ 683 | "Magnificient extinct animals", "Other extinct ideas" ]} 684 | ) do 685 | expect(subject.get("unicorns")).to eq [ 686 | "Magnificient common animals", 687 | "Other common ideas" 688 | ] 689 | end 690 | end 691 | 692 | describe "gsub on multiple fields" do 693 | config ' 694 | filter { 695 | mutate { 696 | gsub => [ "colors", "red", "blue", 697 | "shapes", "square", "circle" ] 698 | } 699 | }' 700 | 701 | sample({"colors" => "One red car", "shapes" => "Four red squares"}) do 702 | expect(subject.get("colors")).to eq "One blue car" 703 | expect(subject.get("shapes")).to eq "Four red circles" 704 | end 705 | end 706 | 707 | describe "gsub on regular expression" do 708 | config ' 709 | filter { 710 | mutate { 711 | gsub => [ "colors", "\d$", "blue"] 712 | } 713 | }' 714 | 715 | sample({"colors" => "red3"}) do 716 | expect(subject.get("colors")).to eq "redblue" 717 | end 718 | end 719 | 720 | describe "regression - mutate should lowercase a field created by grok" do 721 | config <<-CONFIG 722 | filter { 723 | grok { 724 | match => { "message" => "%{WORD:foo}" } 725 | } 726 | mutate { 727 | lowercase => "foo" 728 | } 729 | } 730 | CONFIG 731 | 732 | sample "HELLO WORLD" do 733 | expect(subject.get("foo")).to eq "hello" 734 | end 735 | end 736 | 737 | describe "LOGSTASH-757: rename should do nothing with a missing field" do 738 | config <<-CONFIG 739 | filter { 740 | mutate { 741 | rename => [ "nosuchfield", "hello" ] 742 | } 743 | } 744 | CONFIG 745 | 746 | sample "whatever" do 747 | expect(subject).not_to include("nosuchfield") 748 | expect(subject).not_to include("hello") 749 | end 750 | end 751 | 752 | describe "rename with dynamic origin field (%{})" do 753 | config <<-CONFIG 754 | filter { 755 | mutate { 756 | rename => [ "field_%{x}", "destination" ] 757 | } 758 | } 759 | CONFIG 760 | 761 | sample({"field_one" => "value", "x" => "one"}) do 762 | expect(subject).to_not include("field_one") 763 | expect(subject).to include("destination") 764 | end 765 | end 766 | 767 | describe "rename with dynamic destination field (%{})" do 768 | config <<-CONFIG 769 | filter { 770 | mutate { 771 | rename => [ "origin", "field_%{x}" ] 772 | } 773 | } 774 | CONFIG 775 | 776 | sample({"field_one" => "value", "x" => "one"}) do 777 | expect(subject).to_not include("origin") 778 | expect(subject).to include("field_one") 779 | end 780 | end 781 | 782 | describe "convert should work on nested fields" do 783 | config <<-CONFIG 784 | filter { 785 | mutate { 786 | convert => [ "[foo][bar]", "integer" ] 787 | } 788 | } 789 | CONFIG 790 | 791 | sample({ "foo" => { "bar" => "1000" } }) do 792 | expect(subject.get("[foo][bar]")).to eq 1000 793 | expect(subject.get("[foo][bar]")).to be_a(Integer) 794 | end 795 | end 796 | 797 | describe "convert should work within arrays" do 798 | config <<-CONFIG 799 | filter { 800 | mutate { 801 | convert => [ "[foo][0]", "integer" ] 802 | } 803 | } 804 | CONFIG 805 | 806 | sample({ "foo" => ["100", "200"] }) do 807 | expect(subject.get("[foo][0]")).to eq 100 808 | expect(subject.get("[foo][0]")).to be_a(Integer) 809 | end 810 | end 811 | 812 | describe "convert booleans to integer" do 813 | config <<-CONFIG 814 | filter { 815 | mutate { 816 | convert => { 817 | "[foo][0]" => "integer" 818 | "[foo][1]" => "integer" 819 | "[foo][2]" => "integer" 820 | "[foo][3]" => "integer" 821 | "[foo][4]" => "integer" 822 | } 823 | } 824 | } 825 | CONFIG 826 | 827 | sample({ "foo" => [false, true, "0", "1", "2"] }) do 828 | expect(subject.get("[foo][0]")).to eq 0 829 | expect(subject.get("[foo][0]")).to be_a(Integer) 830 | expect(subject.get("[foo][1]")).to eq 1 831 | expect(subject.get("[foo][1]")).to be_a(Integer) 832 | expect(subject.get("[foo][2]")).to eq 0 833 | expect(subject.get("[foo][2]")).to be_a(Integer) 834 | expect(subject.get("[foo][3]")).to eq 1 835 | expect(subject.get("[foo][3]")).to be_a(Integer) 836 | expect(subject.get("[foo][4]")).to eq 2 837 | expect(subject.get("[foo][4]")).to be_a(Integer) 838 | end 839 | end 840 | 841 | describe "convert various US/UK strings" do 842 | describe "to integer" do 843 | config <<-CONFIG 844 | filter { 845 | mutate { 846 | convert => { 847 | "[foo][0]" => "integer" 848 | "[foo][1]" => "integer" 849 | "[foo][2]" => "integer" 850 | } 851 | } 852 | } 853 | CONFIG 854 | 855 | sample({ "foo" => ["1,000", "1,234,567.8", "123.4"] }) do 856 | expect(subject.get("[foo][0]")).to eq 1000 857 | expect(subject.get("[foo][0]")).to be_a(Integer) 858 | expect(subject.get("[foo][1]")).to eq 1234567 859 | expect(subject.get("[foo][1]")).to be_a(Integer) 860 | expect(subject.get("[foo][2]")).to eq 123 861 | expect(subject.get("[foo][2]")).to be_a(Integer) 862 | end 863 | end 864 | 865 | describe "to float" do 866 | config <<-CONFIG 867 | filter { 868 | mutate { 869 | convert => { 870 | "[foo][0]" => "float" 871 | "[foo][1]" => "float" 872 | "[foo][2]" => "float" 873 | } 874 | } 875 | } 876 | CONFIG 877 | 878 | sample({ "foo" => ["1,000", "1,234,567.8", "123.4"] }) do 879 | expect(subject.get("[foo][0]")).to eq 1000.0 880 | expect(subject.get("[foo][0]")).to be_a(Float) 881 | expect(subject.get("[foo][1]")).to eq 1234567.8 882 | expect(subject.get("[foo][1]")).to be_a(Float) 883 | expect(subject.get("[foo][2]")).to eq 123.4 884 | expect(subject.get("[foo][2]")).to be_a(Float) 885 | end 886 | end 887 | end 888 | 889 | describe "convert various EU style strings" do 890 | describe "to integer" do 891 | config <<-CONFIG 892 | filter { 893 | mutate { 894 | convert => { 895 | "[foo][0]" => "integer_eu" 896 | "[foo][1]" => "integer_eu" 897 | "[foo][2]" => "integer_eu" 898 | } 899 | } 900 | } 901 | CONFIG 902 | 903 | sample({ "foo" => ["1.000", "1.234.567,8", "123,4"] }) do 904 | expect(subject.get("[foo][0]")).to eq 1000 905 | expect(subject.get("[foo][0]")).to be_a(Integer) 906 | expect(subject.get("[foo][1]")).to eq 1234567 907 | expect(subject.get("[foo][1]")).to be_a(Integer) 908 | expect(subject.get("[foo][2]")).to eq 123 909 | expect(subject.get("[foo][2]")).to be_a(Integer) 910 | end 911 | end 912 | 913 | describe "to float" do 914 | config <<-CONFIG 915 | filter { 916 | mutate { 917 | convert => { 918 | "[foo][0]" => "float_eu" 919 | "[foo][1]" => "float_eu" 920 | "[foo][2]" => "float_eu" 921 | } 922 | } 923 | } 924 | CONFIG 925 | 926 | sample({ "foo" => ["1.000", "1.234.567,8", "123,4"] }) do 927 | expect(subject.get("[foo][0]")).to eq 1000.0 928 | expect(subject.get("[foo][0]")).to be_a(Float) 929 | expect(subject.get("[foo][1]")).to eq 1234567.8 930 | expect(subject.get("[foo][1]")).to be_a(Float) 931 | expect(subject.get("[foo][2]")).to eq 123.4 932 | expect(subject.get("[foo][2]")).to be_a(Float) 933 | end 934 | end 935 | end 936 | 937 | describe "convert auto-frozen values to string" do 938 | config <<-CONFIG 939 | filter { 940 | mutate { 941 | convert => { 942 | "true_field" => "string" 943 | "false_field" => "string" 944 | } 945 | } 946 | } 947 | CONFIG 948 | 949 | sample({ "true_field" => true, "false_field" => false }) do 950 | expect(subject.get("true_field")).to eq "true" 951 | expect(subject.get("true_field")).to be_a(String) 952 | expect(subject.get("false_field")).to eq "false" 953 | expect(subject.get("false_field")).to be_a(String) 954 | end 955 | end 956 | 957 | #LOGSTASH-1529 958 | describe "gsub on a String with dynamic fields (%{}) in pattern" do 959 | config ' 960 | filter { 961 | mutate { 962 | gsub => [ "unicorns", "of type %{unicorn_type}", "green" ] 963 | } 964 | }' 965 | 966 | sample({"unicorns" => "Unicorns of type blue are common", "unicorn_type" => "blue"}) do 967 | expect(subject.get("unicorns")).to eq "Unicorns green are common" 968 | end 969 | end 970 | 971 | #LOGSTASH-1529 972 | describe "gsub on a String with dynamic fields (%{}) in pattern and replace" do 973 | config ' 974 | filter { 975 | mutate { 976 | gsub => [ "unicorns2", "of type %{unicorn_color}", "%{unicorn_color} and green" ] 977 | } 978 | }' 979 | 980 | sample({"unicorns2" => "Unicorns of type blue are common", "unicorn_color" => "blue"}) do 981 | expect(subject.get("unicorns2")).to eq "Unicorns blue and green are common" 982 | end 983 | end 984 | 985 | #LOGSTASH-1529 986 | describe "gsub on a String array with dynamic fields in pattern" do 987 | config ' 988 | filter { 989 | mutate { 990 | gsub => [ "unicorns_array", "of type %{color}", "blue and green" ] 991 | } 992 | }' 993 | 994 | sample({"unicorns_array" => [ 995 | "Unicorns of type blue are found in Alaska", "Unicorns of type blue are extinct" ], 996 | "color" => "blue" } 997 | ) do 998 | expect(subject.get("unicorns_array")).to eq [ 999 | "Unicorns blue and green are found in Alaska", 1000 | "Unicorns blue and green are extinct" 1001 | ] 1002 | end 1003 | end 1004 | 1005 | describe "merge string field into inexisting field" do 1006 | config ' 1007 | filter { 1008 | mutate { 1009 | merge => [ "list", "foo" ] 1010 | } 1011 | }' 1012 | 1013 | sample({"foo" => "bar"}) do 1014 | expect(subject.get("list")).to eq ["bar"] 1015 | expect(subject.get("foo")).to eq "bar" 1016 | end 1017 | end 1018 | 1019 | describe "merge string field into empty array" do 1020 | config ' 1021 | filter { 1022 | mutate { 1023 | merge => [ "list", "foo" ] 1024 | } 1025 | }' 1026 | 1027 | sample({"foo" => "bar", "list" => []}) do 1028 | expect(subject.get("list")).to eq ["bar"] 1029 | expect(subject.get("foo")).to eq "bar" 1030 | end 1031 | end 1032 | 1033 | describe "merge string field into existing array" do 1034 | config ' 1035 | filter { 1036 | mutate { 1037 | merge => [ "list", "foo" ] 1038 | } 1039 | }' 1040 | 1041 | sample({"foo" => "bar", "list" => ["baz"]}) do 1042 | expect(subject.get("list")).to eq ["baz", "bar"] 1043 | expect(subject.get("foo")).to eq "bar" 1044 | end 1045 | end 1046 | 1047 | describe "merge non empty array field into existing array" do 1048 | config ' 1049 | filter { 1050 | mutate { 1051 | merge => [ "list", "foo" ] 1052 | } 1053 | }' 1054 | 1055 | sample({"foo" => ["bar"], "list" => ["baz"]}) do 1056 | expect(subject.get("list")).to eq ["baz", "bar"] 1057 | expect(subject.get("foo")).to eq ["bar"] 1058 | end 1059 | end 1060 | 1061 | describe "merge empty array field into existing array" do 1062 | config ' 1063 | filter { 1064 | mutate { 1065 | merge => [ "list", "foo" ] 1066 | } 1067 | }' 1068 | 1069 | sample({"foo" => [], "list" => ["baz"]}) do 1070 | expect(subject.get("list")).to eq ["baz"] 1071 | expect(subject.get("foo")).to eq [] 1072 | end 1073 | end 1074 | 1075 | describe "merge array field into string field" do 1076 | config ' 1077 | filter { 1078 | mutate { 1079 | merge => [ "list", "foo" ] 1080 | } 1081 | }' 1082 | 1083 | sample({"foo" => ["bar"], "list" => "baz"}) do 1084 | expect(subject.get("list")).to eq ["baz", "bar"] 1085 | expect(subject.get("foo")).to eq ["bar"] 1086 | end 1087 | end 1088 | 1089 | describe "merge string field into string field" do 1090 | config ' 1091 | filter { 1092 | mutate { 1093 | merge => [ "list", "foo" ] 1094 | } 1095 | }' 1096 | 1097 | sample({"foo" => "bar", "list" => "baz"}) do 1098 | expect(subject.get("list")).to eq ["baz", "bar"] 1099 | expect(subject.get("foo")).to eq "bar" 1100 | end 1101 | end 1102 | 1103 | describe "coerce arrays fields with default values when null" do 1104 | config ' 1105 | filter { 1106 | mutate { 1107 | coerce => { 1108 | "field1" => "Hello" 1109 | "field2" => "Bye" 1110 | "field3" => 5 1111 | "field4" => false 1112 | } 1113 | } 1114 | }' 1115 | 1116 | 1117 | sample({"field1" => nil, "field2" => nil, "field3" => nil, "field4" => true}) do 1118 | expect(subject.get("field1")).to eq("Hello") 1119 | expect(subject.get("field2")).to eq("Bye") 1120 | expect(subject.get("field3")).to eq("5") 1121 | expect(subject.get("field4")).to eq(true) 1122 | end 1123 | end 1124 | 1125 | end 1126 | --------------------------------------------------------------------------------