├── .gitignore ├── .rspec ├── .travis.yml ├── .yardopts ├── CHANGELOG.md ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── puppet-profile-parser.rb └── spec ├── fixtures ├── puppetserver.log └── zipkin │ ├── README.md │ └── listofspans.json ├── spec_helper.rb └── unit ├── profile_parser_cli_spec.rb ├── profile_parser_spec.rb └── profile_parser_zipkin_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle/ 2 | .yardoc/ 3 | Gemfile.lock 4 | Gemfile.local 5 | doc/ 6 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: ruby 3 | sudo: false 4 | cache: bundler 5 | notifications: 6 | email: false 7 | 8 | matrix: 9 | include: 10 | - rvm: 2.0.0-p648 11 | script: 'bundle exec rake spec:unit' 12 | - rvm: 2.4.9 13 | script: 'bundle exec rake spec:unit' 14 | - rvm: 2.5.7 15 | script: 'bundle exec rake spec:unit' 16 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup markdown 2 | puppet-profile-parser.rb 3 | - 4 | CHANGELOG.md 5 | LICENSE 6 | README.md 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/) 6 | and this project adheres to [Semantic Versioning](http://semver.org/). 7 | 8 | ## [0.3.0] - 2018-05-25 9 | 10 | ### Added 11 | 12 | - A `--debug` flag which enables error backtraces. 13 | 14 | - PuppetDB operations are now parsed explicitly instead of being part 15 | of the "other" span group. 16 | 17 | - HTTP requests are parsed explicitly and tagged with method, url and 18 | peer.certname (if available). 19 | 20 | - Support for Ruby 2.0. 21 | 22 | ### Fixed 23 | 24 | - The script now exits with code 1 if an error is raised. 25 | 26 | - The script now allows for a PROFILE message containing `]` characters. 27 | 28 | - Zipkin output no longer contains a redundant `span.kind` tag. 29 | 30 | - The profile parser exits 1 and prints a clear error message if run 31 | under a Ruby version older than 2.0. 32 | 33 | ### Changed 34 | 35 | - The `get` method of the Trace class is now private. 36 | 37 | - The FunctionSpan, ResourceSpan, and OtherSpan classes have been replaced 38 | by the generic Span class and all parsing logic has been moved to the 39 | TraceParser class. 40 | 41 | - The CsvOutput, FlameGraphOutput, HumanOutput, and ZipkinOutput classes have 42 | been nested under a new Formatter class as Formatter::Csv, Formatter:FlameGraph, 43 | Formatter::Human and Formatter::Zipkin. The `display` method has also been 44 | re-named to `write`. 45 | 46 | 47 | ## [0.2.0] - 2018-03-18 48 | 49 | ### Added 50 | 51 | - Colorization of output can be toggled via the `--color` and `--no-color` 52 | flags. 53 | 54 | - The LogParser class now has a `parse_file` method. 55 | 56 | - Each Trace is assigned a random UUID. 57 | 58 | - CSV output format. 59 | 60 | - JSON output for [v2 of the Zipkin API][zipkin-v2]. 61 | 62 | [zipkin-v2]: https://github.com/openzipkin/zipkin-api 63 | 64 | ### Changed 65 | 66 | - The Namespace class has been re-named to Trace and the Slice classes 67 | have been renamed to Span. This matches the implementation up with 68 | [OpenTracing terminology][opentracing-spec]. 69 | 70 | - The `profile-parser.rb` script has been re-named to `puppet-profile-parser.rb` 71 | for clarity. 72 | 73 | - The PuppetProfiler module has been re-named to PuppetProfileParser for 74 | consistency with the script name. 75 | 76 | - The script prints usage to stderr and exits 1 if no log files are passed. 77 | 78 | [opentracing-spec]: https://github.com/opentracing/specification/blob/master/specification.md 79 | 80 | 81 | ## [0.1.0] - 2018-03-02 82 | 83 | ### Added 84 | 85 | - RSpec tests. 86 | 87 | - Apache 2 license. 88 | 89 | - Support for reading gzipped log files. 90 | 91 | - Support for FlameGraph output. 92 | 93 | ### Changed 94 | 95 | - Profiled events are now separated by request and Java thread id. Incomplete 96 | profiles are dropped. 97 | 98 | - Inclusive and exclusive times are computed for each profile span. Output 99 | uses exclisive time so that hot spots aren't hidden by spans double 100 | counting the time taken by their children. 101 | 102 | ### Removed 103 | 104 | - Dependency on the `colored` and `terminal-table` gems. 105 | 106 | - The `catalog-analyzer.rb` script has been removed. It may return in the 107 | future, but for now the project will focus on parsing profile data. 108 | 109 | 110 | ## [0.0.1] - 2014-05-19 111 | 112 | Initial version by [Adrien Thebo](https://github.com/adrienthebo) 113 | 114 | 115 | [0.3.0]: https://github.com/Sharpie/puppet-profile-parser/compare/0.2.0...0.3.0 116 | [0.2.0]: https://github.com/Sharpie/puppet-profile-parser/compare/0.1.0...0.2.0 117 | [0.1.0]: https://github.com/Sharpie/puppet-profile-parser/compare/0.0.1...0.1.0 118 | [0.0.1]: https://github.com/Sharpie/puppet-profile-parser/compare/53a9d9f...0.0.1 119 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :development do 4 | gem 'rspec', '~> 3.7' 5 | # 2.6.2 was the last json-schema version with support for Ruby 2.0. 6 | gem 'json-schema', '2.6.2' 7 | gem 'rake', '~> 12.3' 8 | gem 'yard', '~> 0.9.12' 9 | end 10 | 11 | if File.exists? "#{__FILE__}.local" 12 | eval(File.read("#{__FILE__}.local"), binding) 13 | end 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Puppet Profile Parser 2 | ===================== 3 | 4 | [![Build Status](https://travis-ci.org/Sharpie/puppet-profile-parser.svg?branch=master)](https://travis-ci.org/Sharpie/puppet-profile-parser) 5 | 6 | Tools for parsing profile information from Puppet Server logs and transforming 7 | to various output formats. 8 | 9 | 10 | Installation 11 | ------------ 12 | 13 | The tool in this repository is the `puppet-profile-parser.rb` script. This 14 | script has no dependencies and can be run from any location where a Ruby 15 | interpreter is present on the `PATH`. 16 | 17 | The script requires Ruby 2.0 or newer. 18 | 19 | The most recent stable release of the script can be downloaded here: 20 | 21 | [Stable release: 0.3.0][stable-release] 22 | 23 | 24 | And the latest development version can be downloaded from: 25 | 26 | [Edge release from master branch][edge-release] 27 | 28 | [stable-release]: https://github.com/Sharpie/puppet-profile-parser/releases/download/0.3.0/puppet-profile-parser.rb 29 | [edge-release]: https://raw.githubusercontent.com/Sharpie/puppet-profile-parser/master/puppet-profile-parser.rb 30 | 31 | 32 | Usage 33 | ----- 34 | 35 | ### Generating profiles 36 | 37 | The `puppet-profile-parser.rb` script scans Puppet Server logs for information 38 | generated when the Puppet profiler is enabled. The profiler is disabled by 39 | default and can be enabled using the [profile setting][profile-setting] 40 | in `puppet.conf`. 41 | 42 | [profile-setting]: https://puppet.com/docs/puppet/5.4/configuration.html#profile 43 | 44 | #### Profiling specific agents 45 | 46 | A single profile can be generated for an agent by executing a test run 47 | with the `--profile` flag: 48 | 49 | puppet agent -t --profile 50 | 51 | Setting `profile=true` in the `[agent]` section of `puppet.conf` and restarting 52 | the `puppet` service will cause all runs to generate profiling information. The 53 | profile results will be located in the Puppet Server logs. 54 | 55 | #### Profiling all agents 56 | 57 | Profiling can be enabled globally by setting `profile=true` in the `[master]` 58 | section of `puppet.conf` and restarting the `puppetserver` service. 59 | 60 | #### Configure Puppet Server logging 61 | 62 | Depending on the version in use, there are certain adjustments you'll want to 63 | make to the Puppet Server logging configuration. These adjustments can be made 64 | in the main logback configuration file: 65 | 66 | /etc/puppetlabs/puppetserver/logback.xml 67 | 68 | If `puppet --version` is less than `4.8.0`, the level for the `puppetserver` 69 | logger will need to be raised to DEBUG by adding the following configuration 70 | towards the bottom of the file: 71 | 72 | 73 | 74 | Or, upgrade the `puppet-agent` package to provide Puppet 4.8.0, where profiling 75 | data is logged at the default INFO level. 76 | 77 | Puppet Server should be configured to include the time zone in log timestamps. 78 | This can be done by adjusting the `pattern` of the `F1` appender: 79 | 80 | %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5p [%t] [%c{2}] %m%n 81 | 82 | Adding the time zone improves allows logs to be processed with accurate time 83 | stamps. 84 | 85 | ### Parsing profiles 86 | 87 | The `puppet-profile-parser.rb` script expects list of Puppet Server 88 | log files containing `PROFILE` entries: 89 | 90 | ./puppet-profile-parser.rb .../path/to/puppetserver.log [.../more/logs] 91 | 92 | The script is capable of reading both plaintext log files and archived 93 | log files that have been compressed with `gzip`. Compressed log files 94 | must have a name that ends in `.gz` in order to be read. 95 | 96 | A full list of options can be displayed using the `--help` flag: 97 | 98 | ./puppet-profile-parser.rb --help 99 | 100 | ### Output formats 101 | 102 | The profile parser extracts a "trace" for each request profiled by the Puppet 103 | Server. Within each trace are a number of nested "spans" representing 104 | instrumented oprations executed to generate a response for the request. The 105 | terminology of traces and spans follows the [OpenTracing specification][opentracing-spec]. 106 | 107 | The parser is capable of rendering traces to `$stdout` in a variety of 108 | output formats which are selected using the `--format` flag. The currently 109 | supported output formats are: 110 | 111 | - Human-readable (default) 112 | - CSV 113 | - FlameGraph stacks 114 | - Zipkin JSON 115 | 116 | **NOTE:** Each trace is currently assigned a randomly-generated UUIDv4 as an 117 | identifier. Re-running the script on the same input will result in the same 118 | traces, but with new randomly generated IDs. These IDs are included in the 119 | CSV and Zipkin output formats. 120 | 121 | [opentracing-spec]: https://github.com/opentracing/specification/blob/master/specification.md 122 | 123 | #### Human readable 124 | 125 | The `human` output format is the default used by the script if the `--format` 126 | flag is not used to select another option This output format displays each 127 | trace parsed from the logs as an indented list followed by summary tables for 128 | function calls, resource evaluations, and "other" operations measured by the 129 | profiler. The summary tables are sorted in terms of "exclusive" time, which is 130 | the time spent on the operation after excluding any time spent on nested child 131 | operations. 132 | 133 | In POSIX environments, the traces are also colorized using ANSI color codes. 134 | This behavior can be toggled using the `--color` and `--no-color` flags. 135 | 136 | 137 | #### CSV 138 | 139 | The CSV output format prints a header row followed by a row for each span in 140 | each trace using a comma-separated format. The columns included in the CSV 141 | output are: 142 | 143 | - `timestamp`: ISO-8601 formatted timestamp with timezone indicating when 144 | the span was logged. 145 | 146 | - `trace_id`: Randomly assigned UUIDv4 for each trace. 147 | 148 | - `span_id`: Dot-delimited sequence of numbers indicating span number 149 | and nesting depth. 150 | 151 | - `name`: Name of the profiled operation. 152 | 153 | - `exclusive_time_ms`: Milliseconds spent on the span, excluding time 154 | spent on nested child spans. 155 | 156 | - `inclusive_time_ms`: Milliseconds spent on the span, including time 157 | spent on nested child spans. 158 | 159 | 160 | #### FlameGraph 161 | 162 | The `flamegraph` output format prints each span in each trace as a semi-colon 163 | delimited call stack followed by the number of milliseconds measured for 164 | that span. This output format can be piped into the `flamegraph.pl` script 165 | from [brendangregg/FlameGraph][flamegraph] to create interactive SVG 166 | visualizations: 167 | 168 | ./puppet-profile-parser.rb -f flamegraph puppetserver.log | \ 169 | path/to/flamegraph.pl --countname ms > puppet_profile.svg 170 | 171 | An example SVG generated by `flamegraph.pl` (click for interactive version): 172 | 173 | [![Example FlameGraph, click for interactive version.][example-flamegraph]][example-flamegraph] 174 | 175 | [flamegraph]: https://github.com/brendangregg/FlameGraph 176 | [example-flamegraph]: https://sharpie.github.io/puppet-profile-parser/assets/puppet_profile.svg 177 | 178 | 179 | #### Zipkin JSON 180 | 181 | The `zipkin` output format prints a JSON array containing each span in each 182 | trace. The JSON array conforms to the `ListOfSpans` data type defined by the 183 | [Zipkin v2 API specification][zipkin-v2-spec]. This allows the JSON output 184 | to be submitted as a POST request to services which implement the API: 185 | 186 | ./puppet-profile-parser.rb -f zipkin puppetserver.log | \ 187 | curl -X POST -H 'Content-Type: application/json' \ 188 | http://:9411/api/v2/spans --data @- 189 | 190 | There are two implementations of the Zipkin API that can be spun up quickly 191 | inside of Docker containers: 192 | 193 | - [Jaeger][jaeger], an implementation by Uber now part of the CNCF: 194 | 195 | docker run -d -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ 196 | -p 16686:16686 -p 9411:9411 jaegertracing/all-in-one:latest 197 | 198 | Visit to view profiling data in the Jaeger UI. 199 | Double check the date range in the "lookback" setting if no traces show up. 200 | 201 | - [Zipkin][zipkin], the original implementation by Twitter: 202 | 203 | docker run -d -p 9411:9411 openzipkin/zipkin 204 | 205 | Visit to view profiling data in the Zipkin UI. 206 | Double check the date range in the "lookback" setting if no traces show up. 207 | 208 | 209 | [zipkin-v2-spec]: https://zipkin.io/zipkin-api/ 210 | [jaeger]: http://jaeger.readthedocs.io/en/latest/ 211 | [zipkin]: https://zipkin.io/ 212 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rspec/core/rake_task' 2 | 3 | namespace(:spec) do 4 | desc 'Run RSpec unit tests' 5 | RSpec::Core::RakeTask.new(:unit) do |task| 6 | task.pattern = 'spec/unit/**{,/*/**}/*_spec.rb' 7 | end 8 | end 9 | 10 | desc 'Run all test suites' 11 | task(:test => ['spec:unit']) 12 | 13 | 14 | require 'yard' 15 | require 'yard/rake/yardoc_task' 16 | 17 | yard = YARD::Rake::YardocTask.new(:doc) 18 | -------------------------------------------------------------------------------- /puppet-profile-parser.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- coding: utf-8 -*- 3 | 4 | # puppet-profile-parser.rb Parse Puppet Server logs for PROFILE data 5 | # and transform to various output formats. 6 | # 7 | # Copyright 2018 Charlie Sharpsteen 8 | # Copyright 2014 Adrien Thebo 9 | # 10 | # Licensed under the Apache License, Version 2.0 (the "License"); 11 | # you may not use this file except in compliance with the License. 12 | # You may obtain a copy of the License at 13 | # 14 | # http://www.apache.org/licenses/LICENSE-2.0 15 | # 16 | # Unless required by applicable law or agreed to in writing, software 17 | # distributed under the License is distributed on an "AS IS" BASIS, 18 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | # See the License for the specific language governing permissions and 20 | # limitations under the License. 21 | 22 | require 'zlib' 23 | require 'optparse' 24 | require 'securerandom' 25 | require 'csv' 26 | require 'time' 27 | require 'json' 28 | require 'digest/sha2' 29 | require 'rubygems/requirement' 30 | 31 | # Tools for parsing and formatting Puppet Server PROFILE logs 32 | # 33 | # This module wraps components that are used to extract data from PROFILE 34 | # lines in Puppet Server logs and then convert the extracted data to 35 | # various output formats. 36 | # 37 | # A CLI is also provided to create an easy to run tool. 38 | # 39 | # @author Charlie Sharpsteen 40 | # @author Adrien Thebo 41 | module PuppetProfileParser 42 | VERSION = '0.3.0'.freeze 43 | REQUIRED_RUBY_VERSION = Gem::Requirement.new('>= 2.0') 44 | 45 | # Utility functions for terminal interaction 46 | # 47 | # A collection of functions for colorizing output. 48 | module Tty 49 | # Pre-defined list of ANSI escape codes 50 | # 51 | # @return [Hash{Symbol => Integer}] A hash mapping human-readable color 52 | # names to ANSI escape numbers. 53 | COLOR_CODES = { 54 | red: 31, 55 | green: 32, 56 | yellow: 33, 57 | }.freeze 58 | 59 | # Detect whether Ruby is running in a Windows environment 60 | # 61 | # @return [Boolean] Returns `true` if the alternate path separator is 62 | # defined to be a backslash. 63 | def self.windows? 64 | @is_windows ||= (::File::ALT_SEPARATOR == "\\") 65 | end 66 | 67 | # Detect whether Ruby is run interactively 68 | # 69 | # @return [Boolean] Returns true if the standard output is a TTY. 70 | def self.tty? 71 | @is_tty ||= $stdout.tty? 72 | end 73 | 74 | # Maybe wrap a string in ANSI escape codes for color 75 | # 76 | # @param color [Integer] The color to apply, as a number from the 77 | # ANSI color table. 78 | # @param string [String] The string to which color should be applied. 79 | # @param enable [Boolean] Override logic for detecting when to apply 80 | # color. 81 | # 82 | # @return [String] A colorized string, if `enable` is set to true or 83 | # if {.windows?} returns false, {.tty?} returns true and `enable` 84 | # is unset. 85 | # @return [String] The original string unmodified if `enable` is set 86 | # to false, or {.windows?} returns true, or {.tty?} returns false. 87 | def self.colorize(color, string, enable = nil) 88 | if (!windows?) && ((enable.nil? && tty?) || enable) 89 | "\033[#{COLOR_CODES[color]}m#{string}\033[0m" 90 | else 91 | string 92 | end 93 | end 94 | 95 | COLOR_CODES.keys.each do |name| 96 | define_singleton_method(name) do |string, enable = nil| 97 | colorize(name, string, enable) 98 | end 99 | end 100 | end 101 | 102 | # Organize nested Span objects 103 | # 104 | # The Trace class implements logic for organizing related {Span} objects 105 | # into a hierarchy representing a single profile operation. The Trace class 106 | # wraps a {Span} instance to provide some core capabilities: 107 | # 108 | # - {#add}: Add a new {Span} object to the trace at a given position in 109 | # the hierarchy. 110 | # 111 | # - {#each}: Implements the functionality of the core Ruby Enumerable 112 | # module by yielding `self` followed by all child spans nested under 113 | # `self`. 114 | # 115 | # - {#finalize!}: Iterates through `self` and all child spans to compute 116 | # summary statistics. Should only be called once when no further spans 117 | # will be added to the Trace. 118 | # 119 | # @see Span Span class. 120 | # @see https://github.com/opentracing/specification/blob/1.1/specification.md#the-opentracing-data-model 121 | # Definitions of "Trace" and "Span" from the OpenTracing project. 122 | class Trace 123 | include Enumerable 124 | 125 | # Array of strings giving the nesting depth and order of the span 126 | # 127 | # @return [Array[String]] 128 | attr_reader :namespace 129 | # Wrapped Span object 130 | # 131 | # @return [Span] 132 | attr_reader :object 133 | # Unique ID for this Trace and its children 134 | # 135 | # @return [String] 136 | attr_reader :trace_id 137 | 138 | # Milliseconds spent on this operation, excluding child operations 139 | # 140 | # Equal to inclusive_time minus the sum of inclusive_time for children. 141 | # 142 | # @return [Integer] 143 | # @return [nil] If {#finalize!} has not been called. 144 | attr_reader :exclusive_time 145 | # Milliseconds spent on this operation, including child operations 146 | # 147 | # @return [Integer] 148 | # @return [nil] If {#finalize!} has not been called. 149 | attr_reader :inclusive_time 150 | # Array of operation names 151 | # 152 | # @return [Array[String]] 153 | # @return [nil] If {#finalize!} has not been called. 154 | attr_reader :stack 155 | 156 | # Initialize a new Trace instance 157 | # 158 | # @param namespace [String] A single string containing a sequence of 159 | # namespace segments. Puppet uses integers separated by `.` characters 160 | # to represent nesting depth and order of profiled operations. 161 | # 162 | # @param object [Span] A {Span} instance representing the operation 163 | # associated with the `namespace`. 164 | # 165 | # @param trace_id [String] A string giving a unique id for this trace 166 | # instance and its children. Defaults to a UUIDv4 produced by 167 | # `SecureRandom.uuid`. 168 | def initialize(namespace, object, trace_id = nil) 169 | @namespace = namespace.split('.') 170 | @object = object 171 | @trace_id = trace_id || SecureRandom.uuid 172 | @children = {} 173 | @exclusive_time = nil 174 | @inclusive_time = nil 175 | end 176 | 177 | # Add a new Span object as a child of the current trace 178 | # 179 | # @param namespace [String] A single string containing a sequence of 180 | # namespace segments. Puppet uses integers separated by `.` characters 181 | # to represent nesting depth and order of profiled operations. 182 | # 183 | # @param object [Span] A {Span} instance representing the operation 184 | # associated with the `namespace`. 185 | # 186 | # @return [void] 187 | def add(namespace, object) 188 | parts = namespace.split('.') 189 | 190 | child_ns = parts[0..-2] 191 | child_id = parts.last 192 | 193 | if(parts == @namespace) 194 | # We are the object 195 | @object = object 196 | elsif @namespace == child_ns 197 | get(child_id).add(namespace, object) 198 | else 199 | id = parts.drop(@namespace.size).first 200 | child = get(id) 201 | child.add(namespace, object) 202 | end 203 | end 204 | 205 | # Yield self followed by all Trace instances that are children of self 206 | # 207 | # @yield [Trace] 208 | def each 209 | yield self 210 | 211 | @children.each do |_, child| 212 | child.each {|grandchild| yield grandchild } 213 | end 214 | end 215 | 216 | # Compute summary statistics 217 | # 218 | # This method should be called once all child spans have been added 219 | # in order to compute summary statistics for the entire trace. The 220 | # {Span#finish!} method is also called on wrapped span objects in 221 | # order to finalize their state. 222 | # 223 | # @see Span#finish! 224 | # 225 | # @return [void] 226 | def finalize! 227 | do_finalize! 228 | end 229 | 230 | protected 231 | 232 | # Internals of finalize that handle object state 233 | def do_finalize!(parent_stack = [], parent_object = nil) 234 | @stack = parent_stack + [object.name] 235 | 236 | @children.each {|_, child| child.do_finalize!(@stack, object) } 237 | 238 | @inclusive_time = Integer(object.time * 1000) 239 | 240 | child_time = @children.inject(0) {|sum, (_, child)| sum + child.inclusive_time } 241 | @exclusive_time = @inclusive_time - child_time 242 | @exclusive_time = 0 unless (@exclusive_time >= 0) 243 | 244 | # Copy state to our wrapped span. 245 | object.context[:trace_id] = @trace_id 246 | object.references << ['child_of', parent_object.id] unless parent_object.nil? 247 | object.finish! 248 | end 249 | 250 | private 251 | 252 | # Get or create a child Trace at the given nesting depth 253 | def get(id) 254 | @children[id] ||= Trace.new([@namespace, id].flatten.join('.'), 255 | nil, 256 | @trace_id) 257 | end 258 | end 259 | 260 | # Profile data from a discrete operation 261 | # 262 | # Instances of the Span class encapsulate details of a single operation 263 | # measured by the Puppet profiler. The data managed by a Span object is 264 | # a subset of of items defined by the OpenTracing specification: 265 | # 266 | # - {context}: Provides ID values that identify the span and the trace 267 | # it is associated with. 268 | # 269 | # - {name}: A name that identifies the operation measured by the 270 | # Span instance. 271 | # 272 | # - {references}: A list of references to related Span instances, such 273 | # as parent Spans. 274 | # 275 | # - {tags}: A key/value map of data extracted from the Puppet Server 276 | # PROFILE log line used to generate the Span instance. 277 | # 278 | # If the logs used to generate the Span included timestamps, then the 279 | # following standard peices of data will also be available: 280 | # 281 | # - {start_time} 282 | # - {finish_time} 283 | # 284 | # Spans also include a non-standard {time} which is present even if the 285 | # logs lacked Timestamp information. 286 | # 287 | # Most of these fields will be `nil` or otherwise incomplete unless the 288 | # Span instance is associated with a {Trace} instance via {Trace#add} 289 | # which is then finanlized via {Trace#finalize!}. The finalize method 290 | # of the Trace class fills in many details of the Span class, such as 291 | # the trace ID. 292 | # 293 | # 294 | # @see Trace Trace class. 295 | # @see https://github.com/opentracing/specification/blob/1.1/specification.md#the-opentracing-data-model 296 | # Definitions of "Trace" and "Span" from the OpenTracing project. 297 | class Span 298 | # Operation name 299 | # 300 | # @return [String] 301 | attr_reader :name 302 | 303 | # Duration of operation measured by the span in seconds 304 | # 305 | # @return [Float] 306 | attr_accessor :time 307 | # Time at which the operation measured by the span started 308 | # 309 | # @return [Time] 310 | # @return [nil] Until {#finish!} is called and {finish_time} is non-nil. 311 | attr_reader :start_time 312 | # Time at which the operation measured by the span finished 313 | # 314 | # @return [Time, nil] 315 | attr_reader :finish_time 316 | 317 | # Values identifying the span and associated {Trace} 318 | # 319 | # @return [Hash{:trace_id, :span_id => String}] 320 | # @return [Hash{:trace_id, :span_id => nil}] Until a span is associated 321 | # with {Trace#add} and {Trace#finalize!} is called on the trace instance. 322 | attr_reader :context 323 | # Data items parsed from PROFILE logs 324 | # 325 | # @see https://github.com/opentracing/specification/blob/1.1/semantic_conventions.md#standard-span-tags-and-log-fields 326 | # List of tags standardized by OpenTracing. 327 | # 328 | # @return [Hash{String => Object}] 329 | attr_reader :tags 330 | # Links to related Span instances 331 | # 332 | # @return [Array] An array of tuples of the form 333 | # `[, ]` that relates this Span instance 334 | # to other Span instances in the same Trace. 335 | attr_reader :references 336 | 337 | def initialize(name = nil, finish_time = nil, tags = {}) 338 | # Typically spans are initialized with the start time. 339 | # But, we're parsing from logs writen _after_ the operation 340 | # completes. So, its finish time. 341 | @finish_time = finish_time 342 | @name = name 343 | 344 | @context = {trace_id: nil, span_id: nil} 345 | @tags = tags.merge({'component' => 'puppetserver', 346 | 'span.kind' => 'server'}) 347 | @references = [] 348 | end 349 | 350 | # Identifier for the span. Unique within a given Trace 351 | # 352 | # @!attribute [r] id 353 | # @return [String] 354 | def id 355 | @context[:span_id] 356 | end 357 | 358 | def inspect 359 | @name 360 | end 361 | 362 | # Finalize Span state 363 | # 364 | # @return [void] 365 | def finish! 366 | unless @finish_time.nil? 367 | @start_time = @finish_time - @time 368 | end 369 | end 370 | end 371 | 372 | # Parser for extracting spans from PROFILE log lines 373 | # 374 | # Instances of the TraceParser class process the messages of log lines 375 | # that include the `PROFILE` keyword and create {Span} instances which 376 | # are eventually grouped into finalized {Trace} instances. 377 | # 378 | # @see LogParser 379 | # @see Trace 380 | # @see Span 381 | class TraceParser 382 | # Regex for extracting span id and duration 383 | COMMON_DATA = /(?[\d\.]+)\s+ 384 | (?.*) 385 | :\stook\s(?[\d\.]+)\sseconds\s*$/x 386 | 387 | FUNCTION_CALL = /Called (?\S+)/ 388 | RESOURCE_EVAL = /Evaluated resource (?(?[\w:]+)\[(?.*)\])/ 389 | PUPPETDB_OP = /PuppetDB: (?[^\(]*)(?:\s\([\w\s]*: \d+\))?\Z/ 390 | # For most versions of PuppetDB, the query function forgot to use a 391 | # helper that added "PuppetDB: " to the beginning of the message. 392 | PUPPETDB_QUERY = /(?Submitted query .*)/ 393 | 394 | HOSTNAME = /\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)/ 395 | CERTNAME_REQUEST = /Processed\srequest\s 396 | (?[A-Z]+)\s 397 | (?.*\/)(?#{HOSTNAME})?\Z/x 398 | HTTP_REQUEST = /Processed request (?[A-Z]+) (?.*\/)/ 399 | 400 | def initialize 401 | @spans = [] 402 | end 403 | 404 | # Parse a log line possibly returning a completed trace 405 | # 406 | # @param line [String] 407 | # @param metadata [Hash] 408 | # 409 | # @return [Trace] A finalized {Trace} instance is returned when a {Span} 410 | # with id `1` is parsed. All span instances parsed thus far are added 411 | # to the trace and the TraceParser re-sets by emptying its list of 412 | # parsed spans. 413 | # 414 | # @return [nil] A `nil` value is returned when the lines parsed thus 415 | # far have not ended in a complete trace. 416 | def parse(line, metadata) 417 | match = COMMON_DATA.match(line) 418 | if match.nil? 419 | $stderr.puts("WARN Could not parse PROFILE message: #{line})") 420 | return nil 421 | end 422 | common_data = LogParser.convert_match(match) {|k, v| v.to_f if k == 'duration' } 423 | 424 | span_data = case common_data['message'] 425 | when FUNCTION_CALL 426 | LogParser.convert_match(Regexp.last_match).merge({ 427 | 'puppet.op_type' => 'function_call'}) 428 | when RESOURCE_EVAL 429 | LogParser.convert_match(Regexp.last_match).merge({ 430 | 'puppet.op_type' => 'resource_eval'}) 431 | when PUPPETDB_OP, PUPPETDB_QUERY 432 | LogParser.convert_match(Regexp.last_match).merge({ 433 | 'puppet.op_type' => 'puppetdb_call'}) 434 | when CERTNAME_REQUEST, HTTP_REQUEST 435 | data = LogParser.convert_match(Regexp.last_match).merge({ 436 | 'puppet.op_type' => 'http_request'}) 437 | 438 | # TODO: Would be nice if there was a reliable way of 439 | # getting the server's hostname so we could use it 440 | # instead of a RFC 2606 example domain. 441 | data['http.url'] = 'https://puppetserver.example:8140' + 442 | data['name'] 443 | 444 | unless data['peer.hostname'].nil? 445 | data['http.url'] += data['peer.hostname'] 446 | end 447 | 448 | data 449 | else 450 | {'name' => common_data['message'], 451 | 'puppet.op_type' => 'other'} 452 | end 453 | 454 | span = Span.new(span_data.delete('name'),metadata['timestamp'], span_data) 455 | span.context[:span_id] = common_data['span_id'] 456 | span.time = common_data['duration'] 457 | 458 | if span.id == '1' 459 | # We've hit the root of a profile, which gets logged last. 460 | trace = Trace.new('1', span) 461 | 462 | @spans.each do |child| 463 | trace.add(child.id, child) 464 | end 465 | 466 | # Re-set for parsing a new profile and return the completed trace. 467 | @spans = [] 468 | 469 | trace.finalize! 470 | return trace 471 | else 472 | @spans << span 473 | 474 | # Return nil to signal we haven't parsed a complete profile yet. 475 | return nil 476 | end 477 | end 478 | end 479 | 480 | # Top-level parser for extracting profile data from logs 481 | # 482 | # Intances of the LogParser class consume content from log files one line 483 | # at a time looking for lines that contain the keyword `PROFILE`. These 484 | # lines are parsed to determine basic data such as the timestamp and 485 | # thread id. Internally the LogParser maintains a hash of {TraceParser} 486 | # instances keyed by thread id that parse log lines into {Trace} instances. 487 | # Completed Trace instances are exposed via the {#traces} method. 488 | # 489 | # @see TraceParser 490 | # @see Trace 491 | class LogParser 492 | # String which identifies log lines containing profiling data 493 | PROFILE_TAG = 'PROFILE'.freeze 494 | 495 | # Regex for parsing ISO 8601 datestamps 496 | # 497 | # A copy of the regex used by Ruby's Time.iso8601 with extensions to 498 | # allow for a space as a separator beteeen date and time segments and a 499 | # comma as a separator between seconds and sub-seconds. 500 | # 501 | # @see https://ruby-doc.org/stdlib-2.4.3/libdoc/time/rdoc/Time.html#method-i-xmlschema 502 | # 503 | # @return [Regex] 504 | ISO_8601 = /(?:\d+)-(?:\d\d)-(?:\d\d) 505 | [T\s] 506 | (?:\d\d):(?:\d\d):(?:\d\d) 507 | (?:[\.,]\d+)? 508 | (?:Z|[+-]\d\d:\d\d)?/ix 509 | 510 | # Regex for parsing Puppet Server logs 511 | # 512 | # Matches log lines that use the default logback pattern for Puppet Server: 513 | # 514 | # %d %-5p [%t] [%c{2}] %m%n 515 | # 516 | # The parser also consumes a leading "Puppet PROFILE [XXXX]" or 517 | # "PROFILE [XXXX]" string where "XXXX" is an id assigned to each 518 | # profiling operation. HTTP requests are assigned a unique per-request id. 519 | # 520 | # @return [Regex] 521 | DEFAULT_PARSER = /^\s* 522 | (?#{ISO_8601})\s+ 523 | (?[A-Z]+)\s+ 524 | \[(?\S+)\]\s+ 525 | \[(?\S+)\]\s+ 526 | (?:Puppet\s+)?PROFILE\s+\[(?[^\]]+)\]\s+ 527 | (?.*)$/x 528 | 529 | # List of completed Trace instances 530 | # 531 | # @return [Array] 532 | attr_reader :traces 533 | 534 | # Convert Regex MatchData to a hash of captures 535 | # 536 | # This function converts MatchData from a Regex to a hash of named 537 | # captures and yeilds each pair to an option block for transformation. 538 | # The function assumes that every capture in the Regex is named. 539 | # 540 | # @yieldparam k [String] Name of the regex capture group. 541 | # @yieldparam v [String] Value of the regex capture group. 542 | # @yieldreturn [nil, Object] An object representing the match data 543 | # transformed to some value. A return value of `nil` will cause the 544 | # original value to be used unmodified. 545 | # 546 | # @return [Hash{String => Object}] a hash mapping the capture names to 547 | # transformed values. 548 | def self.convert_match(match_data) 549 | # NOTE: The zip can be replaced with match.named_captures, which 550 | # was added in Ruby 2.4. 551 | match_pairs = match_data.names.zip(match_data.captures).map do |k, v| 552 | new_v = yield(k, v) if block_given? 553 | v = new_v.nil? ? v : new_v 554 | 555 | [k, v] 556 | end 557 | 558 | Hash[match_pairs] 559 | end 560 | 561 | def initialize 562 | @traces = [] 563 | @trace_parsers = Hash.new {|h,k| h[k] = TraceParser.new } 564 | 565 | # TODO: Could be configurable. Would be a lot of work to implement 566 | # a reasonable intersection of Java and Passenger formats. 567 | @log_parser = DEFAULT_PARSER 568 | end 569 | 570 | # Parse traces from a logfile 571 | # 572 | # @param file [String, IO] A String instance giving a path to the logfile. 573 | # Paths ending in `.gz` will be read using a `Zlib::GzipReader`. 574 | # An `IO` object that returns Puppet Server log lines may also be passed. 575 | # The `close` method will be called on the passed `IO` instance when 576 | # parsing completes. 577 | # 578 | # @return [void] 579 | def parse_file(file) 580 | io = if file.is_a?(IO) 581 | file 582 | else 583 | case File.extname(file) 584 | when '.gz' 585 | Zlib::GzipReader.open(file) 586 | else 587 | File.open(file, 'r') 588 | end 589 | end 590 | 591 | begin 592 | io.each_line do |line| 593 | next unless line.match(PROFILE_TAG) 594 | 595 | parse_line(line) 596 | end 597 | ensure 598 | io.close 599 | end 600 | end 601 | 602 | # Parse a single log line 603 | # 604 | # @param log_line [String] 605 | # 606 | # @return [void] 607 | def parse_line(log_line) 608 | match = @log_parser.match(log_line) 609 | 610 | if match.nil? 611 | $stderr.puts("WARN Could not parse log line: #{log_line})") 612 | return 613 | end 614 | 615 | data = LogParser.convert_match(match) do |k, v| 616 | if k == "timestamp" 617 | # Ruby only allows the ISO 8601 profile defined by RFC 3339. 618 | # The Java %d format prints something that Ruby won't accept. 619 | Time.iso8601(v.sub(' ', 'T').sub(',','.')) 620 | end 621 | end 622 | message = data.delete('message') 623 | 624 | trace_parser = @trace_parsers[data['thread_id']] 625 | result = trace_parser.parse(message, data) 626 | 627 | # The TraceParser returns nil unless the log lines parsed so far 628 | # add up to a complete profile. 629 | traces << result unless result.nil? 630 | end 631 | end 632 | 633 | # Base class for output formats 634 | # 635 | # Subclasses of Formatter render lists of {Trace} instances to particular 636 | # output format and then write them to an IO instance. 637 | # 638 | # @see Trace 639 | # 640 | # @abstract 641 | class Formatter 642 | # Create a new formatter instance 643 | # 644 | # @param output [IO] An IO instance to which formatted data will be written 645 | # during a call to {#write}. 646 | def initialize(output) 647 | end 648 | 649 | # Format a list of traces and write to the wrapped output 650 | # 651 | # @param traces [Trace] 652 | # 653 | # @return [void] 654 | def write(traces) 655 | raise NotImplementedError, "#{self.class.name} is an abstract class." 656 | end 657 | 658 | # Format traces as CSV rows 659 | # 660 | # This Formatter loops over each trace and writes a row of data in CSV 661 | # format for each span. 662 | # 663 | # @see file:README.md#label-CSV 664 | # More details in README 665 | class Csv < Formatter 666 | # (see Formatter#initialize) 667 | def initialize(output) 668 | @output = CSV.new(output) 669 | @header_written = false 670 | end 671 | 672 | # (see Formatter#write) 673 | def write(traces) 674 | traces.each do |trace| 675 | trace.each do |span| 676 | data = convert_span(span.object, span) 677 | 678 | unless @header_written 679 | @output << data.keys 680 | @header_written = true 681 | end 682 | 683 | @output << data.values 684 | end 685 | end 686 | end 687 | 688 | private 689 | 690 | def convert_span(span, trace) 691 | # NOTE: The Puppet::Util::Profiler library prints seconds with 4 digits 692 | # of precision, so preserve that in the output. 693 | # 694 | # TODO: This outputs in ISO 8601 format, which is great but may not be 695 | # the best for programs like Excel. Look into this. 696 | {timestamp: span.start_time.iso8601(4), 697 | trace_id: span.context[:trace_id], 698 | span_id: span.context[:span_id], 699 | name: span.name, 700 | exclusive_time_ms: trace.exclusive_time, 701 | inclusive_time_ms: trace.inclusive_time} 702 | end 703 | end 704 | 705 | # Format traces as input for flamegraph.pl 706 | # 707 | # This Formatter loops over each trace and writes its spans out as a 708 | # semicolon-delimited list of operations followed by the 709 | # {Trace#exclusive_time}. This output format is suitable as input for 710 | # the FlameGraph tool which generates an interactive SVG visualization. 711 | # 712 | # @see https://github.com/brendangregg/FlameGraph 713 | # brendangregg/FlameGraph on GitHub 714 | # @see file:README.md#label-FlameGraph 715 | # More details in README 716 | class FlameGraph < Formatter 717 | # (see Formatter#initialize) 718 | def initialize(output) 719 | @output = output 720 | end 721 | 722 | # (see Formatter#write) 723 | def write(traces) 724 | traces.each do |trace| 725 | trace.each do |span| 726 | span_time = span.exclusive_time 727 | 728 | next if span_time.zero? 729 | 730 | case span.object.tags['puppet.resource_type'] 731 | when nil, 'Class' 732 | else 733 | # Aggregate resources that aren't classes. 734 | span.stack[-1] = span.object.tags['puppet.resource_type'] 735 | end 736 | 737 | # The FlameGraph script uses ; as a separator for namespace segments. 738 | span_label = span.stack.map {|l| l.gsub(';', '') }.join(';') 739 | 740 | @output.puts("#{span_label} #{span_time}") 741 | end 742 | end 743 | end 744 | end 745 | 746 | # Format traces as human-readable output 747 | # 748 | # This Formatter loops over each trace and writes its spans out as an 749 | # indented list. The traces are followed by summary tables that display 750 | # the most expensive operations, sorted by {Trace#exclusive_time}. 751 | # 752 | # @see file:README.md#label-Human+readable 753 | # More details in README 754 | class Human < Formatter 755 | ELLIPSIS = "\u2026".freeze 756 | 757 | # (see Formatter#initialize) 758 | # 759 | # @param use_color [nil, Boolean] Whether or not to colorize output 760 | # using ANSI escape codes. If set to `nil`, the default value 761 | # of {Tty.tty?} will be used. 762 | def initialize(output, use_color = nil) 763 | @output = output 764 | @use_color = if use_color.nil? 765 | output.tty? 766 | else 767 | use_color 768 | end 769 | end 770 | 771 | # (see Formatter#write) 772 | def write(traces) 773 | traces.each do |trace| 774 | trace.each do |span| 775 | indent = " " * span.namespace.length 776 | id = Tty.green(span.object.id, @use_color) 777 | time = Tty.yellow("(#{span.inclusive_time} ms)", @use_color) 778 | 779 | @output.puts(indent + [id, span.object.inspect, time].join(' ')) 780 | end 781 | 782 | @output.write("\n\n") 783 | end 784 | 785 | spans = Hash.new {|h,k| h[k] = [] } 786 | traces.each_with_object(spans) do |trace, span_map| 787 | trace.each do |span| 788 | case span.object.tags['puppet.op_type'] 789 | when 'function_call' 790 | span_map[:functions] << span 791 | when 'resource_eval' 792 | span_map[:resources] << span 793 | when 'puppetdb_call' 794 | span_map[:puppetdb] << span 795 | when 'http_request' 796 | span_map[:http_req] << span 797 | else 798 | span_map[:other] << span 799 | end 800 | end 801 | end 802 | 803 | process_group("Function calls", spans[:functions]) 804 | process_group("Resource evaluations", spans[:resources]) 805 | process_group("PuppetDB operations", spans[:puppetdb]) 806 | process_group("HTTP Requests", spans[:http_req]) 807 | process_group("Other evaluations", spans[:other]) 808 | end 809 | 810 | private 811 | 812 | def truncate(str, width) 813 | if (str.length <= width) 814 | str 815 | else 816 | str[0..(width-2)] + ELLIPSIS 817 | end 818 | end 819 | 820 | def process_group(title, spans) 821 | total = 0 822 | itemized_totals = Hash.new { |h, k| h[k] = 0 } 823 | 824 | spans.each do |span| 825 | total += span.exclusive_time 826 | span_key = case span.object.tags['puppet.resource_type'] 827 | when nil, 'Class' 828 | span.object.name 829 | else 830 | # Aggregate resources that aren't classes. 831 | span.object.tags['puppet.resource_type'] 832 | end 833 | 834 | itemized_totals[span_key] += span.exclusive_time 835 | end 836 | 837 | rows = itemized_totals.to_a.sort { |a, b| b[1] <=> a[1] } 838 | 839 | @output.puts "\n--- #{title} ---" 840 | @output.puts "Total time: #{total} ms" 841 | @output.puts "Itemized:" 842 | 843 | # NOTE: Table formatting fixed to 72 columns. Adjusting this based on 844 | # screen size is possible, but not worth the complexity at this time. 845 | @output.printf("%-50s | %-19s\n", 'Source', 'Time') 846 | @output.puts(('-' * 50) + '-+-' + ('-' * 19)) 847 | rows.each do |k, v| 848 | next if v.zero? 849 | 850 | @output.printf("%-50s | %i ms\n", truncate(k, 50), v) 851 | end 852 | end 853 | end 854 | 855 | # Format traces as Zipkin JSON 856 | # 857 | # This Formatter loops over each trace and writes its spans out as JSON 858 | # data formatted according to the `ListOfSpans` datatype accepted by the 859 | # Zipkin v2 API. 860 | # 861 | # @see https://zipkin.io/zipkin-api/ 862 | # Zipkin v2 API specification 863 | # @see file:README.md#label-Zipkin+JSON 864 | # More details in README 865 | class Zipkin < Formatter 866 | # (see Formatter#initialize) 867 | def initialize(output) 868 | @output = output 869 | end 870 | 871 | # (see Formatter#write) 872 | def write(traces) 873 | first_loop = true 874 | @output.write('[') 875 | 876 | traces.each do |trace| 877 | trace.each do |span| 878 | next unless (span.inclusive_time > 0) 879 | 880 | if first_loop 881 | first_loop = false 882 | else 883 | @output.write(',') 884 | end 885 | 886 | @output.write(convert_span(span.object).to_json) 887 | end 888 | end 889 | 890 | @output.write(']') 891 | end 892 | 893 | private 894 | 895 | def convert_span(span) 896 | # Zipkin requires 16 -- 32 hex characters for trace IDs. We can get that 897 | # by removing the dashes from a UUID. 898 | trace_id = span.context[:trace_id].gsub('-', '') 899 | # And exactly 16 hex characters for span and parent IDs. 900 | span_id = Digest::SHA2.hexdigest(span.context[:span_id])[0..15] 901 | 902 | result = {"traceId" => trace_id, 903 | "id" => span_id, 904 | "name" => span.name, 905 | "kind" => "SERVER", 906 | "localEndpoint" => {"serviceName" => "puppetserver"}} 907 | 908 | if (parent = span.references.find {|r| r.first == "child_of"}) 909 | result["parentId"] = Digest::SHA2.hexdigest(parent.last)[0..15] 910 | end 911 | 912 | # Zipkin reports durations in microseconds and timestamps in microseconds 913 | # since the UNIX epoch. 914 | # 915 | # NOTE: Time#to_i truncates to the nearest second. Using to_f is required 916 | # for sub-second precision. 917 | unless span.start_time.nil? 918 | result["timestamp"] = Integer(span.start_time.to_f * 10**6) 919 | end 920 | result["duration"] = Integer(span.time * 10**6) 921 | 922 | unless span.tags.empty? 923 | result["tags"] = span.tags.dup 924 | result["tags"].delete("span.kind") # Set to SERVER above. 925 | end 926 | 927 | result 928 | end 929 | end 930 | end 931 | 932 | # Manage CLI execution 933 | # 934 | # This class provides logic for command line execution of the profile parser. 935 | # The {#initialize} method handles parsing ARGV for configuration and the 936 | # {#run} method parses the files and generates an exit code. 937 | class CLI 938 | def initialize(argv = []) 939 | @log_files = [] 940 | @outputter = nil 941 | @options = {color: $stdout.tty? } 942 | @action = :parse_logs 943 | 944 | @optparser = OptionParser.new do |parser| 945 | parser.banner = "Usage: puppet-profile-parser [options] puppetserver.log [...]" 946 | 947 | parser.on('-f', '--format FORMAT', String, 948 | 'Output format to use. One of:', 949 | ' human (default)', 950 | ' csv', 951 | ' flamegraph', 952 | ' zipkin') do |format| 953 | @options[:format] = case format 954 | when 'csv', 'human', 'flamegraph', 'zipkin' 955 | format.intern 956 | else 957 | raise ArgumentError, "#{format} is not a supported output format. See --help for details." 958 | end 959 | end 960 | 961 | parser.on('--[no-]color', 'Colorize output.', 962 | 'Defaults to true if run from an interactive POSIX shell.') do |v| 963 | @options[:color] = v 964 | end 965 | 966 | parser.on_tail('-h', '--help', 'Show help') do 967 | @action = :show_help 968 | end 969 | 970 | parser.on_tail('--debug', 'Enable backtraces from errors.') do 971 | @options[:debug] = true 972 | end 973 | 974 | parser.on_tail('--version', 'Show version') do 975 | @action = :show_version 976 | end 977 | end 978 | 979 | args = argv.dup 980 | @optparser.parse!(args) 981 | 982 | # parse! consumes all --flags and their arguments leaving 983 | # file names behind. 984 | @log_files += args 985 | @formatter = case @options[:format] 986 | when :csv 987 | Formatter::Csv.new($stdout) 988 | when :flamegraph 989 | Formatter::FlameGraph.new($stdout) 990 | when :zipkin 991 | Formatter::Zipkin.new($stdout) 992 | else 993 | Formatter::Human.new($stdout, @options[:color]) 994 | end 995 | end 996 | 997 | # Parse files and print results to STDERR 998 | # 999 | # @return [Integer] An integer representing process exit code that can be 1000 | # set by the caller. 1001 | def run 1002 | case @action 1003 | when :show_help 1004 | $stdout.puts(@optparser.help) 1005 | return 0 1006 | when :show_version 1007 | $stdout.puts(VERSION) 1008 | return 0 1009 | end 1010 | 1011 | if not REQUIRED_RUBY_VERSION.satisfied_by?(Gem::Version.new(RUBY_VERSION)) 1012 | $stderr.puts("puppet-profile-parser requires Ruby #{REQUIRED_RUBY_VERSION}") 1013 | return 1 1014 | elsif @log_files.empty? 1015 | $stderr.puts(@optparser.help) 1016 | return 1 1017 | end 1018 | 1019 | parser = LogParser.new 1020 | 1021 | @log_files.each {|f| parser.parse_file(f)} 1022 | 1023 | @formatter.write(parser.traces) 1024 | 1025 | return 0 1026 | rescue => e 1027 | message = if @options[:debug] 1028 | ["ERROR #{e.class}: #{e.message}", 1029 | e.backtrace].join("\n\t") 1030 | else 1031 | "ERROR #{e.class}: #{e.message}" 1032 | end 1033 | 1034 | $stderr.puts(message) 1035 | return 1 1036 | end 1037 | end 1038 | end 1039 | 1040 | 1041 | if File.expand_path(__FILE__) == File.expand_path($PROGRAM_NAME) 1042 | exit_status = PuppetProfileParser::CLI.new(ARGV).run 1043 | exit exit_status 1044 | end 1045 | -------------------------------------------------------------------------------- /spec/fixtures/puppetserver.log: -------------------------------------------------------------------------------- 1 | # Interleaved profiles from two threads 2 | 2018-02-25 18:31:48,735 INFO [123] [puppetserver] Puppet PROFILE [30880] 1.1 Rendered result in Puppet::Network::Format[json]: took 0.0030 seconds 3 | 2018-02-25 18:31:52,260 INFO [456] [puppetserver] Puppet PROFILE [30900] 1.1.1 Called template: took 0.25 seconds 4 | 2018-02-25 18:31:50,946 INFO [456] [puppetserver] Puppet PROFILE [30900] 1.1 Evaluated resource Class[Puppet_enterprise::Mcollective::Server::Facter]: took 0.25 seconds 5 | 2018-02-25 18:31:48,735 INFO [123] [puppetserver] Puppet PROFILE [30880] 1.2 Sent response: took 0.0000 seconds 6 | 2018-02-25 18:31:52,732 INFO [456] [puppetserver] Puppet PROFILE [30900] 1 Processed request POST /puppet/v3/catalog/pe-20181nightly-agent-replica.puppetdebug.vlan: took 1.5 seconds 7 | 2018-02-25 18:31:48,736 INFO [123] [puppetserver] Puppet PROFILE [30880] 1 Processed request GET /puppet/v3/node/pe-20181nightly-agent-replica.puppetdebug.vlan: took 0.1370 seconds 8 | 9 | # Incomplete profile (no span with id "1") 10 | 2018-02-25 18:31:54,524 INFO [123] [puppetserver] Puppet PROFILE [33676] 1.4.10.1.1.8.6 map: took 0.0030 seconds 11 | -------------------------------------------------------------------------------- /spec/fixtures/zipkin/README.md: -------------------------------------------------------------------------------- 1 | Zipkin Schema 2 | ============= 3 | 4 | Generated from the Swagger 2.0 specification `zipkin2-api.yaml`: 5 | 6 | https://github.com/openzipkin/zipkin-api 7 | 8 | Using `openapi2jsonschema`: 9 | 10 | https://github.com/garethr/openapi2jsonschema 11 | 12 | openapi2jsonschema --stand-alone \ 13 | https://raw.githubusercontent.com/openzipkin/zipkin-api/master/zipkin2-api.yaml 14 | 15 | The `--stand-alone` flag causes the generated schemas to embed any schems they 16 | reference. However, a couple tweaks have to be made to the resulting 17 | `listofspans.json` schema to make it usable by Ruby's `json-schma` libarary: 18 | 19 | - The top level `type` has to be changed from `object` to `array`. This seems 20 | like a bug in the convesion tool. 21 | 22 | - The top level `$schema` reference to `http://json-schema.org/schema#` has 23 | to be removed. This is probably fine, we can assume what was generated is 24 | valid. 25 | -------------------------------------------------------------------------------- /spec/fixtures/zipkin/listofspans.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": { 3 | "required": [ 4 | "traceId", 5 | "id" 6 | ], 7 | "type": "object", 8 | "properties": { 9 | "kind": { 10 | "enum": [ 11 | "CLIENT", 12 | "SERVER", 13 | "PRODUCER", 14 | "CONSUMER" 15 | ], 16 | "type": "string", 17 | "description": "When present, clarifies timestamp, duration and remoteEndpoint. When\nabsent, the span is local or incomplete. Unlike client and server,\nthere is no direct critical path latency relationship between producer\nand consumer spans.\n\n* `CLIENT`\n * timestamp - The moment a request was sent (formerly \"cs\")\n * duration - When present indicates when a response was received (formerly \"cr\")\n * remoteEndpoint - Represents the server. Leave serviceName absent if unknown.\n* `SERVER`\n * timestamp - The moment a request was received (formerly \"sr\")\n * duration - When present indicates when a response was sent (formerly \"ss\")\n * remoteEndpoint - Represents the client. Leave serviceName absent if unknown.\n* `PRODUCER`\n * timestamp - The moment a message was sent to a destination (formerly \"ms\")\n * duration - When present represents delay sending the message, such as batching.\n * remoteEndpoint - Represents the broker. Leave serviceName absent if unknown.\n* `CONSUMER`\n * timestamp - The moment a message was received from an origin (formerly \"mr\")\n * duration - When present represents delay consuming the message, such as from backlog.\n * remoteEndpoint - Represents the broker. Leave serviceName absent if unknown.\n" 18 | }, 19 | "traceId": { 20 | "pattern": "[a-z0-9]{16,32}", 21 | "maxLength": 32, 22 | "type": "string", 23 | "description": "Randomly generated, unique identifier for a trace, set on all spans within it.\n\nEncoded as 16 or 32 lowercase hex characters corresponding to 64 or 128 bits.\nFor example, a 128bit trace ID looks like 4e441824ec2b6a44ffdc9bb9a6453df3\n", 24 | "minLength": 16 25 | }, 26 | "name": { 27 | "type": "string", 28 | "description": "The logical operation this span represents in lowercase (e.g. rpc method).\nLeave absent if unknown.\n\nAs these are lookup labels, take care to ensure names are low cardinality.\nFor example, do not embed variables into the name.\n" 29 | }, 30 | "localEndpoint": { 31 | "title": "Endpoint", 32 | "type": "object", 33 | "description": "The network context of a node in the service graph", 34 | "properties": { 35 | "ipv6": { 36 | "type": "string", 37 | "description": "The text representation of the primary IPv6 address associated with this\na connection. Ex. 2001:db8::c001 Absent if unknown.\n\nPrefer using the ipv4 field for mapped addresses.\n", 38 | "format": "ipv6" 39 | }, 40 | "serviceName": { 41 | "type": "string", 42 | "description": "Lower-case label of this node in the service graph, such as \"favstar\". Leave\nabsent if unknown.\n\nThis is a primary label for trace lookup and aggregation, so it should be\nintuitive and consistent. Many use a name from service discovery.\n" 43 | }, 44 | "ipv4": { 45 | "type": "string", 46 | "description": "The text representation of the primary IPv4 address associated with this\na connection. Ex. 192.168.99.100 Absent if unknown.\n", 47 | "format": "ipv4" 48 | }, 49 | "port": { 50 | "type": "integer", 51 | "description": "Depending on context, this could be a listen port or the client-side of a\nsocket. Absent if unknown\n" 52 | } 53 | } 54 | }, 55 | "timestamp": { 56 | "type": "integer", 57 | "description": "Epoch **microseconds** of the start of this span, possibly absent if incomplete.\n\nFor example, 1502787600000000 corresponds to 2017-08-15 09:00 UTC\n\nThis value should be set directly by instrumentation, using the most precise\nvalue possible. For example, gettimeofday or multiplying epoch millis by 1000.\n\nThere are three known edge-cases where this could be reported absent.\n * A span was allocated but never started (ex not yet received a timestamp)\n * The span's start event was lost\n * Data about a completed span (ex tags) were sent after the fact\n", 58 | "format": "int64" 59 | }, 60 | "remoteEndpoint": { 61 | "title": "Endpoint", 62 | "type": "object", 63 | "description": "The network context of a node in the service graph", 64 | "properties": { 65 | "ipv6": { 66 | "type": "string", 67 | "description": "The text representation of the primary IPv6 address associated with this\na connection. Ex. 2001:db8::c001 Absent if unknown.\n\nPrefer using the ipv4 field for mapped addresses.\n", 68 | "format": "ipv6" 69 | }, 70 | "serviceName": { 71 | "type": "string", 72 | "description": "Lower-case label of this node in the service graph, such as \"favstar\". Leave\nabsent if unknown.\n\nThis is a primary label for trace lookup and aggregation, so it should be\nintuitive and consistent. Many use a name from service discovery.\n" 73 | }, 74 | "ipv4": { 75 | "type": "string", 76 | "description": "The text representation of the primary IPv4 address associated with this\na connection. Ex. 192.168.99.100 Absent if unknown.\n", 77 | "format": "ipv4" 78 | }, 79 | "port": { 80 | "type": "integer", 81 | "description": "Depending on context, this could be a listen port or the client-side of a\nsocket. Absent if unknown\n" 82 | } 83 | } 84 | }, 85 | "id": { 86 | "pattern": "[a-z0-9]{16}", 87 | "maxLength": 16, 88 | "type": "string", 89 | "description": "Unique 64bit identifier for this operation within the trace.\n\nEncoded as 16 lowercase hex characters. For example ffdc9bb9a6453df3\n", 90 | "minLength": 16 91 | }, 92 | "duration": { 93 | "minimum": 1, 94 | "type": "integer", 95 | "description": "Duration in **microseconds** of the critical path, if known. Durations of less\nthan one are rounded up. Duration of children can be longer than their parents\ndue to asynchronous operations.\n\nFor example 150 milliseconds is 150000 microseconds.\n", 96 | "format": "int64" 97 | }, 98 | "parentId": { 99 | "pattern": "[a-z0-9]{16}", 100 | "maxLength": 16, 101 | "type": "string", 102 | "description": "The parent span ID or absent if this the root span in a trace.", 103 | "minLength": 16 104 | }, 105 | "debug": { 106 | "type": "boolean", 107 | "description": "True is a request to store this span even if it overrides sampling policy.\n\nThis is true when the `X-B3-Flags` header has a value of 1.\n" 108 | }, 109 | "shared": { 110 | "type": "boolean", 111 | "description": "True if we are contributing to a span started by another tracer (ex on a different host)." 112 | }, 113 | "annotations": { 114 | "uniqueItems": true, 115 | "items": { 116 | "title": "Annotation", 117 | "type": "object", 118 | "description": "Associates an event that explains latency with a timestamp.\nUnlike log statements, annotations are often codes. Ex. \"ws\" for WireSend\n\nZipkin v1 core annotations such as \"cs\" and \"sr\" have been replaced with\nSpan.Kind, which interprets timestamp and duration.\n", 119 | "properties": { 120 | "timestamp": { 121 | "type": "integer", 122 | "description": "Epoch **microseconds** of this event.\n\nFor example, 1502787600000000 corresponds to 2017-08-15 09:00 UTC\n\nThis value should be set directly by instrumentation, using the most precise\nvalue possible. For example, gettimeofday or multiplying epoch millis by 1000.\n" 123 | }, 124 | "value": { 125 | "type": "string", 126 | "description": "Usually a short tag indicating an event, like \"error\"\n\nWhile possible to add larger data, such as garbage collection details, low\ncardinality event names both keep the size of spans down and also are easy\nto search against.\n" 127 | } 128 | } 129 | }, 130 | "type": "array", 131 | "description": "Associates events that explain latency with the time they happened." 132 | }, 133 | "tags": { 134 | "additionalProperties": { 135 | "type": "string" 136 | }, 137 | "type": "object", 138 | "description": "Adds context to a span, for search, viewing and analysis.\n\nFor example, a key \"your_app.version\" would let you lookup traces by version.\nA tag \"sql.query\" isn't searchable, but it can help in debugging when viewing\na trace.\n", 139 | "title": "Tags" 140 | } 141 | }, 142 | "title": "Span" 143 | }, 144 | "type": "array", 145 | "description": "A list of spans with possibly different trace ids, in no particular order", 146 | "title": "ListOfSpans" 147 | } 148 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | PROJECT_ROOT = File.expand_path('..', File.dirname(__FILE__)).freeze 2 | SPEC_ROOT = File.expand_path(File.dirname(__FILE__)).freeze 3 | 4 | require 'json-schema' 5 | 6 | module TestHelpers 7 | def fixture(name) 8 | File.join(SPEC_ROOT, 'fixtures', name) 9 | end 10 | end 11 | 12 | RSpec.configure do |config| 13 | config.include TestHelpers 14 | end 15 | -------------------------------------------------------------------------------- /spec/unit/profile_parser_cli_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require "#{PROJECT_ROOT}/puppet-profile-parser.rb" 4 | 5 | describe PuppetProfileParser::CLI do 6 | # Caputure output from tests. 7 | # TODO: Add better output control to the CLI. 8 | original_stdout = $stdout 9 | let(:output) { StringIO.new } 10 | before(:each) { $stdout = output } 11 | after(:each) { $stdout = original_stdout } 12 | 13 | it 'uses file extensions to determine which IO class to use' do 14 | cli = described_class.new(['foo.log', 'bar.log.gz', 'baz']) 15 | 16 | expect(File).to receive(:open).with('foo.log', any_args).and_return(StringIO.new) 17 | expect(File).to receive(:open).with('baz', any_args).and_return(StringIO.new) 18 | expect(Zlib::GzipReader).to receive(:open).with('bar.log.gz').and_return(StringIO.new) 19 | 20 | cli.run 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/unit/profile_parser_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require "#{PROJECT_ROOT}/puppet-profile-parser.rb" 4 | 5 | describe PuppetProfileParser::LogParser do 6 | describe 'ISO 8601 timestamp parser' do 7 | subject { PuppetProfileParser::LogParser::ISO_8601 } 8 | 9 | it 'matches timestamps produced by the logback %date pattern' do 10 | expect(subject.match('2017-09-18 12:01:43,344')).to be_a(MatchData) 11 | end 12 | end 13 | 14 | describe 'Puppet Server default logback layout parser' do 15 | subject { PuppetProfileParser::LogParser::DEFAULT_PARSER } 16 | 17 | it 'matches PROFILE output formatted with the layout' do 18 | result = subject.match("2018-02-18 18:43:53,501 INFO [qtp1732817189-1224] [puppetserver] Puppet PROFILE [39776666] 1 Processed request GET /puppet/v3/node/pe-201734-master.puppetdebug.vlan: took 0.1640 seconds\n") 19 | 20 | expect(result).to be_a(MatchData) 21 | expect(result[:timestamp]).to eq('2018-02-18 18:43:53,501') 22 | expect(result[:log_level]).to eq('INFO') 23 | expect(result[:thread_id]).to eq('qtp1732817189-1224') 24 | expect(result[:java_class]).to eq('puppetserver') 25 | expect(result[:request_id]).to eq('39776666') 26 | expect(result[:message]).to eq('1 Processed request GET /puppet/v3/node/pe-201734-master.puppetdebug.vlan: took 0.1640 seconds') 27 | end 28 | end 29 | 30 | describe 'when parsing a logfile' do 31 | subject { described_class.new } 32 | let(:log_file) { fixture('puppetserver.log') } 33 | 34 | before(:each) { subject.parse_file(log_file) } 35 | 36 | it 'creates a trace for each complete PROFILE with unique ids' do 37 | expect(subject.traces.flat_map {|t| t.map(&:trace_id)}.uniq.length).to eq(2) 38 | end 39 | 40 | it 'computes inclusive and exclusive time for each trace' do 41 | expect(subject.traces.first.inclusive_time).to eq(1500) 42 | expect(subject.traces.first.exclusive_time).to eq(1250) 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/unit/profile_parser_zipkin_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'stringio' 3 | 4 | require "#{PROJECT_ROOT}/puppet-profile-parser.rb" 5 | 6 | describe PuppetProfileParser::Formatter::Zipkin do 7 | subject { described_class.new(output) } 8 | 9 | let(:output) { StringIO.new } 10 | let(:parser) { PuppetProfileParser::LogParser.new } 11 | let(:log_file) { fixture('puppetserver.log') } 12 | 13 | before(:each) do 14 | parser.parse_file(log_file) 15 | subject.write(parser.traces) 16 | end 17 | 18 | it 'creates valid JSON' do 19 | expect{ JSON.parse(output.string) }.to_not raise_error 20 | end 21 | 22 | it 'creates JSON that conforms to a Zipkin APIv2 ListOfSpans' do 23 | result = JSON.parse(output.string) 24 | schema = JSON.parse(File.read(fixture('zipkin/listofspans.json'))) 25 | 26 | validation_result = JSON::Validator.fully_validate(schema, result) 27 | 28 | expect(validation_result).to eq([]) 29 | end 30 | end 31 | --------------------------------------------------------------------------------