├── .chglog ├── CHANGELOG.tpl.md └── config.yml ├── .circleci └── config.yml ├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── cli └── cli.go ├── commontest └── utility_test.go ├── constants └── constants.go ├── eventbreaker ├── eventbreaker.go └── eventbreaker_test.go ├── go.mod ├── go.sum ├── loadgenerator ├── generator.go ├── generator_test.go ├── loadgen.go └── loadgen_test.go ├── logo.png ├── main.go ├── metrics ├── metrics.go └── server.go └── util └── utility.go /.chglog/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ range .Versions }} 2 | 3 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }}) 4 | 5 | {{ range .CommitGroups -}} 6 | ### {{ .Title }} 7 | 8 | {{ range .Commits -}} 9 | * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 10 | {{ end }} 11 | {{ end -}} 12 | 13 | {{- if .RevertCommits -}} 14 | ### Reverts 15 | 16 | {{ range .RevertCommits -}} 17 | * {{ .Revert.Header }} 18 | {{ end }} 19 | {{ end -}} 20 | 21 | {{- if .MergeCommits -}} 22 | ### Pull Requests 23 | 24 | {{ range .MergeCommits -}} 25 | * {{ .Header }} 26 | {{ end }} 27 | {{ end -}} 28 | 29 | {{- if .NoteGroups -}} 30 | {{ range .NoteGroups -}} 31 | ### {{ .Title }} 32 | 33 | {{ range .Notes }} 34 | {{ .Body }} 35 | {{ end }} 36 | {{ end -}} 37 | {{ end -}} 38 | {{ end -}} -------------------------------------------------------------------------------- /.chglog/config.yml: -------------------------------------------------------------------------------- 1 | style: github 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: y 6 | options: 7 | commits: 8 | # filters: 9 | # Type: 10 | # - feat 11 | # - fix 12 | # - perf 13 | # - refactor 14 | commit_groups: 15 | # title_maps: 16 | # feat: Features 17 | # fix: Bug Fixes 18 | # perf: Performance Improvements 19 | # refactor: Code Refactoring 20 | header: 21 | pattern: "^(\\w*)(?:\$([\\w\\$\\.\\-\\*\\s]*)\$)?\\:\\s(.*)$" 22 | pattern_maps: 23 | - Type 24 | - Scope 25 | - Subject 26 | notes: 27 | keywords: 28 | - BREAKING CHANGE -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Golang CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-go/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | - image: circleci/golang:1.12 9 | environment: 10 | GO111MODULE: "on" 11 | 12 | working_directory: /go/src/github.com/intuit/go-loadgen 13 | steps: 14 | - checkout 15 | - run: go get -v -t ./... 16 | - run: go test ./... 17 | - run: go build main.go 18 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | Open source projects are “living.” Contributions in the form of issues and pull requests are welcomed and encouraged. When you contribute, you explicitly say you are part of the community and abide by its Code of Conduct. 2 | 3 | # The Code 4 | 5 | At Intuit, we foster a kind, respectful, harassment-free cooperative community. Our open source community works to: 6 | 7 | - Be kind and respectful; 8 | - Act as a global community; 9 | - Conduct ourselves professionally. 10 | 11 | As members of this community, we will not tolerate behaviors including, but not limited to: 12 | 13 | - Violent threats or language; 14 | - Discriminatory or derogatory jokes or language; 15 | - Public or private harassment of any kind; 16 | - Other conduct considered inappropriate in a professional setting. 17 | 18 | ## Reporting Concerns 19 | 20 | If you see someone violating the Code of Conduct please email TechOpenSource@intuit.com 21 | 22 | ## Scope 23 | 24 | This code of conduct applies to: 25 | 26 | All repos and communities for Intuit-managed projects, whether or not the text is included in a Intuit-managed project’s repository; 27 | 28 | Individuals or teams representing projects in official capacity, such as via official social media channels or at in-person meetups. 29 | 30 | ## Attribution 31 | 32 | This Code of Conduct is partly inspired by and based on those of Amazon, CocoaPods, GitHub, Microsoft, thoughtbot, and on the Contributor Covenant version 1.4.1. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | --- 8 | 9 | **Describe the bug** 10 | 11 | 12 | 13 | **To Reproduce** 14 | 15 | 16 | 17 | **Expected behavior** 18 | 19 | 20 | 21 | **Screenshots** 22 | 23 | 24 | 25 | **Desktop (please complete the following information):** 26 | 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Additional context** 32 | 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 12 | 13 | 14 | **Describe the solution you'd like** 15 | 16 | 17 | 18 | **Describe alternatives you've considered** 19 | 20 | 21 | 22 | **Additional context** 23 | 24 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # What Changed 2 | 3 | # Why 4 | 5 | Todo: 6 | 7 | - [ ] Add tests 8 | - [ ] Add docs 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.code-workspace 3 | *.log 4 | 5 | *.swp 6 | test-logger 7 | /.idea 8 | 1 9 | loadgen 10 | test-data/ 11 | go-loadgen 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.3.x 5 | 6 | before_install: 7 | - go get -t -v ./... 8 | 9 | script: 10 | - go test ./.. -coverprofile=coverage.txt 11 | 12 | after_success: 13 | - bash <(curl -s https://codecov.io/bash) 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## [v1.0.2](y/compare/v1.0.1...v1.0.2) (2020-03-03) 4 | 5 | 6 | 7 | ## [v1.0.1](y/compare/v1.0.0...v1.0.1) (2020-03-03) 8 | 9 | 10 | 11 | ## v1.0.0 (2020-03-03) 12 | 13 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | ## Contributing 3 | 4 | 1. Fork it 5 | 2. Create your feature branch (`git checkout -b my-new-feature`) 6 | 3. Commit your changes (`git commit -am 'Add some feature'`) 7 | 4. Push to the branch (`git push origin my-new-feature`) 8 | 5. Create new Pull Request 9 | 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Now copy it into our base image. 2 | FROM golang:1.12 3 | LABEL maintainer="chaitanya_bhatt@intuit.com" 4 | RUN mkdir /app 5 | # copy everything in the root directory into our /app directory 6 | ADD . /app 7 | # execute further commands inside our /app directory 8 | WORKDIR /app 9 | 10 | RUN tar -xvzf test-data/qbo-1.log.tar --directory test-data/. 11 | 12 | RUN go build -o loadgen . 13 | 14 | CMD ["/app/loader"] 15 | 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Intuit Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | [![intuit](https://circleci.com/gh/intuit/go-loadgen.svg?style=svg)]() 3 | [![License: Apache2.0](https://img.shields.io/badge/License-Apache2.0-blue.svg)](https://opensource.org/licenses/Apache2.0) 4 | [![Go version](https://img.shields.io/github/go-mod/go-version/intuit/go-loadgen)](https://img.shields.io/github/go-mod/go-version/intuit/go-loadgen) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/intuit/go-loadgen)](https://goreportcard.com/report/github.com/intuit/go-loadgen) 6 | 7 | ## go-loadgen 8 | ![logo](logo.png) 9 | 10 | go-loadgen is a log load generator tool meant for testing the logging infrastructure. It is capable of producing structured or unstructured/structure logs with random alphanumeric strings and can also playback any input file at a specified rate. 11 | ## Build 12 | This program requires GO 1.19 13 | ``` 14 | go install github.com/intuit/go-loadgen@latest 15 | ln -s $GOPATH/bin/go-loadgen /usr/bin/loadgen 16 | ``` 17 | 18 | ### developers 19 | ``` 20 | //Build command 21 | go build -o loadgen 22 | ``` 23 | ## Usage 24 | Use the help command to find all available commands and flags. 25 | 26 | HHNLWB76D9289E:go-loadgen cbhatt1$ ./loadgen --help 27 | loadgen is a tool which generates test-data (synthetic/replay) at a controlled rate to test logging infrastructure. 28 | 29 | Usage: 30 | loadgen [COMMANDS] [flags] 31 | loadgen [command] 32 | 33 | Available Commands: 34 | help Help about any command 35 | random-strings test data is generated by producing random alphanumeric strings. 36 | replay-file test data is generated by replaying a file. 37 | 38 | Flags: 39 | --config string -config /path/to/source/jsonfile (default "cfg") 40 | --custom-timestamp-format string Optional custom timestamp format.The default format is 2006-01-02T15:04:05.000-07:00 41 | -x, --disable-timestamp use --disable-timestamp false will disable timestamp injection while replaying logs. When this flag is not used. Do not modify this parameter for auto-generated random alphanumberic logs.Disabling timestamps is typically suitable while replaying a log-file with pre-existing timestamps. 42 | -d, --duration int Duration of the job in seconds 43 | --enable-config use config file for configuration (default false) 44 | -r, --enable-log-rotation by default log rotation is off, use this flag to turn it on. Example --enable-log-rotation will turn log rotation on 45 | --enable-metrics true, enables prometheus metrics on http://:/metric HTTP endpoint for line emit rates and byte count. default is off. 46 | --file-count int target requests/sec. Note: log rotation does not work in multi file scenario. (default 1) 47 | -h, --help help for loadgen 48 | -s, --lines-per-second int target lines/second (default 1000) 49 | --log-rotation-max-size-megabytes int Rotate after N MegaBytes. example 100. (default 100) 50 | -p, --metrics-server-port string prometheus metrics server port. default is port 8080. (default "8080") 51 | -o, --output string file path of output log file ex: /var/tmp/test.log 52 | --result-log string Path of the results log where metrics such as total_lines_generated will be log. 53 | --tags string Custom hardcoded tags in the form of K=V (default "t") 54 | 55 | **Example-1:** To start a test with random-strings in the log file 56 | Most of the local flags to this command is mandatory. 57 | 58 | > The command generates 5 lines/second, with each line of length 59 | > 1KiloBytes (1000 ASCII characters) and log entry spanning not more 60 | > than 1 line (line-count) per event (no line breaks). 61 | > The command produces a log file which grows at a rate of 5 KiloBytes/second and runs for 5 seconds. The overall size of the file after the test completes will be equal to 5 KB*5 seconds = 25KB 62 | 63 | ./loadgen random-strings --file-count 1 \ 64 | --duration 5 \ 65 | --enable-log-rotation \ 66 | --line-count 1 \ 67 | --lines-per-second 5 \ 68 | --multi-line-percent 0 \ 69 | --line-length 1000 \ 70 | --output /var/log/test-alphanumeric.log 71 | 72 | 73 | **Example-2:** To start a test with random-strings in the log file with 100% of the log entries containing multi-line. 74 | 75 | > The flag --line-count and --multi-line-percent determines the number of line-breaks in the multi-line log entry. The resulting log file will be written at a byte rate of ***line-length\*line-count\*lines-per-second*** = 1000\*5\*5 = 25000 bytes/second. 100% of the log entries in the resulting log file contains line-break character at the end of each line, with each line spanning 1000 characters in length 76 | 77 | ./loadgen random-strings --file-count 1 \ 78 | --duration 5 \ 79 | --enable-log-rotation \ 80 | --line-count 5 \ 81 | --lines-per-second 5 \ 82 | --multi-line-percent 100 \ 83 | --line-length 1000 \ 84 | --output /var/log/test-alphanumeric.log 85 | 86 | **Example-3:** To output log lines to "**stdout**" instead of a log file, simply declar '--output **stdout**' parameter. Refer to the example below. 87 | Generating logs to stdout is especially useful to simulate logging performance for application pod/containers deployed in Kubernetes. 88 | 89 | ./loadgen random-strings --file-count 1 \ 90 | --duration 5 \ 91 | --line-count 1 \ 92 | --lines-per-second 5 \ 93 | --multi-line-percent 0 \ 94 | --line-length 1000 \ 95 | --output stdout 96 | ## File Replay/File Playback 97 | To start a log replay test with a specified input file. 98 | > Note: Local flags such as duration and lines-per-second is required. 99 | 100 | ./loadgen replay-file \ 101 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 102 | --output test.log\ 103 | --lines-per-second 1 \ 104 | --duration 5 105 | 106 | You can also specify the number of times a file should be replayed using the `--replay-count` flag. 107 | ### Disable timestamp injection for replay tests 108 | You can disable timestamp injection if your replay reference log files already have timestamp using `--disable-timestamp` boolean flag 109 | > If you use the `--replay-count` flag, then the flag `--duration` cannot be used, since the 110 | > former flag becomes the test termination criteria, and you cannot simultaneously use two different termination criteria for the test. 111 | 112 | In the below example, the test will complete after replaying the file "5" times. 113 | 114 | Example: 115 | 116 | ./loadgen replay-file \ 117 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 118 | --output test.log\ 119 | --lines-per-second 1 \ 120 | --replay-count 5 121 | 122 | 123 | ## Customized fields using tags 124 | go-loadgen can introduce custom fields using the tags flag. Provide tags in the form of `K=V` and those tags will be included in the log event. This particularly useful when metadata such as testId and/or line count should be present in the logs for backend validation. 125 | 126 | ./loadgen replay-file \ 127 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 128 | --output test.log\ 129 | --enable-log-rotation \ 130 | --log-rotation-max-size-megabytes 10 \ 131 | --lines-per-second 1 \ 132 | --file-count 3 \ 133 | --duration 5 \ 134 | --tags "lineCount=1 testId=1" 135 | 136 | ## Defining a test results log file 137 | go-loadgen can generate test results log file in the specified path. `--result-log` is the persistent flag which enables that. Use it as shown below. 138 | It generates a file with "total_lines_generated=x" entry, which can be used as a way to assert whether the generated load is equal to the load received on the backend. 139 | 140 | ./loadgen replay-file \ 141 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 142 | --output test.log\ 143 | --enable-log-rotation \ 144 | --log-rotation-max-size-megabytes 10 \ 145 | --lines-per-second 1 \ 146 | --file-count 1 \ 147 | --duration 5 \ 148 | --result-log /var/tmp/result.log" 149 | 150 | 151 | ## Log rotation and file count 152 | For certain tests, it is critical to generate multiple log files along with log-rotation and backup file retention policies. Here is how you can achieve that using go-loadgen: 153 | 154 | ./loadgen replay-file \ 155 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 156 | --output test.log\ 157 | --enable-log-rotation \ 158 | --log-rotation-max-size-megabytes 10 \ 159 | --lines-per-second 1 \ 160 | --file-count 3 \ 161 | --duration 5 162 | 163 | In the above CLI command : 164 | 165 | - The argument `--enable-log-rotation` enables log-rotation 166 | - The argument pair `--rotation-max-file-size-megabytes 10` specifies how 167 | large a file can grow before the file is rotated with a new one. In 168 | this example the limit is 10MB. 169 | - The argument pair `--file-count 3` determines how many log files the test has to create and update simultenously. 170 | 171 | go-loadgen currently allows for a maximum of 3 backup files after which the oldest file will be deleted. 172 | 173 | Log rotation default settings 174 | 175 | | Property| Default | Comments | 176 | |--|:--:|:--| 177 | | MaxFileSize | 500MB| Maximum size the file is allowed to grow| 178 | | MaxAge | 1 Day|After MaxAge exceeds the file is rotated automatically even if size is within the max limit.| 179 | | MaxBackups | 3| number of rotated files.| 180 | | File Compression | Disabled| | 181 | 182 | 183 | ## Common options 184 | 185 | - Customizing log timestamp 186 | Use `2006-01-02T15:04:05.000-07:00` with the --custom-timestamp-format flag to provide custom timestamp format. 187 | Example: 188 | 189 | ``` 190 | ./loadgen ... --custom-timestamp-format 2006/01/02T15-04-05 ... 191 | ``` 192 | The above custom timestamp will override the default timestamp in both alphanumeric random logs and replay logs depending on which mode the test is executed. 193 | 194 | ## Prometheus Metrics 195 | 196 | To start a test with prometheus metrics exposed to ```http://localhost:/metrics``` endpoint, use the "enable-metrics" global flag 197 | Note: the default prometheus listener port is "8080" unless specified using the *`--metrics-server-port `* flag 198 | 199 | 200 | ./loadgen replay-file \ 201 | --input-file-path /Users/cbhatt1/git-logging/go-loadgen/test-data/sbg-pp-data.txt \ 202 | --output test.log\ 203 | --lines-per-second 1 \ 204 | --duration 5 \ 205 | --enable-metrics \ 206 | --metrics-server-port 8081 207 | 208 | Example output using curl: 209 | 210 | cbhatt1$ curl http://localhost:8081/metrics 211 | # HELP total_bytes_processed Total bytes processed 212 | # TYPE total_bytes_processed counter 213 | total_bytes_processed 2.23982e+07 214 | # HELP total_events_processed Total log events generated by the tool. 215 | 216 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/intuit/go-loadgen/constants" 7 | loadgen "github.com/intuit/go-loadgen/loadgenerator" 8 | "github.com/intuit/go-loadgen/metrics" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/spf13/cobra" 11 | "os" 12 | "sync" 13 | ) 14 | 15 | const ( 16 | lineCount = "line-count" 17 | lineLength = "line-length" 18 | fileCount = "file-count" 19 | duration = "duration" 20 | enableRotate = "enable-log-rotation" 21 | linesPerSecond = "lines-per-second" 22 | multiLinePercent = "multi-line-percent" 23 | logRotationMaxSizeMegabytes = "log-rotation-max-size-megabytes" 24 | enableMetrics = "enable-metrics" 25 | metricsServerPort = "metrics-server-port" 26 | inputFilePath = "input-file-path" 27 | replayCount = "replay-count" 28 | disableTimestamp = "disable-timestamp" 29 | customTimestampFormat = "custom-timestamp-format" 30 | output = "output" 31 | config = "config" 32 | tags = "tags" 33 | enableConfig = "enable-config" 34 | resultLog = "result-log" 35 | ) 36 | 37 | func InputValidation(props *loadgen.LoadGenProperties) { 38 | 39 | if !props.Rotate { 40 | fmt.Println("Warning. File rotation is disabled. You may run out of disk space if the file grows too large!") 41 | } 42 | if props.FileCount > constants.MaxFileCount { 43 | fmt.Printf("Too many files in --file-count parameter. This program can only support upto %d files\n", constants.MaxFileCount) 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | var cfgFile string 49 | 50 | func printInput(props *loadgen.LoadGenProperties) { 51 | fmt.Printf("Input values = ") 52 | fmt.Printf(lineCount+" = %d\n", props.NumOfLinesInMultiLineLog) 53 | fmt.Printf(lineLength+" = %d\n", props.LineLength) 54 | fmt.Printf(duration+" = %d\n", props.Duration) 55 | fmt.Printf(linesPerSecond+" = %d\n", props.Lps) 56 | fmt.Println(output + " = %s" + props.FilePath) 57 | fmt.Printf(multiLinePercent+" = %d\n", props.MultiLinePercent) 58 | fmt.Printf(disableTimestamp+" = %t\n", props.DisableTimestamp) 59 | fmt.Printf(customTimestampFormat+" = %s\n", props.CustomTimestampFormat) 60 | fmt.Printf(enableRotate+" = %t\n", props.Rotate) 61 | fmt.Printf(logRotationMaxSizeMegabytes+" = %d\n", props.RotateSizeMB) 62 | } 63 | 64 | /* 65 | Accepts input commands and flags as arguments and the OS CLI 66 | and binds to the relevant fields within the specified loadgenerator properties struct. 67 | */ 68 | 69 | func checkInsufficientArgs(cmd *cobra.Command, args []string) { 70 | if len(args) >= 1 && (args[0] == "-h" || args[0] == "--h" || args[0] == "--help") { 71 | cmd.Help() 72 | os.Exit(1) 73 | } 74 | } 75 | 76 | func Run(props *loadgen.LoadGenProperties) { 77 | 78 | var wg sync.WaitGroup 79 | wg.Add(1) 80 | props.Wg = &wg 81 | var metricsServerPortStr string 82 | var enableConfigVar bool 83 | promRegistry := prometheus.NewRegistry() 84 | 85 | rootCmd := &cobra.Command{ 86 | Use: "loadgen [COMMANDS]", 87 | Short: "loadgen is a tool which generates test-data (synthetic/replay) at a controlled rate to test logging infrastructure.", 88 | Run: func(cmd *cobra.Command, args []string) { 89 | checkInsufficientArgs(cmd, args) 90 | printInput(props) 91 | }, 92 | } 93 | replayFileCmd := &cobra.Command{ 94 | Use: "replay-file", 95 | Short: "test data is generated by replaying a file.", 96 | Args: func(cmd *cobra.Command, args []string) error { 97 | flagSet := cmd.Flags() 98 | if flagSet.Changed(inputFilePath) == true { 99 | filePath := flagSet.Lookup(inputFilePath).Value.String() 100 | fmt.Println(filePath) 101 | info, err := os.Stat(filePath) 102 | if err != nil { 103 | return errors.New("File does not exist. input-file-path = " + filePath) 104 | } 105 | if info.Size() <= 0 { 106 | return errors.New("Invalid file size. filePath = " + filePath) 107 | } 108 | } 109 | 110 | //Check if multiple test termination criteria is defined. 111 | if flagSet.Changed(duration) && flagSet.Changed(replayCount) { 112 | return errors.New("Multiple test termination criteria found. You can either define --duration or --replay-count as termination criteria, not both. ") 113 | } 114 | return nil 115 | }, 116 | Run: func(cmd *cobra.Command, args []string) { 117 | if props.EnableMetrics { 118 | go metrics.StartMetricsServer(promRegistry, &metricsServerPortStr) 119 | } 120 | go loadgen.GenerateLoadFromInputFile(promRegistry, props) 121 | printInput(props) 122 | wg.Wait() 123 | }, 124 | } 125 | randomStringsCmd := &cobra.Command{ 126 | Use: "random-strings", 127 | Short: "test data is generated by producing random alphanumeric strings.", 128 | Long: "test data is generated by producing random alphanumeric strings.", 129 | Args: func(cmd *cobra.Command, args []string) error { 130 | flagSet := cmd.Flags() 131 | if !flagSet.Changed(duration) { 132 | return errors.New("Duration is not specified. This command requires a termination criteria. Specify a duration (s) for this job. ") 133 | } 134 | return nil 135 | }, 136 | Run: func(cmd *cobra.Command, args []string) { 137 | checkInsufficientArgs(cmd, args) 138 | if props.EnableMetrics { 139 | go metrics.StartMetricsServer(promRegistry, &metricsServerPortStr) 140 | } 141 | go loadgen.GenerateAlphaNumeric(promRegistry, props) 142 | wg.Wait() 143 | }, 144 | } 145 | 146 | //Defining persistent flags for root command 147 | rootCmd.PersistentFlags().Int64VarP(&props.Duration, duration, "d", constants.DefaultDurationInSeconds, "Duration of the job in seconds") 148 | rootCmd.PersistentFlags().Int64VarP(&props.Lps, linesPerSecond, "s", constants.DefaultLinesPerSecond, "target lines/second") 149 | rootCmd.PersistentFlags().Int64VarP(&props.FileCount, fileCount, "", constants.DefaultFileCount, "target requests/sec. Note: log rotation does not work in multi file scenario.") 150 | rootCmd.PersistentFlags().StringVarP(&cfgFile, config, "", "cfg", "-config /path/to/source/jsonfile") 151 | rootCmd.PersistentFlags().BoolVarP(&enableConfigVar, enableConfig, "", false, "use config file for configuration (default false)") 152 | rootCmd.PersistentFlags().StringVarP(&metricsServerPortStr, metricsServerPort, "p", constants.DefaultMetricsServerPort, "prometheus metrics server port. default is port 8080. ") 153 | rootCmd.PersistentFlags().BoolVarP(&props.EnableMetrics, enableMetrics, "", false, "true, enables prometheus metrics on http://:/metric HTTP endpoint for line emit rates and byte count. default is off.") 154 | rootCmd.PersistentFlags().StringVarP(&props.FilePath, output, "o", "", "file path of output log file ex: /var/tmp/test.log") 155 | rootCmd.PersistentFlags().BoolVarP(&props.Rotate, enableRotate, "r", false, "by default log rotation is off, use this flag to turn it on. Example --enable-log-rotation will turn log rotation on") 156 | rootCmd.PersistentFlags().Int64VarP(&props.RotateSizeMB, logRotationMaxSizeMegabytes, "", constants.DefaultMaxFileSize, "Rotate after N MegaBytes. example 100. ") 157 | rootCmd.PersistentFlags().StringVar(&props.Tags, tags, "", "Custom hardcoded tags in the form of K=V") 158 | rootCmd.PersistentFlags().StringVar(&props.ResultLog, resultLog, "", "Path of the results log where metrics such as total_lines_generated will be log.") 159 | rootCmd.PersistentFlags().StringVar(&props.CustomTimestampFormat, customTimestampFormat, "", "Optional custom timestamp format.The default format is 2006-01-02T15:04:05.000-07:00") 160 | 161 | rootCmd.MarkPersistentFlagRequired(output) 162 | rootCmd.MarkPersistentFlagRequired(linesPerSecond) 163 | 164 | //Defining local flags for replay file command 165 | replayFileCmd.Flags().StringVarP(&props.InputSourceFile, inputFilePath, "i", "default.log", "file path of output log file ex: /var/tmp/test.log") 166 | replayFileCmd.MarkFlagRequired(inputFilePath) 167 | 168 | replayFileCmd.Flags().Int64VarP(&props.ReplayCount, replayCount, "", 0, "max number of replays. default is infinite, the replay stops when test duration expires.") 169 | rootCmd.PersistentFlags().BoolVarP(&props.DisableTimestamp, disableTimestamp, "x", false, "use --disable-timestamp false will disable timestamp injection while replaying logs. "+ 170 | "When this flag is not used. Do not modify this parameter for auto-generated random alphanumberic logs."+ 171 | "Disabling timestamps is typically suitable while replaying a log-file with pre-existing timestamps.") 172 | 173 | //Defining local flags for random string generator command 174 | randomStringsCmd.Flags().IntVarP(&props.MultiLinePercent, multiLinePercent, "m", constants.DefaultMultiLinePercentage, " % of log entries which have multiple lines, example: 1% will generate ") 175 | randomStringsCmd.Flags().IntVarP(&props.NumOfLinesInMultiLineLog, lineCount, "n", constants.DefaultMultiLineCount, "max number of lines in multiline log entry") 176 | randomStringsCmd.Flags().Int64VarP(&props.LineLength, lineLength, "l", constants.DefaultLineLength, "line length(number of characters)") 177 | randomStringsCmd.MarkFlagRequired(lineCount) 178 | randomStringsCmd.MarkFlagRequired(lineLength) 179 | randomStringsCmd.MarkFlagRequired(multiLinePercent) 180 | 181 | rootCmd.AddCommand(randomStringsCmd, replayFileCmd) 182 | 183 | err := rootCmd.Execute() 184 | InputValidation(props) 185 | if err != nil { 186 | error.Error(err) 187 | os.Exit(1) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /commontest/utility_test.go: -------------------------------------------------------------------------------- 1 | package commontest 2 | 3 | import ( 4 | "github.com/google/uuid" 5 | "github.com/intuit/go-loadgen/constants" 6 | "github.com/intuit/go-loadgen/eventbreaker" 7 | loadgen "github.com/intuit/go-loadgen/loadgenerator" 8 | "github.com/intuit/go-loadgen/util" 9 | "github.com/sirupsen/logrus" 10 | "io/ioutil" 11 | "os" 12 | "strings" 13 | "testing" 14 | ) 15 | 16 | func TestGetFormatter(t *testing.T) { 17 | 18 | outputFile := constants.TestOutputFileRootPath + "/" + uuid.New().String()[:5] + ".log" 19 | file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_RDWR, 0666) 20 | defer os.Remove(outputFile) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | log := logrus.New() 25 | log.Out = file 26 | props := loadgen.SetupLogProps(false, true, &loadgen.LoadGenProperties{ 27 | DisableTimestamp: false, 28 | Tags: "", 29 | }) 30 | 31 | log.SetFormatter(utility.GetFormatter(props)) 32 | testString := "This is a test!" 33 | log.Info(testString) 34 | bytes, err := ioutil.ReadFile(file.Name()) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | fileContents := string(bytes) 39 | eb := eventbreaker.NewEventBreakers() 40 | result := eb.HasKnownEventBreakerString(fileContents) 41 | if !result { 42 | t.Fatal("Well known date pattern not found. Formatter has an error") 43 | } 44 | 45 | if !strings.Contains(fileContents, testString) { 46 | t.Fatal("Test string not found in the file after formatting!") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | // DefaultLogRotation sets default behavior log rotation 4 | const DefaultLogRotation = false 5 | 6 | // DefaultLogRotation sets default behavior log rotation 7 | const DisableTimestamp = true 8 | 9 | // MaxFileCount is the limit for maximum files that can be created by the tool 10 | const MaxFileCount = 30 11 | 12 | // DefaultFileCount is the default number of files the generator creates. 13 | const DefaultFileCount = 1 14 | 15 | // DefaultMetricsServerPortal is the default port on which the prometheus metrics server starts 16 | const DefaultMetricsServerPort = "8080" 17 | 18 | // DefaultLineLength is the default line length 19 | const DefaultLineLength = 1000 //1 KiloBytes 20 | 21 | // DefaultMaxFileSize is the maximum size in MBytes after which the file will be rotated. 22 | const DefaultMaxFileSize = 100 23 | 24 | // DefaultDurationInSeconds the default time the test would run if no duration is specified. 25 | const DefaultDurationInSeconds = 0 //0mins 26 | 27 | // DefaultReplayCount is the default replay count if the duration is not specified and the replay count is not specified. 28 | const DefaultReplayCount = 0 //5mins 29 | 30 | // DeafaultLinesPerSecond is the default LPS when it is not explicitly specified. 31 | const DefaultLinesPerSecond = 1000 // 1 Lps 32 | 33 | // DefaultMultiLinePercentage is the default % of lineCount > 1 when the percentage is not explicitly defined. 34 | const DefaultMultiLinePercentage = 0 //0% 35 | 36 | // DefaultMultiLineCount is the default multi-line count when it is not explicitly defined. 37 | const DefaultMultiLineCount = 1 //1 line 38 | 39 | // DefaultMaxFileRotationSize is max file size in MB after which the file will be rotated. 40 | const DefaultMaxFileRotationSize = 500 41 | 42 | const TestOutputFileRootPath = "/tmp" 43 | 44 | const DefaultLogTimestampFormat = "2006-01-02T15:04:05.000-07:00" 45 | -------------------------------------------------------------------------------- /eventbreaker/eventbreaker.go: -------------------------------------------------------------------------------- 1 | package eventbreaker 2 | 3 | import "regexp" 4 | 5 | type KnownEventBreakers struct { 6 | regexExp []regexp.Regexp 7 | } 8 | 9 | /* 10 | NewEventBreakers Initializes a known set of log event breakers. (Note: Usually dates and ip addresses are commonly used as event breakers.) 11 | */ 12 | func NewEventBreakers() *KnownEventBreakers { 13 | eventBreakers := new(KnownEventBreakers) 14 | //127.0.0.1 15 | ipPattern, _ := regexp.Compile(`^\s*(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) 16 | //2019-12-05 18:44:57 17 | datePattern1, _ := regexp.Compile(`^\s*\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1} [0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}`) 18 | //[2019-12-05 18:44:57] 19 | datePattern2, _ := regexp.Compile(`^\s*\[\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1} [0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}\]`) 20 | //2019-12-05T18:44:57,007 21 | datePattern3, _ := regexp.Compile(`^\s*\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1}`) 22 | //[2019-12-05T18:44:57,007+0000] 23 | datePattern4, _ := regexp.Compile(`^\[\d{4}-[01]{1}\d{1}-[0-3]{1}\d{1}T[0-2]{1}\d{1}:[0-6]{1}\d{1}:[0-6]{1}\d{1},\d{3}\+\d{4}\]`) 24 | eventBreakers.regexExp = append(eventBreakers.regexExp, *ipPattern) 25 | eventBreakers.regexExp = append(eventBreakers.regexExp, *datePattern1) 26 | eventBreakers.regexExp = append(eventBreakers.regexExp, *datePattern2) 27 | eventBreakers.regexExp = append(eventBreakers.regexExp, *datePattern3) 28 | eventBreakers.regexExp = append(eventBreakers.regexExp, *datePattern4) 29 | return eventBreakers 30 | } 31 | 32 | func (eventBreakers KnownEventBreakers) HasKnownEventBreakerString(line string) bool { 33 | for _, rex := range eventBreakers.regexExp { 34 | if rex.MatchString(line) { 35 | return true 36 | } 37 | } 38 | return false 39 | } 40 | func (eventBreakers KnownEventBreakers) HasKnownEventBreakerBytes(line []byte) bool { 41 | return eventBreakers.HasKnownEventBreakerString(string(line)) 42 | } 43 | -------------------------------------------------------------------------------- /eventbreaker/eventbreaker_test.go: -------------------------------------------------------------------------------- 1 | package eventbreaker 2 | 3 | import "testing" 4 | 5 | func TestHasKnownEventBreakerString(t *testing.T) { 6 | eb := NewEventBreakers() 7 | result := eb.HasKnownEventBreakerString("[2019-12-05T18:44:57,007+0000] randomField=1 msg=\"this is a test message\"") 8 | if !result { 9 | t.Errorf("eventbreaker did not detect a known date pattern [2019-12-05T18:44:57,007+0000] in the input message") 10 | } 11 | result = eb.HasKnownEventBreakerString("2019-12-05T18:44:57,007 randomField=1 msg=\"this is a test message\"") 12 | if !result { 13 | t.Errorf("eventbreaker did not detect a known date pattern 2019-12-05T18:44:57,007 in the input message") 14 | } 15 | result = eb.HasKnownEventBreakerString("2019-12-05 18:44:57 randomField=1 msg=\"this is a test message\"") 16 | if !result { 17 | t.Errorf("eventbreaker did not detect a known date pattern 2019-12-05 18:44:57 in the input message") 18 | } 19 | //with whitespace in the beginning 20 | result = eb.HasKnownEventBreakerString(" 2019-12-05 18:44:57 randomField=1 msg=\"this is a test message\"") 21 | if !result { 22 | t.Errorf("eventbreaker did not detect a known date pattern \" 2019-12-05 18:44:57\" in the input message") 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/intuit/go-loadgen 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/google/uuid v1.1.1 7 | github.com/prometheus/client_golang v1.2.1 8 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 9 | github.com/sirupsen/logrus v1.4.2 10 | github.com/spf13/cobra v0.0.5 11 | github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 12 | go.uber.org/ratelimit v0.2.0 13 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 14 | ) 15 | 16 | require ( 17 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.1.0 // indirect 20 | github.com/golang/protobuf v1.3.2 // indirect 21 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 22 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect 23 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 24 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect 25 | github.com/prometheus/common v0.7.0 // indirect 26 | github.com/prometheus/procfs v0.0.5 // indirect 27 | github.com/spf13/pflag v1.0.3 // indirect 28 | golang.org/x/sys v0.3.0 // indirect 29 | gopkg.in/yaml.v2 v2.2.7 // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 4 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 5 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 6 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 7 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= 8 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= 9 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 10 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 11 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 12 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 13 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 14 | github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= 15 | github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= 16 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 17 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 18 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 19 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 20 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 22 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 23 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 24 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 25 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 26 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 27 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 28 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 29 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 30 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 31 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 32 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 33 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 34 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 35 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 36 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 37 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 38 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 39 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 40 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 41 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 42 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 43 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 44 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 45 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 46 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 47 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 48 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 49 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 50 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 51 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 52 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 53 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 54 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 55 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 56 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 57 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 58 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 59 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 60 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 61 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 62 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 63 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 64 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 65 | github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= 66 | github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= 67 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 68 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 69 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 70 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 71 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 72 | github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= 73 | github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= 74 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 75 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 76 | github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= 77 | github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= 78 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= 79 | github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 80 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 81 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 82 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 83 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 84 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 85 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 86 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 87 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 88 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 89 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 90 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 91 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 92 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 93 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 94 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 95 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 96 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 97 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 98 | github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816 h1:J6v8awz+me+xeb/cUTotKgceAYouhIB3pjzgRd6IlGk= 99 | github.com/t-tomalak/logrus-easy-formatter v0.0.0-20190827215021-c074f06c5816/go.mod h1:tzym/CEb5jnFI+Q0k4Qq3+LvRF4gO3E2pxS8fHP8jcA= 100 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 101 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 102 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 103 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 104 | go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= 105 | go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= 106 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 107 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 108 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 109 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 110 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 111 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 112 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 113 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 114 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 115 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 116 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 117 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 118 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 119 | golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= 120 | golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 121 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 122 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 123 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 124 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= 125 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 126 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 127 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 128 | gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= 129 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 130 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 131 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 132 | -------------------------------------------------------------------------------- /loadgenerator/generator.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/intuit/go-loadgen/eventbreaker" 7 | metricsUtility "github.com/intuit/go-loadgen/metrics" 8 | utility "github.com/intuit/go-loadgen/util" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/rcrowley/go-metrics" 11 | "github.com/sirupsen/logrus" 12 | "go.uber.org/ratelimit" 13 | "io" 14 | "os" 15 | "strings" 16 | "time" 17 | ) 18 | 19 | func wrapUpTask(props *LoadGenProperties, totalLineCount int64) { 20 | var log = logrus.New() 21 | if props.ResultLog == "" { 22 | fmt.Printf("ResultLog file was not specified for this test. ResutLog file helps summarize the outcome of the test. " + 23 | "It is a recommended paramter.") 24 | } else { 25 | resultsLog, err := os.OpenFile(props.ResultLog, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 26 | if err != nil { 27 | fmt.Print(err) 28 | os.Exit(1) 29 | } 30 | log.Out = resultsLog 31 | log.Info(props.Tags+" total_lines_generated=", totalLineCount) 32 | resultsLog.Close() 33 | } 34 | 35 | fmt.Printf("\n\n total_lines_generated=%v\n\n ", totalLineCount) 36 | props.Wg.Done() 37 | } 38 | 39 | /* 40 | * 41 | GenerateLoadFromInputFile replays an input sample file from the top of the head and adheres to the specified format, line rate and duration. 42 | */ 43 | func GenerateLoadFromInputFile(promRegistry *prometheus.Registry, props *LoadGenProperties) { 44 | 45 | if props.ReplayCount == 0 && props.durationInSeconds() == 0 { 46 | fmt.Print("Error! Stopping the test. \n" + 47 | " Test termination factor is missing! Either specify `--duration` or `replay-count`") 48 | os.Exit(1) 49 | } 50 | 51 | fmt.Println("Generating logs by replaying input file...") 52 | const MultiLineLimit = 1000 53 | 54 | var log = logrus.New() 55 | var totalLineCount int64 56 | var fileArray = make([]*os.File, props.FileCount, props.FileCount) 57 | var logHandlers = make([]*logrus.Logger, props.FileCount) 58 | f, err := os.Open(props.InputSourceFile) 59 | if err != nil { 60 | fmt.Println(err) 61 | os.Exit(1) 62 | } 63 | 64 | if props.Rotate == true { 65 | fmt.Println("log rotation enabled") 66 | logHandlers = props.fetchLogHandlers(false) 67 | props.setupLogRotation(props.FilePath, logHandlers) 68 | } else { 69 | fmt.Println("log rotation is disabled") 70 | fileArray, _ = props.fileRef() 71 | } 72 | 73 | rateLimit := ratelimit.New(int(props.Lps)) 74 | timer := time.After(props.durationInSeconds()) 75 | numberOfFiles := int(props.FileCount) 76 | 77 | var multiLineString strings.Builder 78 | 79 | eventBreakers := eventbreaker.NewEventBreakers() 80 | fileCountIndex := 0 81 | var totalBytesProcessed int64 82 | 83 | counter := metrics.NewCounter() 84 | goGenMetricsRegistry := metrics.NewRegistry() 85 | goGenMetricsRegistry.Register("total-events-processed", counter) 86 | metricsLogger := logrus.New() 87 | metricsLogger.Out = os.Stdout 88 | logProps := SetupLogProps(false, true, props) 89 | metricsLogger.Formatter = utility.GetFormatter(logProps) 90 | 91 | go metrics.Log(goGenMetricsRegistry, 1*time.Second, metricsLogger) 92 | 93 | //prometheus stuff 94 | var promCounter, promTotalBytesProcessedCounter prometheus.Counter 95 | if props.EnableMetrics { 96 | promCounter = metricsUtility.GetEventsProcessedCounter() 97 | promTotalBytesProcessedCounter = metricsUtility.GetTotalBytesProcessedCounter() 98 | promRegistry.MustRegister(promCounter) 99 | promRegistry.MustRegister(promTotalBytesProcessedCounter) 100 | } 101 | 102 | metrics.NewRegisteredFunctionalGauge("total-bytes-processed", goGenMetricsRegistry, func() int64 { 103 | if props.EnableMetrics { 104 | promTotalBytesProcessedCounter.Add(float64(totalBytesProcessed)) 105 | } 106 | return totalBytesProcessed 107 | }) 108 | 109 | var fileLoopCount int64 110 | for { 111 | if props.ReplayCount != 0 && props.ReplayCount == fileLoopCount { 112 | wrapUpTask(props, totalLineCount) 113 | return 114 | } 115 | fileLoopCount++ 116 | 117 | f.Seek(0, io.SeekStart) 118 | reader := bufio.NewReader(f) 119 | for { 120 | now := rateLimit.Take() 121 | 122 | if props.durationInSeconds() > 0 { 123 | select { 124 | case _ = <-timer: 125 | wrapUpTask(props, totalLineCount) 126 | return 127 | default: 128 | } 129 | } 130 | counter.Inc(1) 131 | if props.EnableMetrics { 132 | promCounter.Add(1) 133 | } 134 | 135 | prev := time.Now() 136 | if !props.Rotate { 137 | log.Out = fileArray[fileCountIndex] 138 | logProps := SetupLogProps(false, false, props) 139 | log.SetFormatter(utility.GetFormatter(logProps)) 140 | } else { 141 | log = logHandlers[fileCountIndex] 142 | } 143 | currentLine, err := reader.ReadString('\n') 144 | if err != nil { 145 | if err == io.EOF { 146 | break 147 | } 148 | log.Fatalf("read file line error: %v", err) 149 | return 150 | } 151 | 152 | nextLine, err := reader.Peek(40) 153 | isNotMultiLine := eventBreakers.HasKnownEventBreakerString(currentLine) && eventBreakers.HasKnownEventBreakerBytes(nextLine) 154 | if isNotMultiLine { 155 | log.Info(currentLine) 156 | totalLineCount++ 157 | totalBytesProcessed += int64(len(currentLine)) 158 | } else { 159 | currentMultiLineCount := 0 160 | for { 161 | multiLineString.WriteString(currentLine) 162 | currentMultiLineCount++ 163 | totalLineCount++ 164 | nextLineBytes, _ := reader.Peek(40) 165 | if eventBreakers.HasKnownEventBreakerBytes(nextLineBytes) || currentMultiLineCount >= MultiLineLimit { 166 | break 167 | } 168 | currentLine, _ = reader.ReadString('\n') 169 | } 170 | log.Info(multiLineString.String()) 171 | totalBytesProcessed += int64(multiLineString.Len()) 172 | multiLineString.Reset() 173 | } 174 | 175 | if fileCountIndex == numberOfFiles-1 { 176 | fileCountIndex = 0 177 | } else { 178 | fileCountIndex++ 179 | } 180 | now.Sub(prev) 181 | prev = now 182 | } 183 | } 184 | f.Close() 185 | } 186 | 187 | /* 188 | * 189 | GenerateAlphaNumeric generates random alphanumeric strings and writes it to a specified file at specified line rate(lps) and for specified duration. 190 | */ 191 | func GenerateAlphaNumeric(promRegistry *prometheus.Registry, props *LoadGenProperties) { 192 | fmt.Print("Generating random alphanumeric characters...") 193 | var log = logrus.New() 194 | var fileArray = make([]*os.File, props.FileCount, props.FileCount) 195 | var logHandlers = make([]*logrus.Logger, props.FileCount, props.FileCount) 196 | if props.Rotate == true { 197 | fmt.Println("log rotation enabled") 198 | logHandlers = props.fetchLogHandlers(true) 199 | props.setupLogRotation(props.FilePath, logHandlers) 200 | } else { 201 | fileArray, _ = props.fileRef() 202 | } 203 | 204 | //rate limiter settings 205 | rateLimit := ratelimit.New(int(props.Lps)) 206 | //duration settings 207 | timer := time.After(props.durationInSeconds()) 208 | 209 | var totalLineCount float64 210 | var singleLineCount float64 211 | var multiLineCount float64 212 | singleLineResult := props.buildLine() 213 | multiLineResult := props.buildMultiLine() 214 | numberOfFiles := int(props.FileCount) 215 | fileCountIndex := 0 216 | prev := time.Now() 217 | 218 | //metrics 219 | counter := metrics.NewCounter() 220 | goGenMetricsRegistry := metrics.NewRegistry() 221 | goGenMetricsRegistry.Register("total_events_processed", counter) 222 | metricsLogger := logrus.New() 223 | metricsLogger.Out = os.Stdout 224 | 225 | go metrics.Log(goGenMetricsRegistry, 1*time.Second, metricsLogger) 226 | 227 | //prometheus stuff 228 | var promCounter, promTotalBytesProcessedCounter prometheus.Counter 229 | if props.EnableMetrics { 230 | promCounter = metricsUtility.GetEventsProcessedCounter() 231 | promTotalBytesProcessedCounter = metricsUtility.GetTotalBytesProcessedCounter() 232 | promRegistry.MustRegister(promCounter) 233 | promRegistry.MustRegister(promTotalBytesProcessedCounter) 234 | } 235 | 236 | metrics.NewRegisteredFunctionalGauge("bytes-per-second", goGenMetricsRegistry, func() int64 { 237 | value := props.LineLength * props.Lps 238 | //export to prom 239 | if props.EnableMetrics { 240 | promTotalBytesProcessedCounter.Add(float64(value)) 241 | } 242 | return value 243 | }) 244 | 245 | for { 246 | now := rateLimit.Take() 247 | select { 248 | case _ = <-timer: 249 | wrapUpTask(props, int64(totalLineCount)) 250 | return 251 | default: 252 | { 253 | counter.Inc(1) 254 | if props.EnableMetrics { 255 | promCounter.Add(1) 256 | } 257 | 258 | } 259 | } 260 | if !props.Rotate { 261 | log.Out = fileArray[fileCountIndex] 262 | logProps := SetupLogProps(false, true, props) 263 | log.SetFormatter(utility.GetFormatter(logProps)) 264 | } else { 265 | log = logHandlers[fileCountIndex] 266 | } 267 | 268 | if int(multiLineCount/totalLineCount*100) <= props.MultiLinePercent && props.MultiLinePercent != 0 { 269 | log.Info(multiLineResult) 270 | multiLineCount++ 271 | } else { 272 | log.Info(singleLineResult) 273 | singleLineCount++ 274 | } 275 | if fileCountIndex == numberOfFiles-1 { 276 | fileCountIndex = 0 277 | } else { 278 | fileCountIndex++ 279 | } 280 | now.Sub(prev) 281 | prev = now 282 | totalLineCount++ 283 | } 284 | 285 | } 286 | -------------------------------------------------------------------------------- /loadgenerator/generator_test.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "fmt" 5 | "github.com/google/uuid" 6 | "github.com/intuit/go-loadgen/constants" 7 | "math" 8 | "os" 9 | "path/filepath" 10 | "sync" 11 | "testing" 12 | ) 13 | 14 | const fileRootPath string = constants.TestOutputFileRootPath 15 | 16 | func setupProps(props *LoadGenProperties) { 17 | var wg sync.WaitGroup 18 | wg.Add(1) 19 | if props.LineLength == 0 { 20 | props.LineLength = 10 21 | } 22 | if props.Lps == 0 { 23 | props.Lps = 10 24 | } 25 | props.ResultLog = fileRootPath + "/" + "result.log" 26 | 27 | if props.Duration == 0 { 28 | props.Duration = 2 29 | } 30 | props.EnableMetrics = false 31 | 32 | props.FileCount = 1 33 | 34 | props.DisableTimestamp = true 35 | 36 | props.Wg = &wg 37 | 38 | if !props.Rotate { 39 | os.OpenFile(props.FilePath, os.O_CREATE, 0644) 40 | } 41 | } 42 | 43 | func TestGenerateRandomAlphaNumeric_singleLine(t *testing.T) { 44 | props := new(LoadGenProperties) 45 | props.MultiLinePercent = 0 46 | props.NumOfLinesInMultiLineLog = 1 47 | outputFilePath := fileRootPath + "/" + uuid.New().String()[:5] + ".log" 48 | props.FilePath = outputFilePath 49 | setupProps(props) 50 | GenerateAlphaNumeric(nil, props) 51 | totalNewLineCharacters := props.Duration * props.Lps 52 | expectedTotalBytesInFile := (props.Duration * props.Lps * props.LineLength) + totalNewLineCharacters 53 | expectedTotalBytesInFileWithErrorMargin := (props.Duration * props.Lps * props.LineLength) + (totalNewLineCharacters + props.LineLength + 1) 54 | 55 | defer os.Remove(outputFilePath) 56 | fmt.Println("Output path = " + outputFilePath) 57 | result, actualSize := hasExpectedLengthOfBytes(expectedTotalBytesInFile, outputFilePath) 58 | if !result && actualSize == expectedTotalBytesInFileWithErrorMargin { 59 | result = true 60 | } 61 | if !result { 62 | t.Errorf("The generated output file does not contain expected length of bytes. expected = %d, actual = %d", expectedTotalBytesInFile, actualSize) 63 | } 64 | } 65 | 66 | func TestGenerateRandomAlphaNumeric_multiLinePercent(t *testing.T) { 67 | props := new(LoadGenProperties) 68 | props.MultiLinePercent = 50 69 | props.NumOfLinesInMultiLineLog = 2 70 | outputFilePath := fileRootPath + "/" + uuid.New().String()[:5] + ".log" 71 | props.FilePath = outputFilePath 72 | props.Lps = 1 73 | props.Duration = 4 74 | //props.LogFormat = utility.GetFormatter(true, constants.DisableTimestamp) 75 | setupProps(props) 76 | GenerateAlphaNumeric(nil, props) 77 | totalNewLineCharacters := (props.Duration*props.Lps)/2 + (props.Duration*props.Lps*int64(props.NumOfLinesInMultiLineLog))/2 78 | expectedTotalBytesInFile := (props.Duration*props.Lps*props.LineLength)/2 + 79 | (props.Duration*props.Lps*props.LineLength*int64(props.NumOfLinesInMultiLineLog))/2 + 80 | totalNewLineCharacters 81 | 82 | defer os.Remove(outputFilePath) 83 | fmt.Println("Output path = " + outputFilePath) 84 | result, actualSize := hasExpectedLengthOfBytes(expectedTotalBytesInFile, outputFilePath) 85 | if !result { 86 | t.Errorf("The generated output file does not contain expected length of bytes. expected = %d, actual = %d", expectedTotalBytesInFile, actualSize) 87 | } 88 | } 89 | 90 | func deleteFile(files []string) { 91 | //cleanup any existing files before staring this test 92 | if len(files) > 0 { 93 | for _, v := range files { 94 | os.Remove(v) 95 | } 96 | } 97 | } 98 | 99 | func TestGenerateRandomAlphaNumeric_rotation(t *testing.T) { 100 | props := new(LoadGenProperties) 101 | props.MultiLinePercent = 0 102 | props.NumOfLinesInMultiLineLog = 1 103 | props.Rotate = true 104 | props.RotateSizeMB = 1 // 1MB 105 | props.LineLength = 1000000 //1MB 106 | props.Duration = 3 //in seconds 107 | props.Lps = 1 //in seconds 108 | props.FilePath = fileRootPath + "/" + uuid.New().String()[:5] 109 | 110 | setupProps(props) 111 | GenerateAlphaNumeric(nil, props) 112 | 113 | files, _ := filepath.Glob(props.FilePath + "*") 114 | defer deleteFile(files) 115 | expectedNumOfRotatedFiles := props.Duration * props.Lps 116 | 117 | if len(files) != int(expectedNumOfRotatedFiles) { 118 | t.Errorf("Log rotation validation tests failed because expected number of rotated files = %d, actual = %d", expectedNumOfRotatedFiles, len(files)) 119 | } 120 | } 121 | 122 | func TestGenerateRandomAlphaNumericLoad_multiLine(t *testing.T) { 123 | props := new(LoadGenProperties) 124 | props.DisableTimestamp = true 125 | props.MultiLinePercent = 100 //100% of lines are multi-lines in nature. 126 | props.NumOfLinesInMultiLineLog = 2 127 | outputFilePath := uuid.New().String()[:5] + ".log" 128 | props.FilePath = outputFilePath 129 | setupProps(props) 130 | fmt.Printf("%+v", props) 131 | GenerateAlphaNumeric(nil, props) 132 | 133 | totalNewLineCharacters := props.Duration * props.Lps * int64(props.NumOfLinesInMultiLineLog) 134 | expectedTotalBytesInFile := props.Duration*props.Lps*int64(props.NumOfLinesInMultiLineLog)*props.LineLength + totalNewLineCharacters 135 | 136 | defer os.Remove(outputFilePath) 137 | fmt.Println("Output path = " + outputFilePath) 138 | result, actualSize := hasExpectedLengthOfBytes(expectedTotalBytesInFile, outputFilePath) 139 | if !result { 140 | t.Errorf("The generated output file does not contain expected length of bytes. expected = %d, actual = %d", expectedTotalBytesInFile, actualSize) 141 | } 142 | } 143 | 144 | func hasExpectedLengthOfBytes(expectedTotalBytesInFile int64, output string) (bool, int64) { 145 | 146 | info, err := os.Stat(output) 147 | if err != nil { 148 | return false, info.Size() 149 | } 150 | if approximatelyEqual(expectedTotalBytesInFile, info.Size()) { 151 | return true, info.Size() 152 | } 153 | return false, info.Size() 154 | } 155 | 156 | func approximatelyEqual(expected, actual int64) bool { 157 | errorMargin := 0.05 // 5% error is tolerable 158 | fivePercentOfExpected := float64(expected) * errorMargin 159 | diff := math.Abs(float64(expected - actual)) 160 | fmt.Println(fivePercentOfExpected) 161 | fmt.Println(diff) 162 | if diff <= fivePercentOfExpected { 163 | return true 164 | } 165 | return false 166 | } 167 | -------------------------------------------------------------------------------- /loadgenerator/loadgen.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/intuit/go-loadgen/constants" 7 | utility "github.com/intuit/go-loadgen/util" 8 | "github.com/sirupsen/logrus" 9 | easy "github.com/t-tomalak/logrus-easy-formatter" 10 | "gopkg.in/natefinch/lumberjack.v2" 11 | "io/ioutil" 12 | "math/rand" 13 | "os" 14 | "strconv" 15 | "strings" 16 | "sync" 17 | "time" 18 | ) 19 | 20 | type LoadGenProperties struct { 21 | Duration int64 `json:"duration"` 22 | FileCount int64 `json:"file-count"` 23 | LineLength int64 `json:"line-length"` 24 | MultiLinePercent int `json:"multiline-percentage"` 25 | NumOfLinesInMultiLineLog int `json:"line-count"` 26 | FilePath string `json:"file-path"` 27 | InputSourceFile string `json:"input-source-file"` 28 | ReplayCount int64 `json:"replay-count"` 29 | CustomTimestampFormat string `json:"custom-timestamp-format"` 30 | DisableTimestamp bool `json:"disable-timestamp"` 31 | Lps int64 `json:"lines-per-second"` 32 | Rotate bool `json:"rotate"` 33 | RotateSizeMB int64 `json:"rotation-max-file-size-megabytes"` 34 | EnableMetrics bool `json:"enable-metrics"` 35 | Tags string `json:"tags"` 36 | LogFormat *easy.Formatter `json:"-"` 37 | Wg *sync.WaitGroup `json:"-"` 38 | ResultLog string `json:"results-log"` 39 | } 40 | 41 | const chars = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "1234567890" 42 | 43 | func (props *LoadGenProperties) readFromConfig(configFile *string) { 44 | config, err := os.Open(*configFile) 45 | defer config.Close() 46 | if err != nil && *configFile != "" { 47 | fmt.Println(err) 48 | os.Exit(1) 49 | } 50 | 51 | rawJsonConfig, err := ioutil.ReadAll(config) 52 | if err != nil && *configFile != "" { 53 | fmt.Println(err) 54 | os.Exit(1) 55 | } 56 | err = json.Unmarshal(rawJsonConfig, &props) 57 | if err != nil && *configFile != "" { 58 | fmt.Println(err) 59 | os.Exit(1) 60 | } 61 | } 62 | 63 | func (props *LoadGenProperties) durationInSeconds() time.Duration { 64 | return time.Second * (time.Duration(props.Duration)) 65 | } 66 | 67 | func (props *LoadGenProperties) fetchLogHandlers(isLineEndsWithNewLine bool) []*logrus.Logger { 68 | logHandlers := make([]*logrus.Logger, props.FileCount) 69 | for n := 0; n < int(props.FileCount); n++ { 70 | logHandlers[n] = logrus.New() 71 | if props.LogFormat == nil { 72 | logProps := SetupLogProps(false, true, props) 73 | logHandlers[n].SetFormatter(utility.GetFormatter(logProps)) 74 | } else { 75 | logHandlers[n].SetFormatter(props.LogFormat) 76 | } 77 | } 78 | return logHandlers 79 | } 80 | 81 | /** 82 | * Creates n number of files and returns an array of pointer references to the created files. 83 | */ 84 | func (props *LoadGenProperties) fileRef() ([]*os.File, error) { 85 | var fileArray = make([]*os.File, props.FileCount, props.FileCount) 86 | 87 | // if output path is declared as stdout then return reference to stdout and skip rest 88 | if props.FilePath != "" { 89 | if strings.ToLower(props.FilePath) == "stdout" { 90 | fileArray[0] = os.Stdout 91 | return fileArray, nil 92 | } 93 | } 94 | var err error = nil 95 | for n := 0; n < int(props.FileCount); n++ { 96 | fileVar := props.FilePath 97 | if props.FileCount > 1 { 98 | if strings.Contains(fileVar, ".") { 99 | //attach file number before the file extension 100 | fileVar = strings.Replace(fileVar, ".", strconv.Itoa(n)+".", 1) 101 | } else { 102 | fileVar = fileVar + strconv.Itoa(n) 103 | } 104 | } 105 | fileArray[n], err = os.OpenFile(fileVar, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 106 | if err != nil { 107 | break 108 | } 109 | } 110 | return fileArray, err 111 | } 112 | 113 | func (props *LoadGenProperties) buildLine() string { 114 | var seededRand = rand.New( 115 | rand.NewSource(time.Now().UnixNano())) 116 | randString := make([]byte, props.LineLength) 117 | for i, _ := range randString { 118 | randNumber := seededRand.Intn(int(len(chars) - 1)) 119 | randString[i] = chars[randNumber] 120 | } 121 | 122 | return string(randString) 123 | } 124 | 125 | func (props *LoadGenProperties) setupLogRotation(filePath string, logHandlers []*logrus.Logger) { 126 | 127 | for i := 0; i < len(logHandlers); i++ { 128 | var maxFileSize int = 0 129 | if props.RotateSizeMB == 0 { 130 | maxFileSize = constants.DefaultMaxFileRotationSize 131 | } else { 132 | maxFileSize = int(props.RotateSizeMB) 133 | } 134 | 135 | var suffix string 136 | if props.FileCount > 1 { 137 | suffix = "-" + strconv.Itoa(i) 138 | } 139 | if props.FileCount == 1 { 140 | suffix = "" 141 | } 142 | 143 | (logHandlers)[i].SetOutput(&lumberjack.Logger{ 144 | Filename: strings.Replace(filePath, ".", suffix+".", 1), 145 | MaxSize: maxFileSize, //megabytes 146 | MaxAge: 1, //day 147 | MaxBackups: 3, //max of n files. 148 | Compress: false, //dont compress to avoid CPU spikes on load generator. 149 | }) 150 | } 151 | 152 | } 153 | 154 | func (props *LoadGenProperties) buildMultiLine() string { 155 | lineLength := int(props.LineLength) 156 | numberOfLines := int(props.NumOfLinesInMultiLineLog) 157 | maxCapacityRequired := (numberOfLines * (lineLength)) + (numberOfLines - 1) 158 | multiLineString := make([]byte, maxCapacityRequired, maxCapacityRequired) 159 | totalLineLength := 0 160 | for i := 0; i < numberOfLines; i++ { 161 | 162 | copy(multiLineString[totalLineLength:], []byte(props.buildLine())) 163 | totalLineLength = totalLineLength + lineLength 164 | if i < numberOfLines { 165 | copy(multiLineString[totalLineLength:], []byte("\n")) 166 | } 167 | totalLineLength += 1 168 | } 169 | return string(multiLineString) 170 | } 171 | 172 | func SetupLogProps(isMetricsLogs bool, insertNewline bool, props *LoadGenProperties) *utility.LogProperties { 173 | logProps := new(utility.LogProperties) 174 | 175 | if isMetricsLogs { 176 | logProps.DisableTimestamp = false 177 | logProps.CustomTimestampFormat = "" 178 | logProps.Tags = "" 179 | logProps.IsLineEndsWithNewLine = true 180 | } else { 181 | logProps.DisableTimestamp = props.DisableTimestamp 182 | logProps.CustomTimestampFormat = props.CustomTimestampFormat 183 | logProps.Tags = props.Tags 184 | if insertNewline { 185 | logProps.IsLineEndsWithNewLine = true 186 | } else { 187 | logProps.IsLineEndsWithNewLine = false 188 | } 189 | } 190 | 191 | return logProps 192 | } 193 | -------------------------------------------------------------------------------- /loadgenerator/loadgen_test.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestBuildLineMethod(t *testing.T) { 11 | var limit int = 10 12 | loadGen := new(LoadGenProperties) 13 | loadGen.LineLength = int64(limit) 14 | if len(loadGen.buildLine()) != limit { 15 | t.Error("buildLine() has generated a string which is not equal to the specified lineLength") 16 | } 17 | } 18 | 19 | func tearDownTestFiles(fileArray []*os.File) { 20 | for _, v := range fileArray { 21 | _ = os.Remove(v.Name()) 22 | } 23 | } 24 | 25 | func TestMultiLineBuilder(t *testing.T) { 26 | loadGen := new(LoadGenProperties) 27 | var limit int = 10 28 | loadGen.LineLength = int64(limit) 29 | loadGen.NumOfLinesInMultiLineLog = 1 30 | expectedNumberOfLines := loadGen.NumOfLinesInMultiLineLog 31 | expectedNumberOfBytes := (int(loadGen.LineLength) * loadGen.NumOfLinesInMultiLineLog) + (loadGen.NumOfLinesInMultiLineLog - 1) 32 | fmt.Print(expectedNumberOfBytes) 33 | 34 | outputLine := loadGen.buildMultiLine() 35 | if len(outputLine) != expectedNumberOfBytes { 36 | t.Errorf("Multiline does not have the expected number of bytes. actual = %d, expected %d", len(outputLine), expectedNumberOfBytes) 37 | } 38 | lines := strings.Split(outputLine, "\n") 39 | if len(lines) != expectedNumberOfLines { 40 | t.Errorf("Multiline does not contain the expected number of new lines. actual = %d, expected %d", len(lines), expectedNumberOfLines) 41 | } 42 | 43 | } 44 | 45 | func TestFileRefs(t *testing.T) { 46 | var limit int = 10 47 | loadGen := new(LoadGenProperties) 48 | loadGen.FilePath = "xtest.log" 49 | loadGen.FileCount = int64(limit) 50 | files, err := loadGen.fileRef() 51 | defer tearDownTestFiles(files) 52 | 53 | if err != nil { 54 | t.Errorf("fileRef() method output is invalid %s", err) 55 | } 56 | if files == nil { 57 | t.Errorf("fileRef() output is nil, expected %d", limit) 58 | } 59 | if len(files) != limit { 60 | t.Errorf("length of file array returned by fileRef() has a mismatch, contains %d, expected %d", len(files), limit) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/go-loadgen/ffc47e437b5ecd75297702c303e56887b7a8aa39/logo.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/intuit/go-loadgen/cli" 5 | loadgen "github.com/intuit/go-loadgen/loadgenerator" 6 | ) 7 | 8 | func main() { 9 | 10 | props := new(loadgen.LoadGenProperties) 11 | cli.Run(props) 12 | } -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | // GetTotalBytesProcessedCounter returns a prom counter for total_bytes_processed metric 8 | func GetTotalBytesProcessedCounter() (promTotalBytesProcessedCounter prometheus.Counter) { 9 | promTotalBytesProcessedCounter = prometheus.NewCounter( 10 | prometheus.CounterOpts{ 11 | Name: "total_bytes_processed", 12 | Help: "Total bytes processed", 13 | }, 14 | ) 15 | return 16 | } 17 | 18 | func GetEventsProcessedCounter() (promCounter prometheus.Counter) { 19 | promCounter = prometheus.NewCounter( 20 | prometheus.CounterOpts{ 21 | Name: "total_events_processed", 22 | Help: "Total log events generated by the tool.", 23 | }, 24 | ) 25 | return 26 | } 27 | -------------------------------------------------------------------------------- /metrics/server.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promhttp" 6 | "log" 7 | "net/http" 8 | ) 9 | 10 | 11 | 12 | func StartMetricsServer(r *prometheus.Registry, metricsServerPort *string) { 13 | 14 | handler := promhttp.HandlerFor(r, promhttp.HandlerOpts{}) 15 | 16 | http.Handle("/metrics", handler) 17 | log.Fatal(http.ListenAndServe(":" + *metricsServerPort, nil)) 18 | 19 | } -------------------------------------------------------------------------------- /util/utility.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "github.com/intuit/go-loadgen/constants" 5 | easy "github.com/t-tomalak/logrus-easy-formatter" 6 | ) 7 | 8 | type LogProperties struct { 9 | IsLineEndsWithNewLine bool 10 | CustomTimestampFormat string 11 | DisableTimestamp bool 12 | Tags string 13 | LogFormat *easy.Formatter 14 | } 15 | 16 | func GetFormatter(logProps *LogProperties) *easy.Formatter { 17 | var format string 18 | 19 | if !logProps.DisableTimestamp { 20 | format += "%time%" + " " 21 | } 22 | 23 | if logProps.Tags != "" { 24 | format += logProps.Tags + " " 25 | } 26 | 27 | if logProps.IsLineEndsWithNewLine { 28 | format += "%msg%\n" 29 | } else { 30 | format += "%msg%" 31 | } 32 | 33 | var timestampFormat = "" 34 | if logProps.CustomTimestampFormat != "" { 35 | timestampFormat = logProps.CustomTimestampFormat 36 | } else { 37 | timestampFormat = constants.DefaultLogTimestampFormat 38 | 39 | } 40 | var logFormat = &easy.Formatter{ 41 | LogFormat: format, 42 | TimestampFormat: timestampFormat, 43 | } 44 | return logFormat 45 | } 46 | --------------------------------------------------------------------------------