├── .github └── workflows │ ├── go.yml │ └── release.yml ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── Makefile ├── README.md ├── bench.sh ├── cli.go ├── cli_test.go ├── cmd └── querydigest │ └── main.go ├── dialect └── mysql.go ├── go.mod ├── go.sum ├── internal └── dart │ ├── dart.go │ └── dart_test.go ├── replace.go ├── slowquery.go ├── slowquery_test.go ├── stat.go ├── stat.sh ├── summarizer.go ├── summary.go └── testdata ├── mysql-slow.createtable.log ├── mysql-slow.header.log └── mysql-slow.insert.log /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: make 35 | 36 | - name: Test 37 | run: go test -v ./... 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | on: 3 | push: 4 | tags: 5 | - "v[0-9]+.[0-9]+.[0-9]+" 6 | jobs: 7 | goreleaser: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v1 12 | with: 13 | fetch-depth: 1 14 | - name: Setup Go 15 | uses: actions/setup-go@v1 16 | with: 17 | go-version: 1.13 18 | - name: Run GoReleaser 19 | uses: goreleaser/goreleaser-action@v1 20 | with: 21 | version: latest 22 | args: release --rm-dist 23 | env: 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/go 3 | # Edit at https://www.gitignore.io/?templates=go 4 | 5 | ### Go ### 6 | # Binaries for programs and plugins 7 | *.exe 8 | *.exe~ 9 | *.dll 10 | *.so 11 | *.dylib 12 | 13 | # Test binary, built with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | ### Go Patch ### 23 | /vendor/ 24 | /Godeps/ 25 | 26 | # End of https://www.gitignore.io/api/go 27 | # 28 | bin 29 | slow.log 30 | *.pprof 31 | 32 | benchdata 33 | 34 | querydigest.out 35 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: querydigest 2 | env: 3 | - GO111MODULE=on 4 | before: 5 | hooks: 6 | - go mod tidy 7 | builds: 8 | - main: ./cmd/querydigest/main.go 9 | binary: querydigest 10 | ldflags: 11 | - -s -w 12 | - -X main.Version={{.Version}} 13 | - -X main.Revision={{.ShortCommit}} 14 | env: 15 | - CGO_ENABLED=0 16 | archives: 17 | - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 18 | replacements: 19 | darwin: darwin 20 | linux: linux 21 | windows: windows 22 | 386: i386 23 | amd64: x86_64 24 | format_overrides: 25 | - goos: windows 26 | format: zip 27 | release: 28 | prerelease: auto -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | build: bin/querydigest 3 | 4 | .PHONY: bin/querydigest 5 | bin/querydigest: vendor 6 | go build -o bin/querydigest -mod vendor cmd/querydigest/main.go 7 | 8 | # .PHONY: generate 9 | # generate: 10 | 11 | .PHONY: test 12 | test: vendor 13 | go test ./... -cover -count=1 -v 14 | 15 | vendor: go.mod go.sum 16 | go mod vendor 17 | 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | querydigest 2 | ---- 3 | 4 | MySQL slow query log analyzer. 5 | 6 | This project is very limited version of [pt-query-digest](https://www.percona.com/doc/percona-toolkit/LATEST/pt-query-digest.html). 7 | 8 | ## Getting Started 9 | 10 | ### Prerequisites 11 | - Go 1.12+ 12 | 13 | ### Installing 14 | ``` 15 | $ go get -u github.com/akito0107/querydigest/cmd/querydigest 16 | ``` 17 | 18 | ### How To Use 19 | 20 | ``` 21 | $ querydigest -f path/to/slow_query_log 22 | ``` 23 | 24 | then, summaries appear as below: 25 | 26 | ``` 27 | Query 0 28 | 51.103328% 29 | 30 | Summary: 31 | total query time: 107.51s 32 | total query count: 2969 33 | 34 | \+--------------+---------+------+-------+------+------+--------+--------+ 35 | | ATTRIBUTE | TOTAL | MIN | MAX | AVG | 95% | STDDEV | MEDIAN | 36 | \+--------------+---------+------+-------+------+------+--------+--------+ 37 | | Exec Time | 108s | 13us | 3s | 36ms | 83ms | 218ms | 293us | 38 | | Lock Time | 27s | 0us | 926ms | 9ms | 27ms | 55ms | 27us | 39 | | Rows Sent | 1417.00 | 0.00 | 2.00 | 0.48 | 1.00 | 0.54 | 0.00 | 40 | | Rows Examine | 2834.00 | 0.00 | 4.00 | 0.95 | 2.00 | 1.08 | 0.00 | 41 | \+--------------+---------+------+-------+------+------+--------+--------+ 42 | 43 | Query_time distribution: 44 | 1us: 45 | 10us: ########################################### 46 | 100us: ########################################################################### 47 | 1ms: ############################### 48 | 10ms: ######################################## 49 | 100ms: ##### 50 | 1s: # 51 | 10s~: 52 | 53 | QueryExample: 54 | select * from example_table; 55 | 56 | 57 | Query 1 58 | 10.935321% 59 | 60 | Summary: 61 | total query time: 23.01s 62 | total query count: 2469 63 | 64 | \+--------------+-------+-------+-------+------+------+--------+--------+ 65 | | ATTRIBUTE | TOTAL | MIN | MAX | AVG | 95% | STDDEV | MEDIAN | 66 | \+--------------+-------+-------+-------+------+------+--------+--------+ 67 | | Exec Time | 23s | 119us | 349ms | 9ms | 47ms | 21ms | 2ms | 68 | | Lock Time | 4s | 10us | 101ms | 2ms | 11ms | 7ms | 23us | 69 | | Rows Sent | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 70 | | Rows Examine | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 71 | \+--------------+-------+-------+-------+------+------+--------+--------+ 72 | 73 | Query_time distribution: 74 | 1us: 75 | 10us: 76 | 100us: #################################################################### 77 | 1ms: ########################################################################### 78 | 10ms: ################################### 79 | 100ms: # 80 | 1s: 81 | 10s~: 82 | 83 | QueryExample: 84 | select * from example_table2; 85 | 86 | ..... 87 | ``` 88 | 89 | By default, `querydigest` analyzes and shows all queries from given slow query log. If you want to display only top `n` items, please use `-n` option. 90 | 91 | ``` 92 | $ querydigest -f path/to/slow_query_log -n 10 93 | ``` 94 | 95 | ## Limitations 96 | Currently, `querydigest` can't parse and analyze all queries supported by MySQL. These queries are excluded from analysis. 97 | 98 | All statistics are approximate value, and there are no guarantee of accuracy. 99 | 100 | ## Options 101 | ``` 102 | $ querydigest -help 103 | Usage of bin/querydigest: 104 | -f string 105 | slow log filepath (default "slow.log") 106 | -j int 107 | concurrency (default = num of cpus) 108 | -n int 109 | count 110 | ``` 111 | 112 | ## License 113 | This project is licensed under the Apache License 2.0 License - see the [LICENSE](LICENSE) file for details 114 | -------------------------------------------------------------------------------- /bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +eu 4 | 5 | now=`date +%s` 6 | count=${2:-1} 7 | tag=${3:-$now} 8 | core=${4:-"2,4,6,8,12"} 9 | 10 | resultPath="benchresult/${1}/${tag}" 11 | 12 | mkdir -p "${resultPath}" 13 | 14 | case "$1" in 15 | "cli" ) go test -bench BenchmarkRun \ 16 | -benchmem -o "${resultPath}/querydigest.out" \ 17 | -cpuprofile "${resultPath}/cpu.pprof" \ 18 | -memprofile "${resultPath}/mem.pprof" \ 19 | -count "${count}"\ 20 | -cpu "${core}" \ 21 | | tee "${resultPath}/cli.txt";; 22 | 23 | "scanner" ) go test -bench BenchmarkSlowQueryScanner_SlowQueryInfo \ 24 | -benchmem -o "${resultPath}/querydigest.out" \ 25 | -cpuprofile "${resultPath}/cpu.pprof" \ 26 | -memprofile "${resultPath}/mem.pprof" \ 27 | -count "${count}"\ 28 | | tee "${resultPath}/scanner.txt";; 29 | *) 30 | rm -rf ${resultPath}; 31 | echo "unknown command"; 32 | esac 33 | -------------------------------------------------------------------------------- /cli.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "sync" 8 | ) 9 | 10 | func Run(w io.Writer, src io.Reader, previewSize, concurrency int) { 11 | 12 | results, total, err := analyzeSlowQuery(src, concurrency) 13 | if err != nil { 14 | log.Fatal("analyzeSlowQuery:", err) 15 | } 16 | 17 | if previewSize != 0 && previewSize <= len(results) { 18 | results = results[0:previewSize] 19 | } 20 | 21 | print(w, results, total) 22 | } 23 | 24 | func print(w io.Writer, summaries []*SlowQuerySummary, totalTime float64) { 25 | for i, s := range summaries { 26 | fmt.Fprintln(w) 27 | fmt.Fprintf(w, "Query %d\n", i) 28 | fmt.Fprintf(w, "%f%%\n\n", (s.TotalTime/totalTime)*100) 29 | fmt.Fprintf(w, "%s", s.String()) 30 | fmt.Fprintln(w) 31 | } 32 | } 33 | 34 | func analyzeSlowQuery(r io.Reader, concurrency int) ([]*SlowQuerySummary, float64, error) { 35 | if concurrency > 1 { 36 | return analyzeSlowQueryParallel(r, concurrency) 37 | } 38 | summarizer := NewSummarizer() 39 | slowQueryScanner := NewSlowQueryScanner(r) 40 | for slowQueryScanner.Next() { 41 | s := slowQueryScanner.SlowQueryInfo() 42 | res, err := ReplaceWithZeroValue(s.RawQuery) 43 | if err != nil { 44 | b := s.RawQuery 45 | if len(b) > 60 { 46 | b = b[:60] 47 | } 48 | log.Print("replace failed: ", string(b)) 49 | continue 50 | } 51 | s.ParsedQuery = res 52 | summarizer.Collect(s) 53 | } 54 | if err := slowQueryScanner.Err(); err != nil { 55 | return nil, 0, err 56 | } 57 | qs := summarizer.Summarize() 58 | return qs, summarizer.TotalQueryTime(), nil 59 | } 60 | 61 | func analyzeSlowQueryParallel(r io.Reader, concurrency int) ([]*SlowQuerySummary, float64, error) { 62 | parsequeue := make(chan *SlowQueryInfo, 500) 63 | go parseRawFile(r, parsequeue) 64 | summarizer := NewSummarizer() 65 | var wg sync.WaitGroup 66 | 67 | for i := 0; i < concurrency; i++ { 68 | 69 | wg.Add(1) 70 | go func() { 71 | defer wg.Done() 72 | for s := range parsequeue { 73 | res, err := ReplaceWithZeroValue(s.RawQuery) 74 | if err != nil { 75 | b := s.RawQuery 76 | if len(b) > 60 { 77 | b = b[:60] 78 | } 79 | log.Print("replace failed: ", string(b)) 80 | continue 81 | } 82 | s.ParsedQuery = res 83 | summarizer.Collect(s) 84 | } 85 | }() 86 | } 87 | wg.Wait() 88 | 89 | qs := summarizer.Summarize() 90 | 91 | return qs, summarizer.TotalQueryTime(), nil 92 | } 93 | 94 | func parseRawFile(r io.Reader, parsequeue chan *SlowQueryInfo) { 95 | slowqueryscanner := NewSlowQueryScanner(r) 96 | 97 | for slowqueryscanner.Next() { 98 | parsequeue <- slowqueryscanner.SlowQueryInfo().clone() 99 | } 100 | if err := slowqueryscanner.Err(); err != nil { 101 | log.Fatal("slowQueryScanner:", err) 102 | } 103 | close(parsequeue) 104 | } 105 | -------------------------------------------------------------------------------- /cli_test.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "runtime" 7 | "testing" 8 | ) 9 | 10 | func BenchmarkRun(b *testing.B) { 11 | b.ResetTimer() 12 | 13 | for i := 0; i < b.N; i++ { 14 | f, err := os.Open("./benchdata/mysql-slow.log") 15 | if err != nil { 16 | b.Fatal(err) 17 | } 18 | 19 | Run(ioutil.Discard, f, 0, runtime.GOMAXPROCS(0)) 20 | 21 | f.Close() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cmd/querydigest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "os" 7 | "runtime" 8 | 9 | "github.com/akito0107/querydigest" 10 | ) 11 | 12 | var slowLogPath = flag.String("f", "slow.log", "slow log filepath") 13 | var previewSize = flag.Int("n", 0, "count") 14 | var concurrency = flag.Int("j", 0, "concurrency (default = num of cpus)") 15 | 16 | func main() { 17 | // defer profile.Start(profile.ProfilePath("."), profile.TraceProfile).Stop() 18 | // defer profile.Start(profile.ProfilePath("."), profile.CPUProfile).Stop() 19 | // defer profile.Start(profile.ProfilePath("."), profile.MemProfile).Stop() 20 | 21 | flag.Parse() 22 | 23 | f, err := os.Open(*slowLogPath) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | defer f.Close() 28 | 29 | if *concurrency == 0 { 30 | *concurrency = runtime.NumCPU() 31 | } 32 | 33 | querydigest.Run(os.Stdout, f, *previewSize, *concurrency) 34 | } 35 | -------------------------------------------------------------------------------- /dialect/mysql.go: -------------------------------------------------------------------------------- 1 | package dialect 2 | 3 | import "github.com/akito0107/xsqlparser/dialect" 4 | 5 | type MySQLDialect struct { 6 | dialect.GenericSQLDialect 7 | } 8 | 9 | func NewMySQLDialect() *MySQLDialect { 10 | return &MySQLDialect{} 11 | } 12 | 13 | func (m *MySQLDialect) IsDelimitedIdentifierStart(r rune) bool { 14 | return r == '"' || r == '`' 15 | } 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/akito0107/querydigest 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/akito0107/xsqlparser v1.0.0-alpha.6 7 | github.com/go-openapi/strfmt v0.19.3 // indirect 8 | github.com/google/go-cmp v0.4.0 9 | github.com/jedib0t/go-pretty v4.3.0+incompatible 10 | github.com/mattn/go-runewidth v0.0.7 // indirect 11 | github.com/stuartcarnie/go-simd v0.0.0-20181029150639-4ad6cd8935a6 12 | gonum.org/v1/gonum v0.6.1 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= 2 | github.com/akito0107/xsqlparser v1.0.0-alpha.6 h1:R3EsroeOwuPaguJUaKL69l1C/oU+oxEHBHWwK2Nr9/I= 3 | github.com/akito0107/xsqlparser v1.0.0-alpha.6/go.mod h1:+6r2ZNjHjXmqDLYcNdC3axnVByJ6Wyzx9ZVQs+JB53k= 4 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= 5 | github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= 6 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= 7 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= 12 | github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= 13 | github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 14 | github.com/go-openapi/strfmt v0.19.3 h1:eRfyY5SkaNJCAwmmMcADjY31ow9+N7MCLW7oRkbsINA= 15 | github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= 16 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 17 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 18 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= 19 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 20 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 21 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 22 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 23 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 24 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 25 | github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= 26 | github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= 27 | github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= 28 | github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= 29 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 30 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 31 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 32 | github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= 33 | github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 34 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 35 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 36 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 37 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 38 | github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= 39 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 42 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 43 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 44 | github.com/stuartcarnie/go-simd v0.0.0-20181029150639-4ad6cd8935a6 h1:5gmpBEKDaJNivPtyaSh80n+sIxJwNja/7I1uVChNUL4= 45 | github.com/stuartcarnie/go-simd v0.0.0-20181029150639-4ad6cd8935a6/go.mod h1:UOQhSGtLc+pMxOqrLHjJKLxe8PLSEy8E1tmIvh2Awr4= 46 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= 47 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 48 | go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= 49 | go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= 50 | golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 51 | golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 52 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= 53 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 54 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= 55 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= 57 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 59 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 60 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 61 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 62 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 63 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= 64 | gonum.org/v1/gonum v0.6.1 h1:/LSrTrgZtpbXyAR6+0e152SROCkJJSh7goYWVmdPFGc= 65 | gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= 66 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= 67 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 68 | gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= 69 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 70 | -------------------------------------------------------------------------------- /internal/dart/dart.go: -------------------------------------------------------------------------------- 1 | package dart 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type node struct { 8 | base int 9 | check int 10 | } 11 | 12 | type PrefixMatcher struct { 13 | nodes []node 14 | } 15 | 16 | const sentinelCode = 26 17 | 18 | type trie struct { 19 | next [sentinelCode + 1]*trie 20 | } 21 | 22 | func Build(keys []string) (*PrefixMatcher, error) { 23 | root, err := buildTrie(keys) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | nodes := make([]node, 256) 29 | nodes[0] = node{base: 1, check: 0} 30 | 31 | var dfs func(*trie, int) 32 | dfs = func(t *trie, id int) { 33 | for c, n := range t.next { 34 | if n == nil { 35 | continue 36 | } 37 | nid := nodes[id].base+c 38 | nodes[nid].check = id 39 | nodes[nid].base = 1 40 | } 41 | for c, n := range t.next { 42 | if n == nil { 43 | continue 44 | } 45 | i := nodes[id].base + c 46 | for base := 1; ; base++ { 47 | ok := true 48 | for nc, nn := range n.next { 49 | if nn == nil { 50 | continue 51 | } 52 | if nodes[base+nc].base != 0 { 53 | ok = false 54 | break 55 | } 56 | } 57 | if ok { 58 | nodes[i].base = base 59 | dfs(n, i) 60 | break 61 | } 62 | } 63 | } 64 | } 65 | dfs(root, 0) 66 | return &PrefixMatcher{nodes: nodes}, nil 67 | } 68 | 69 | func Must(pm *PrefixMatcher, err error) *PrefixMatcher { 70 | if err != nil { 71 | panic(err) 72 | } 73 | return pm 74 | } 75 | 76 | func (m *PrefixMatcher) Match(b []byte) bool { 77 | n := 0 78 | for _, x := range b { 79 | base := m.nodes[n].base 80 | if m.nodes[base+sentinelCode].check == n { 81 | return true 82 | } 83 | c := toCode(x) 84 | if c < 0 { 85 | return false 86 | } 87 | if m.nodes[base+c].check != n { 88 | return false 89 | } 90 | n = base + c 91 | } 92 | base := m.nodes[n].base 93 | return m.nodes[base+sentinelCode].check == n 94 | } 95 | 96 | func toCode(b byte) int { 97 | if 'A' <= b && b <= 'Z' { 98 | return int(b - 'A') 99 | } 100 | if 'a' <= b && b <= 'z' { 101 | return int(b - 'a') 102 | } 103 | return -1 104 | } 105 | 106 | func buildTrie(keys []string) (*trie, error) { 107 | root := &trie{} 108 | 109 | for _, key := range keys { 110 | t := root 111 | for _, b := range []byte(key) { 112 | c := toCode(b) 113 | if c < 0 { 114 | return nil, fmt.Errorf("key(%q) contains unknown", key) 115 | } 116 | if t.next[c] == nil { 117 | t.next[c] = &trie{} 118 | } 119 | t = t.next[c] 120 | } 121 | if t.next[sentinelCode] == nil { 122 | t.next[sentinelCode] = &trie{} 123 | } 124 | } 125 | return root, nil 126 | } 127 | -------------------------------------------------------------------------------- /internal/dart/dart_test.go: -------------------------------------------------------------------------------- 1 | package dart 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestPrefixMatcher_Match(t *testing.T) { 9 | statement := []string{ 10 | "SELECT", 11 | "INSERT", 12 | "UPDATE", 13 | "DELETE", 14 | "WITH", 15 | "ALTER", 16 | } 17 | 18 | tests := []struct { 19 | name string 20 | keys []string 21 | s string 22 | want bool 23 | }{ 24 | { 25 | keys: []string{ 26 | "A", 27 | "BC", 28 | }, 29 | s: "BCD", 30 | want: true, 31 | }, 32 | { 33 | keys: statement, 34 | s: "SELECT * FROM t;", 35 | want: true, 36 | }, 37 | { 38 | keys: statement, 39 | s: "SELECT", 40 | want: true, 41 | }, 42 | { 43 | keys: statement, 44 | s: "INSERT", 45 | want: true, 46 | }, 47 | { 48 | keys: statement, 49 | s: "UPDATE", 50 | want: true, 51 | }, 52 | { 53 | keys: statement, 54 | s: "WITH", 55 | want: true, 56 | }, 57 | { 58 | keys: statement, 59 | s: "DELETE", 60 | want: true, 61 | }, 62 | { 63 | keys: statement, 64 | s: "ALTER", 65 | want: true, 66 | }, 67 | { 68 | keys: statement, 69 | s: " SELECT", 70 | want: false, 71 | }, 72 | } 73 | for _, tt := range tests { 74 | t.Run(tt.name, func(t *testing.T) { 75 | m, err := Build(tt.keys) 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | if got := m.Match([]byte(tt.s)); got != tt.want { 80 | t.Errorf("Match() = %v, want %v", got, tt.want) 81 | } 82 | }) 83 | } 84 | } 85 | 86 | func TestBuild(t *testing.T) { 87 | type args struct { 88 | keys []string 89 | } 90 | tests := []struct { 91 | name string 92 | args args 93 | want *PrefixMatcher 94 | wantErr bool 95 | }{ 96 | { 97 | args: args{[]string{"A", "BC"}}, 98 | want: &PrefixMatcher{ 99 | nodes: func() []node { 100 | n := make([]node, 256) 101 | n[0] = node{base: 1, check: 0} 102 | n[1+0] = node{base: 1, check: 0} 103 | n[1+26] = node{base: 1, check: 1} 104 | 105 | n[1+1] = node{base: 1, check: 0} 106 | n[1+2] = node{base: 2, check: 2} 107 | n[2+26] = node{base: 1, check: 3} 108 | 109 | return n 110 | }(), 111 | }, 112 | }, 113 | { 114 | args: args{[]string{"SELECT", "INSERT"}}, 115 | want: &PrefixMatcher{ 116 | nodes: func() []node { 117 | n := make([]node, 256) 118 | n[0] = node{base: 1, check: 0} 119 | 120 | n[1+8] = node{base: 1, check: 0} // I 121 | n[1+18] = node{base: 2, check: 0} // S 122 | 123 | n[1+13] = node{base: 2, check: 9} // N 124 | n[2+18] = node{base: 1, check: 14} // S 125 | n[1+4] = node{base: 1, check: 20} // E 126 | n[1+17] = node{base: 2, check: 5} // R 127 | n[2+19] = node{base: 1, check: 18} // T 128 | n[1+26] = node{base: 1, check: 21} // $ 129 | 130 | n[2+4] = node{base: 1, check: 19} // E 131 | n[1+11] = node{base: 3, check: 6} // L 132 | n[3+4] = node{base: 1, check: 12} // E 133 | n[1+2] = node{base: 3, check: 7} // C 134 | n[3+19] = node{base: 2, check: 3} // T 135 | n[2+26] = node{base: 1, check: 22} // $ 136 | 137 | return n 138 | }(), 139 | }, 140 | }, 141 | } 142 | for _, tt := range tests { 143 | t.Run(tt.name, func(t *testing.T) { 144 | got, err := Build(tt.args.keys) 145 | if (err != nil) != tt.wantErr { 146 | t.Errorf("Build() error = %v, wantErr %v", err, tt.wantErr) 147 | return 148 | } 149 | if !reflect.DeepEqual(got, tt.want) { 150 | t.Errorf("Build() got = %+v, want %+v", got.nodes[:32], tt.want.nodes[:32]) 151 | } 152 | }) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /replace.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "log" 8 | "sync" 9 | "time" 10 | 11 | "github.com/akito0107/xsqlparser" 12 | "github.com/akito0107/xsqlparser/dialect" 13 | "github.com/akito0107/xsqlparser/sqlast" 14 | "github.com/akito0107/xsqlparser/sqlastutil" 15 | "github.com/akito0107/xsqlparser/sqltoken" 16 | ) 17 | 18 | var tokensPool = sync.Pool{ 19 | New: func() interface{} { 20 | return make([]*sqltoken.Token, 0, 2048) 21 | }, 22 | } 23 | 24 | var tokenizerPool = sync.Pool{ 25 | New: func() interface{} { 26 | return sqltoken.NewTokenizerWithOptions(nil, sqltoken.Dialect(&dialect.MySQLDialect{}), sqltoken.DisableParseComment()) 27 | }, 28 | } 29 | 30 | func ReplaceWithZeroValue(src []byte) (string, error) { 31 | // FIXME evil work around 32 | defer func() { 33 | if err := recover(); err != nil { 34 | // log.Printf("fatal err: %v", err) 35 | log.Printf("fatal err") 36 | return 37 | } 38 | }() 39 | tokenizer := tokenizerPool.Get().(*sqltoken.Tokenizer) 40 | tokenizer.Line = 1 41 | tokenizer.Col = 1 42 | tokenizer.Scanner.Init(bytes.NewReader(src)) 43 | defer tokenizerPool.Put(tokenizer) 44 | 45 | tokset := tokensPool.Get().([]*sqltoken.Token) 46 | tokset = tokset[:0] 47 | defer func() { 48 | tokensPool.Put(tokset) 49 | }() 50 | 51 | for { 52 | var tok *sqltoken.Token 53 | if len(tokset) < cap(tokset) { 54 | tok = tokset[:len(tokset)+1][len(tokset)] 55 | } 56 | if tok == nil { 57 | tok = &sqltoken.Token{} 58 | } 59 | t, err := tokenizer.Scan(tok) 60 | if err == io.EOF { 61 | break 62 | } 63 | if err != nil { 64 | return "", fmt.Errorf("tokenize failed src: %s : %w", string(src), err) 65 | } 66 | if t == nil { 67 | continue 68 | } 69 | tokset = append(tokset, tok) 70 | } 71 | 72 | parser := xsqlparser.NewParserWithOptions() 73 | parser.SetTokens(tokset) 74 | 75 | stmt, err := parser.ParseStatement() 76 | if err != nil { 77 | log.Printf("Parse failed: invalied sql: %s \n", src[:50]) 78 | return "", err 79 | } 80 | 81 | res := sqlastutil.Apply(stmt, func(cursor *sqlastutil.Cursor) bool { 82 | switch node := cursor.Node().(type) { 83 | case *sqlast.LongValue: 84 | cursor.Replace(sqlast.NewLongValue(0)) 85 | case *sqlast.DoubleValue: 86 | cursor.Replace(sqlast.NewDoubleValue(0)) 87 | case *sqlast.BooleanValue: 88 | cursor.Replace(sqlast.NewBooleanValue(true)) 89 | case *sqlast.SingleQuotedString: 90 | cursor.Replace(sqlast.NewSingleQuotedString("")) 91 | case *sqlast.TimestampValue: 92 | cursor.Replace(sqlast.NewTimestampValue(time.Date(1970, 1, 1, 0, 0, 0, 0, nil))) 93 | case *sqlast.TimeValue: 94 | cursor.Replace(sqlast.NewTimeValue(time.Date(1970, 1, 1, 0, 0, 0, 0, nil))) 95 | case *sqlast.DateTimeValue: 96 | cursor.Replace(sqlast.NewDateTimeValue(time.Date(1970, 1, 1, 0, 0, 0, 0, nil))) 97 | case *sqlast.InList: 98 | cursor.Replace(&sqlast.InList{ 99 | Expr: node.Expr, 100 | Negated: node.Negated, 101 | RParen: node.RParen, 102 | }) 103 | } 104 | return true 105 | }, nil) 106 | return res.ToSQLString(), nil 107 | } 108 | -------------------------------------------------------------------------------- /slowquery.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "log" 9 | "strconv" 10 | "strings" 11 | "unsafe" 12 | 13 | "github.com/stuartcarnie/go-simd/unicode/utf8" 14 | 15 | "github.com/akito0107/querydigest/internal/dart" 16 | ) 17 | 18 | type SlowQueryScanner struct { 19 | reader *bufio.Reader 20 | line string 21 | currentInfo SlowQueryInfo 22 | err error 23 | queryBuf *bytes.Buffer 24 | queryTimeBuf *bytes.Buffer 25 | } 26 | 27 | const ioBufSize = 128 * 1024 * 1024 28 | 29 | func NewSlowQueryScanner(r io.Reader) *SlowQueryScanner { 30 | return &SlowQueryScanner{ 31 | reader: bufio.NewReaderSize(r, ioBufSize), 32 | queryBuf: &bytes.Buffer{}, 33 | queryTimeBuf: &bytes.Buffer{}, 34 | } 35 | } 36 | 37 | func (s *SlowQueryScanner) SlowQueryInfo() *SlowQueryInfo { 38 | return &s.currentInfo 39 | } 40 | 41 | func (s *SlowQueryScanner) Err() error { 42 | return s.err 43 | } 44 | 45 | func (s *SlowQueryScanner) Next() bool { 46 | if s.err != nil { 47 | return false 48 | } 49 | for { 50 | for !strings.HasPrefix(s.line, "# Time:") { 51 | if err := s.nextLine(); err == io.EOF { 52 | return false 53 | } else if err != nil { 54 | s.err = err 55 | return false 56 | } 57 | } 58 | 59 | if err := s.nextLine(); err != nil { 60 | s.err = err 61 | return false 62 | } 63 | 64 | if err := s.nextLine(); err != nil { 65 | s.err = err 66 | return false 67 | } 68 | 69 | s.queryTimeBuf.Reset() 70 | s.queryTimeBuf.WriteString(s.line) 71 | 72 | for { 73 | if err := s.nextLine(); err == io.EOF { 74 | return false 75 | } else if err != nil { 76 | s.err = err 77 | return false 78 | } 79 | 80 | s.queryBuf.Reset() 81 | 82 | for { 83 | s.queryBuf.WriteString(s.line) 84 | if strings.HasSuffix(s.line, ";") { 85 | break 86 | } 87 | if err := s.nextLine(); err != nil { 88 | s.err = err 89 | return false 90 | } 91 | } 92 | 93 | b := s.queryBuf.Bytes() 94 | if len(b) > 6 && parsableQueryLine(b[:6]) { 95 | if cap(s.currentInfo.RawQuery) < len(b) { 96 | s.currentInfo.RawQuery = make([]byte, len(b)) 97 | } 98 | s.currentInfo.RawQuery = s.currentInfo.RawQuery[:len(b)] 99 | copy(s.currentInfo.RawQuery, b) 100 | parseQueryTime(&s.currentInfo.QueryTime, unsafeString(s.queryTimeBuf.Bytes())) 101 | return true 102 | } else if strings.HasPrefix(s.line, "#") { 103 | break 104 | } 105 | } 106 | } 107 | } 108 | 109 | func (s *SlowQueryScanner) nextLine() error { 110 | l, _, err := s.reader.ReadLine() 111 | if err != nil { 112 | return err 113 | } 114 | if utf8.Valid(l) { 115 | s.line = unsafeString(l) 116 | } else { 117 | s.line = fmt.Sprintf("%q", l) 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func unsafeString(b []byte) string { 124 | return *(*string)(unsafe.Pointer(&b)) 125 | } 126 | 127 | var supportedSQLs = dart.Must(dart.Build([]string{ 128 | "SELECT", 129 | "INSERT", 130 | "UPDATE", 131 | "DELETE", 132 | "WITH", 133 | "ALTER", 134 | })) 135 | 136 | func parsableQueryLine(b []byte) bool { 137 | return supportedSQLs.Match(b) 138 | } 139 | 140 | type QueryTime struct { 141 | QueryTime float64 142 | LockTime float64 143 | RowsSent int 144 | RowsExamined int 145 | } 146 | 147 | type SlowQueryInfo struct { 148 | ParsedQuery string 149 | RawQuery []byte 150 | QueryTime QueryTime 151 | } 152 | 153 | func (i *SlowQueryInfo) clone() *SlowQueryInfo { 154 | rawQuery := make([]byte, len(i.RawQuery)) 155 | copy(rawQuery, i.RawQuery) 156 | return &SlowQueryInfo{ 157 | RawQuery: rawQuery, 158 | QueryTime: i.QueryTime, 159 | } 160 | } 161 | 162 | func parseHeader(str string) (queryTime, lockTime, rowsSent, rowsExamined string) { 163 | // skip `# Query_time: ` 164 | str = str[14:] 165 | 166 | var i int 167 | for i = 0; i < len(str); i++ { 168 | if str[i] == ' ' { 169 | break 170 | } 171 | } 172 | queryTime = str[:i] 173 | str = str[i+13:] 174 | for i = 0; i < len(str); i++ { 175 | if str[i] == ' ' { 176 | break 177 | } 178 | } 179 | lockTime = str[:i] 180 | str = str[i+12:] 181 | for i = 0; i < len(str); i++ { 182 | if str[i] == ' ' { 183 | break 184 | } 185 | } 186 | rowsSent = str[:i] 187 | rowsExamined = str[i+17:] 188 | 189 | return queryTime, lockTime, rowsSent, rowsExamined 190 | } 191 | 192 | func parseQueryTime(q *QueryTime, str string) { 193 | 194 | queryTime, lockTime, rowsSent, rowsExamined := parseHeader(str) 195 | 196 | // queryTimes := strings.SplitN(str, ":", 5) 197 | // Query_time 198 | qt, err := strconv.ParseFloat(queryTime, 64) 199 | if err != nil { 200 | log.Fatal(err) 201 | } 202 | // Lock_time 203 | lt, err := strconv.ParseFloat(lockTime, 64) 204 | if err != nil { 205 | log.Fatal(err) 206 | } 207 | // Rows_sent 208 | rs, err := strconv.ParseInt(rowsSent, 10, 64) 209 | if err != nil { 210 | log.Fatal(err) 211 | } 212 | // Rows_examined 213 | re, err := strconv.ParseInt(rowsExamined, 10, 64) 214 | if err != nil { 215 | log.Fatal(err) 216 | } 217 | q.QueryTime = qt 218 | q.LockTime = lt 219 | q.RowsSent = int(rs) 220 | q.RowsExamined = int(re) 221 | } 222 | -------------------------------------------------------------------------------- /slowquery_test.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "testing" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | ) 10 | 11 | func TestSlowQueryScanner_Next(t *testing.T) { 12 | 13 | // test fixtures created by using https://github.com/isucon/isucon9-qualify application. 14 | cases := []struct { 15 | name string 16 | fixturesPath string 17 | expect SlowQueryInfo 18 | }{ 19 | { 20 | name: "header", 21 | fixturesPath: "header", 22 | expect: SlowQueryInfo{ 23 | RawQuery: bytes.NewBufferString("select @@version_comment limit 1;").Bytes(), 24 | QueryTime: QueryTime{ 25 | QueryTime: 0.000126, 26 | LockTime: 0, 27 | RowsSent: 1, 28 | RowsExamined: 0, 29 | }, 30 | }, 31 | }, 32 | { 33 | name: "insert", 34 | fixturesPath: "insert", 35 | expect: SlowQueryInfo{ 36 | RawQuery: bytes.NewBufferString("INSERT INTO categories (`id`,`parent_id`,`category_name`) VALUES" + 37 | "(1,0,\"ソファー\")," + 38 | "(2,1,\"一人掛けソファー\")," + 39 | "(3,1,\"二人掛けソファー\")," + 40 | "(4,1,\"コーナーソファー\");").Bytes(), 41 | QueryTime: QueryTime{ 42 | QueryTime: 0.012964, 43 | LockTime: 0.001197, 44 | RowsSent: 0, 45 | RowsExamined: 0, 46 | }, 47 | }, 48 | }, 49 | } 50 | 51 | for _, c := range cases { 52 | t.Run(c.name, func(t *testing.T) { 53 | f, err := os.Open("./testdata/mysql-slow." + c.fixturesPath + ".log") 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | defer f.Close() 58 | 59 | scanner := NewSlowQueryScanner(f) 60 | 61 | scanner.Next() 62 | 63 | if scanner.Err() != nil { 64 | t.Fatal(scanner.Err()) 65 | } 66 | 67 | info := scanner.SlowQueryInfo() 68 | 69 | if info == nil { 70 | t.Fatal("info is nil") 71 | } 72 | 73 | if diff := cmp.Diff(*info, c.expect); diff != "" { 74 | t.Errorf("diff: %s", diff) 75 | } 76 | }) 77 | } 78 | } 79 | 80 | func Test_parseHeader(t *testing.T) { 81 | 82 | src := `# Query_time: 0.004370 Lock_time: 0.001289 Rows_sent: 2 Rows_examined: 2` 83 | 84 | queryTime, lockTime, rowsSent, rowsExamined := parseHeader(src) 85 | 86 | if queryTime != "0.004370" { 87 | t.Errorf("expect: `%s` but `%s`", "0.004370", queryTime) 88 | } 89 | if lockTime != "0.001289" { 90 | t.Errorf("expect: `%s` but `%s`", "0.001289", lockTime) 91 | } 92 | if rowsSent != "2" { 93 | t.Errorf("expect: `%s` but `%s`", "2", rowsSent) 94 | } 95 | if rowsExamined != "2" { 96 | t.Errorf("expect: `%s` but `%s`", "2", rowsExamined) 97 | } 98 | 99 | } 100 | 101 | func BenchmarkSlowQueryScanner_SlowQueryInfo(b *testing.B) { 102 | b.ResetTimer() 103 | 104 | for i := 0; i < b.N; i++ { 105 | f, err := os.Open("./benchdata/mysql-slow.log") 106 | if err != nil { 107 | b.Fatal(err) 108 | } 109 | sc := NewSlowQueryScanner(f) 110 | 111 | for sc.Next() { 112 | } 113 | 114 | if err := sc.Err(); err != nil { 115 | b.Fatal(err) 116 | } 117 | 118 | f.Close() 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /stat.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "sort" 7 | "strings" 8 | ) 9 | 10 | type Histogram []float64 11 | 12 | var divider = []float64{1, 10, 100, 1000, 10000, 100000, 1000000, math.Inf(0)} 13 | var dividerLabel = []string{ 14 | " 1us", 15 | " 10us", 16 | "100us", 17 | " 1ms", 18 | " 10ms", 19 | "100ms", 20 | " 1s", 21 | " 10s~", 22 | } 23 | 24 | const maxLength = 75 25 | 26 | func (h Histogram) String() string { 27 | tmp := make([]float64, len(h)) 28 | copy(tmp, h) 29 | 30 | sort.Float64Slice(tmp).Sort() 31 | max := tmp[len(h)-1] 32 | unit := maxLength / max 33 | 34 | var b strings.Builder 35 | 36 | for i := 0; i < len(divider); i++ { 37 | if len(h) <= i { 38 | fmt.Fprintf(&b, "%s:\t\n", dividerLabel[i]) 39 | continue 40 | } 41 | length := int(unit * h[i]) 42 | if length <= 0 { 43 | length = 0 44 | } 45 | fmt.Fprintf(&b, "%s:\t%s\n", dividerLabel[i], strings.Repeat("#", length)) 46 | } 47 | 48 | return b.String() 49 | } 50 | -------------------------------------------------------------------------------- /stat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +eu 4 | 5 | old=${2} 6 | current=${3} 7 | 8 | benchstat "benchresult/${1}/${old}/${1}.txt" "benchresult/${1}/${current}/${1}.txt" 9 | -------------------------------------------------------------------------------- /summarizer.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | ) 7 | 8 | type Summarizer struct { 9 | m map[string]*SlowQuerySummary 10 | mu sync.Mutex 11 | totalTime float64 12 | } 13 | 14 | func NewSummarizer() *Summarizer { 15 | return &Summarizer{ 16 | m: make(map[string]*SlowQuerySummary), 17 | } 18 | } 19 | 20 | func (s *Summarizer) Map() map[string]*SlowQuerySummary { 21 | return s.m 22 | } 23 | 24 | func (s *Summarizer) TotalQueryTime() float64 { 25 | return s.totalTime 26 | } 27 | 28 | func (s *Summarizer) Collect(i *SlowQueryInfo) { 29 | s.mu.Lock() 30 | summary, ok := s.m[i.ParsedQuery] 31 | if !ok { 32 | summary = &SlowQuerySummary{ 33 | RowSample: string(i.RawQuery), 34 | } 35 | } 36 | summary.appendQueryTime(i) 37 | s.m[i.ParsedQuery] = summary 38 | s.totalTime += i.QueryTime.QueryTime 39 | s.mu.Unlock() 40 | } 41 | 42 | func (s *Summarizer) Summarize() []*SlowQuerySummary { 43 | qs := make([]*SlowQuerySummary, 0, len(s.m)) 44 | for _, v := range s.m { 45 | v.ComputeHistogram() 46 | v.ComputeStats() 47 | qs = append(qs, v) 48 | } 49 | 50 | sort.Slice(qs, func(i, j int) bool { 51 | return qs[i].TotalTime > qs[j].TotalTime 52 | }) 53 | 54 | return qs 55 | } 56 | -------------------------------------------------------------------------------- /summary.go: -------------------------------------------------------------------------------- 1 | package querydigest 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "sort" 7 | "strings" 8 | 9 | "github.com/jedib0t/go-pretty/table" 10 | "gonum.org/v1/gonum/stat" 11 | ) 12 | 13 | type SlowQuerySummary struct { 14 | RowSample string 15 | TotalTime float64 16 | TotalLockTime float64 17 | TotalQueryCount int 18 | TotalRowsSent int 19 | TotalRowsExamined int 20 | QueryTimes []QueryTime 21 | stats *slowQueryStats 22 | queryTimeHistogram Histogram 23 | } 24 | 25 | func (s *SlowQuerySummary) String() string { 26 | var b strings.Builder 27 | 28 | fmt.Fprintf(&b, "Summary:\n") 29 | fmt.Fprintf(&b, "total query time:\t%0.2fs\n", s.TotalTime) 30 | fmt.Fprintf(&b, "total query count:\t%d\n\n", s.TotalQueryCount) 31 | 32 | fmt.Fprintf(&b, "%s\n", s.stats) 33 | 34 | fmt.Fprintf(&b, "Query_time distribution:\n%v\n", s.queryTimeHistogram) 35 | 36 | fmt.Fprintf(&b, "QueryExample:\n%s\n", s.RowSample) 37 | 38 | return b.String() 39 | } 40 | 41 | func (s *SlowQuerySummary) ComputeStats() { 42 | queryTimes := make([]float64, 0, len(s.QueryTimes)) 43 | lockTimes := make([]float64, 0, len(s.QueryTimes)) 44 | rowsSents := make([]float64, 0, len(s.QueryTimes)) 45 | rowsExamines := make([]float64, 0, len(s.QueryTimes)) 46 | 47 | for _, r := range s.QueryTimes { 48 | queryTimes = append(queryTimes, r.QueryTime) 49 | lockTimes = append(lockTimes, r.LockTime) 50 | rowsSents = append(rowsSents, float64(r.RowsSent)) 51 | rowsExamines = append(rowsExamines, float64(r.RowsExamined)) 52 | } 53 | 54 | s.stats = &slowQueryStats{ 55 | ExecTime: computeStatSeconds("Exec Time", queryTimes, s.TotalTime), 56 | LockTime: computeStatSeconds("Lock Time", lockTimes, s.TotalLockTime), 57 | RowsSent: computeStatCount("Rows Sent", rowsSents, float64(s.TotalRowsSent)), 58 | RowsExamine: computeStatCount("Rows Examine", rowsExamines, float64(s.TotalRowsExamined)), 59 | } 60 | } 61 | 62 | func (s *SlowQuerySummary) ComputeHistogram() { 63 | src := make([]float64, 0, len(s.QueryTimes)) 64 | for _, r := range s.QueryTimes { 65 | qus := r.QueryTime * 1000 * 1000 66 | if qus > 1 { 67 | src = append(src, qus) 68 | } 69 | } 70 | 71 | sort.Float64Slice(src).Sort() 72 | 73 | hist := stat.Histogram(nil, divider, src, nil) 74 | 75 | s.queryTimeHistogram = Histogram(hist) 76 | } 77 | 78 | func (s *SlowQuerySummary) appendQueryTime(info *SlowQueryInfo) { 79 | s.TotalLockTime += info.QueryTime.LockTime 80 | s.TotalTime += info.QueryTime.QueryTime 81 | s.TotalRowsSent += info.QueryTime.RowsSent 82 | s.TotalRowsExamined += info.QueryTime.RowsExamined 83 | s.QueryTimes = append(s.QueryTimes, info.QueryTime) 84 | 85 | s.TotalQueryCount++ 86 | } 87 | 88 | func computeStatSeconds(label string, x []float64, total float64) slowQueryStatSeconds { 89 | if len(x) == 0 { 90 | return slowQueryStatSeconds{label: label, total: seconds(total)} 91 | } 92 | 93 | sort.Float64Slice(x).Sort() 94 | 95 | // var min float64 96 | // for _, f := range x { 97 | // if f > 0 { 98 | // min = f 99 | // break 100 | // } 101 | // } 102 | min := x[0] 103 | max := x[len(x)-1] 104 | median := stat.Quantile(0.5, stat.Empirical, x, nil) 105 | avg := total / float64(len(x)) 106 | stddev := stat.StdDev(x, nil) 107 | quantile := stat.Quantile(0.95, stat.Empirical, x, nil) 108 | 109 | return slowQueryStatSeconds{ 110 | label: label, 111 | total: seconds(total), 112 | min: seconds(min), 113 | max: seconds(max), 114 | avg: seconds(avg), 115 | stddev: seconds(stddev), 116 | quantile: seconds(quantile), 117 | median: seconds(median), 118 | } 119 | } 120 | 121 | func computeStatCount(label string, x []float64, total float64) slowQueryStatCount { 122 | if len(x) == 0 { 123 | return slowQueryStatCount{label: label, total: count(total)} 124 | } 125 | 126 | sort.Float64Slice(x).Sort() 127 | 128 | min := x[0] 129 | max := x[len(x)-1] 130 | median := stat.Quantile(0.5, stat.LinInterp, x, nil) 131 | avg := total / float64(len(x)-1) 132 | stddev := stat.StdDev(x, nil) 133 | quantile := stat.Quantile(0.95, stat.LinInterp, x, nil) 134 | 135 | return slowQueryStatCount{ 136 | label: label, 137 | total: count(total), 138 | min: count(min), 139 | max: count(max), 140 | avg: count(avg), 141 | stddev: count(stddev), 142 | quantile: count(quantile), 143 | median: count(median), 144 | } 145 | } 146 | 147 | type slowQueryStats struct { 148 | ExecTime slowQueryStatSeconds 149 | LockTime slowQueryStatSeconds 150 | RowsSent slowQueryStatCount 151 | RowsExamine slowQueryStatCount 152 | } 153 | 154 | func (s *slowQueryStats) String() string { 155 | var b strings.Builder 156 | t := table.NewWriter() 157 | 158 | t.SetOutputMirror(&b) 159 | t.AppendHeader(table.Row{"Attribute", "total", "min", "max", "avg", "95%", "stddev", "median"}) 160 | t.AppendRows([]table.Row{ 161 | {s.ExecTime.label, s.ExecTime.total, s.ExecTime.min, s.ExecTime.max, s.ExecTime.avg, s.ExecTime.quantile, s.ExecTime.stddev, s.ExecTime.median}, 162 | {s.LockTime.label, s.LockTime.total, s.LockTime.min, s.LockTime.max, s.LockTime.avg, s.LockTime.quantile, s.LockTime.stddev, s.LockTime.median}, 163 | {s.RowsSent.label, s.RowsSent.total, s.RowsSent.min, s.RowsSent.max, s.RowsSent.avg, s.RowsSent.quantile, s.RowsSent.stddev, s.RowsSent.median}, 164 | {s.RowsExamine.label, s.RowsExamine.total, s.RowsExamine.min, s.RowsExamine.max, s.RowsExamine.avg, s.RowsExamine.quantile, s.RowsExamine.stddev, s.RowsExamine.median}, 165 | }) 166 | t.Render() 167 | 168 | return b.String() 169 | } 170 | 171 | type seconds float64 172 | 173 | func (r seconds) String() string { 174 | if math.IsNaN(float64(r)) || math.IsInf(float64(r), 0) { 175 | return "-" 176 | } 177 | 178 | nano := r * 1000 * 1000 179 | if nano < 1000 { 180 | return fmt.Sprintf("%.0fus", nano) 181 | } 182 | if nano < 1000000 { 183 | return fmt.Sprintf("%.0fms", nano/1000) 184 | } 185 | return fmt.Sprintf("%.0fs", r) 186 | } 187 | 188 | type slowQueryStatSeconds struct { 189 | label string 190 | total seconds 191 | min seconds 192 | max seconds 193 | avg seconds 194 | quantile seconds 195 | stddev seconds 196 | median seconds 197 | } 198 | 199 | type count float64 200 | 201 | func (c count) String() string { 202 | if math.IsNaN(float64(c)) || math.IsInf(float64(c), 0) { 203 | return "-" 204 | } 205 | return fmt.Sprintf("%.2f", c) 206 | } 207 | 208 | type slowQueryStatCount struct { 209 | label string 210 | total count 211 | min count 212 | max count 213 | avg count 214 | quantile count 215 | stddev count 216 | median count 217 | } 218 | -------------------------------------------------------------------------------- /testdata/mysql-slow.createtable.log: -------------------------------------------------------------------------------- 1 | /usr/sbin/mysqld, Version: 5.7.28-0ubuntu0.18.04.4-log ((Ubuntu)). started with: 2 | Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock 3 | Time Id Command Argument 4 | # Time: 2020-01-17T05:59:09.832280Z 5 | # User@Host: isucari[isucari] @ localhost [] Id: 2 6 | # Query_time: 0.000126 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 7 | SET timestamp=1579240749; 8 | select @@version_comment limit 1; 9 | # Time: 2020-01-17T05:59:12.346053Z 10 | # User@Host: isucari[isucari] @ localhost [] Id: 2 11 | # Query_time: 0.004370 Lock_time: 0.001289 Rows_sent: 2 Rows_examined: 2 12 | SET timestamp=1579240752; 13 | show databases; 14 | # Time: 2020-01-17T05:59:14.250521Z 15 | # User@Host: isucari[isucari] @ localhost [] Id: 2 16 | # Query_time: 0.000078 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 17 | SET timestamp=1579240754; 18 | SELECT DATABASE(); 19 | # Time: 2020-01-17T06:06:15.114474Z 20 | # User@Host: isucari[isucari] @ localhost [127.0.0.1] Id: 3 21 | # Query_time: 0.023334 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 22 | SET timestamp=1579241175; 23 | DROP TABLE IF EXISTS `shippings`; 24 | # Time: 2020-01-17T06:06:15.162769Z 25 | # User@Host: isucari[isucari] @ localhost [127.0.0.1] Id: 3 26 | # Query_time: 0.048253 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 27 | SET timestamp=1579241175; 28 | CREATE TABLE `shippings` ( 29 | `transaction_evidence_id` bigint NOT NULL PRIMARY KEY, 30 | `status` enum('initial', 'wait_pickup', 'shipping', 'done') NOT NULL, 31 | `item_name` varchar(191) NOT NULL, 32 | `item_id` bigint NOT NULL, 33 | `reserve_id` varchar(191) NOT NULL, 34 | `reserve_time` bigint NOT NULL, 35 | `to_address` varchar(191) NOT NULL, 36 | `to_name` varchar(191) NOT NULL, 37 | `from_address` varchar(191) NOT NULL, 38 | `from_name` varchar(191) NOT NULL, 39 | `img_binary` mediumblob NOT NULL, 40 | `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 41 | `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP 42 | ) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4; -------------------------------------------------------------------------------- /testdata/mysql-slow.header.log: -------------------------------------------------------------------------------- 1 | /usr/sbin/mysqld, Version: 5.7.28-0ubuntu0.18.04.4-log ((Ubuntu)). started with: 2 | Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock 3 | Time Id Command Argument 4 | # Time: 2020-01-17T05:59:09.832280Z 5 | # User@Host: isucari[isucari] @ localhost [] Id: 2 6 | # Query_time: 0.000126 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 7 | SET timestamp=1579240749; 8 | select @@version_comment limit 1; 9 | -------------------------------------------------------------------------------- /testdata/mysql-slow.insert.log: -------------------------------------------------------------------------------- 1 | /usr/sbin/mysqld, Version: 5.7.28-0ubuntu0.18.04.4-log ((Ubuntu)). started with: 2 | Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock 3 | Time Id Command Argument 4 | # administrator command: Init DB; 5 | # Time: 2020-01-17T06:06:15.236547Z 6 | # User@Host: isucari[isucari] @ localhost [127.0.0.1] Id: 3 7 | # Query_time: 0.012964 Lock_time: 0.001197 Rows_sent: 0 Rows_examined: 0 8 | SET timestamp=1579241175; 9 | INSERT INTO categories (`id`,`parent_id`,`category_name`) VALUES 10 | (1,0,"ソファー"), 11 | (2,1,"一人掛けソファー"), 12 | (3,1,"二人掛けソファー"), 13 | (4,1,"コーナーソファー"); --------------------------------------------------------------------------------