├── testdata ├── base.json.gz ├── day-later.json.gz ├── report-stats │ ├── ssl-access_log.200912200000.gz │ ├── ssl-access_log.200912210000.gz │ ├── ssl-access_log.200912220000.gz │ ├── ssl-access_log.200912230000.gz │ ├── ssl-access_log.200912240000.gz │ ├── ssl-access_log.200912250000.gz │ ├── ssl-access_log.200912260000.gz │ ├── ssl-access_log.200912270000.gz │ ├── ssl-access_log.200912280000.gz │ ├── ssl-access_log.200912290000.gz │ ├── ssl-access_log.200912300000.gz │ ├── ssl-access_log.200912310000.gz │ ├── ssl-access_log.201001010000.gz │ ├── ssl-access_log.201001020000.gz │ ├── ssl-access_log.201001030000.gz │ ├── ssl-access_log.201001040000.gz │ ├── ssl-access_log.201001050000.gz │ ├── ssl-access_log.201001060000.gz │ ├── ssl-access_log.201001070000.gz │ ├── ssl-access_log.201001080000.gz │ ├── ssl-access_log.201001090000.gz │ ├── ssl-access_log.201001100000.gz │ └── README.md ├── fixtures │ ├── jvm_versions.yml │ ├── job_types.yml │ ├── report_files.yml │ ├── os_types.yml │ └── jenkins_versions.yml ├── reports │ ├── JobCountsForMonth.json │ ├── GetJVMReports.json │ ├── ExecutorCountsForMonth.csv │ ├── OSCountsForMonth.json │ ├── GetInstallCountsForVersions.json │ ├── GetCapabilities.json │ ├── GetLatestPluginNumbers.json │ └── ExecutorCountsForMonth.svg ├── day-later.json └── base.json ├── etc └── migrations │ ├── 000001_initial_setup.down.sql │ └── 000001_initial_setup.up.sql ├── .gitignore ├── templates ├── versionDistroIndex.html.tmpl ├── pitIndex.html.tmpl ├── versionDistroPlugin.html.tmpl └── svgs.html.tmpl ├── cmd └── jenkins-usage-stats │ ├── main.go │ ├── report.go │ ├── import.go │ └── fetch.go ├── input_test.go ├── .golangci.yml ├── parser_test.go ├── input.go ├── integration_test.go ├── testutil └── db.go ├── go.mod ├── parser.go ├── README.md ├── Makefile ├── report_test.go ├── db_test.go ├── db.go └── report.go /testdata/base.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/base.json.gz -------------------------------------------------------------------------------- /testdata/day-later.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/day-later.json.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912200000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912200000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912210000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912210000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912220000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912220000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912230000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912230000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912240000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912240000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912250000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912250000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912260000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912260000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912270000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912270000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912280000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912280000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912290000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912290000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912300000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912300000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.200912310000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.200912310000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001010000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001010000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001020000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001020000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001030000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001030000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001040000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001040000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001050000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001050000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001060000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001060000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001070000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001070000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001080000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001080000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001090000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001090000.gz -------------------------------------------------------------------------------- /testdata/report-stats/ssl-access_log.201001100000.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jenkins-infra/jenkins-usage-stats/main/testdata/report-stats/ssl-access_log.201001100000.gz -------------------------------------------------------------------------------- /testdata/fixtures/jvm_versions.yml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | name: "1.5" 3 | - id: 2 4 | name: "1.6" 5 | - id: 3 6 | name: N/A 7 | - id: 4 8 | name: "1.7" 9 | - id: 5 10 | name: R27 11 | -------------------------------------------------------------------------------- /etc/migrations/000001_initial_setup.down.sql: -------------------------------------------------------------------------------- 1 | drop table os_types; 2 | drop table job_types; 3 | drop table plugins; 4 | drop table instance_reports; 5 | drop table jenkins_versions; 6 | drop table report_files; 7 | drop table jvm_versions; 8 | -------------------------------------------------------------------------------- /testdata/reports/JobCountsForMonth.json: -------------------------------------------------------------------------------- 1 | { 2 | "hudson-drools-DroolsProject": 2, 3 | "hudson-matrix-MatrixProject": 3133, 4 | "hudson-maven-MavenModuleSet": 48997, 5 | "hudson-model-ExternalJob": 160, 6 | "hudson-model-FreeStyleProject": 143360 7 | } -------------------------------------------------------------------------------- /testdata/reports/GetJVMReports.json: -------------------------------------------------------------------------------- 1 | { 2 | "jvmStatsPerMonth": { 3 | "1259625600000": { 4 | "1.5": 1976, 5 | "1.6": 6518, 6 | "1.7": 9 7 | }, 8 | "1262304000000": { 9 | "1.5": 1814, 10 | "1.6": 6141, 11 | "1.7": 7 12 | } 13 | }, 14 | "jvmStatsPerMonth_2.x": {} 15 | } -------------------------------------------------------------------------------- /testdata/fixtures/job_types.yml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | name: hudson-model-FreeStyleProject 3 | - id: 2 4 | name: hudson-maven-MavenModuleSet 5 | - id: 3 6 | name: hudson-matrix-MatrixProject 7 | - id: 4 8 | name: hudson-model-ExternalJob 9 | - id: 5 10 | name: hudson-drools-DroolsProject 11 | - id: 6 12 | name: hudson-plugins-BranchMonitor 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | build 18 | .idea 19 | -------------------------------------------------------------------------------- /testdata/report-stats/README.md: -------------------------------------------------------------------------------- 1 | ## report-stats 2 | 3 | This directory contains daily gzipped data to be ingested for report function testing. If any changes are made here, the directory should be ingested into a fresh database and then dumped to `testdata/fixtures` by running `make dump-fixtures`. 4 | 5 | Any changes to `testdata/fixtures` will mean that `report_test.go`'s test will need to be re-run with the `UPDATE_GOLDEN` env var set to `true` to update the contents of `testdata/reports` as well. 6 | -------------------------------------------------------------------------------- /templates/versionDistroIndex.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | Install Counts by Plugin Version and Jenkins Version 4 | 9 | 10 | 11 |

Install Counts by Plugin Version and Jenkins Version

12 | 13 |

Plugins

14 | 17 |
18 |
19 | This page generated by 20 | Jenkins Usage Stats 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /templates/pitIndex.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | Installation Trends JSON 4 | 9 | 10 | 11 |

Installation Trends JSON

12 | 13 | 16 |

Plugins

17 | 20 |
21 |
22 | This page generated by 23 | this script 24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /templates/versionDistroPlugin.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | Plugin and Core Version Matrix for the {{.pluginName}} Plugin 4 | 5 | 9 | 10 | 17 | 18 | 19 |
20 | 21 | 22 | -------------------------------------------------------------------------------- /testdata/reports/ExecutorCountsForMonth.csv: -------------------------------------------------------------------------------- 1 | "0","211" 2 | "1","1138" 3 | "2","4769" 4 | "3","533" 5 | "4","508" 6 | "5","283" 7 | "6","175" 8 | "7","85" 9 | "8","122" 10 | "9","56" 11 | "10","101" 12 | "11","35" 13 | "12","40" 14 | "13","26" 15 | "14","21" 16 | "15","20" 17 | "16","28" 18 | "17","17" 19 | "18","9" 20 | "19","12" 21 | "20","15" 22 | "21","6" 23 | "22","7" 24 | "23","9" 25 | "24","10" 26 | "25","11" 27 | "26","5" 28 | "27","6" 29 | "28","5" 30 | "29","4" 31 | "30","6" 32 | "32","3" 33 | "33","2" 34 | "34","3" 35 | "35","1" 36 | "36","4" 37 | "37","1" 38 | "38","2" 39 | "39","2" 40 | "40","5" 41 | "41","2" 42 | "42","2" 43 | "43","1" 44 | "44","2" 45 | "45","1" 46 | "47","3" 47 | "48","1" 48 | "49","1" 49 | "50","2" 50 | "51","1" 51 | "52","2" 52 | "53","1" 53 | "55","1" 54 | "56","1" 55 | "58","1" 56 | "59","1" 57 | "61","2" 58 | "63","1" 59 | "66","1" 60 | "67","2" 61 | "69","1" 62 | "76","2" 63 | "77","1" 64 | "79","1" 65 | "108","1" 66 | "114","1" 67 | "138","1" 68 | "152","1" 69 | "192","1" 70 | -------------------------------------------------------------------------------- /testdata/fixtures/report_files.yml: -------------------------------------------------------------------------------- 1 | - filename: ssl-access_log.200912200000.gz 2 | - filename: ssl-access_log.200912210000.gz 3 | - filename: ssl-access_log.200912220000.gz 4 | - filename: ssl-access_log.200912230000.gz 5 | - filename: ssl-access_log.200912240000.gz 6 | - filename: ssl-access_log.200912250000.gz 7 | - filename: ssl-access_log.200912260000.gz 8 | - filename: ssl-access_log.200912270000.gz 9 | - filename: ssl-access_log.200912280000.gz 10 | - filename: ssl-access_log.200912290000.gz 11 | - filename: ssl-access_log.200912300000.gz 12 | - filename: ssl-access_log.200912310000.gz 13 | - filename: ssl-access_log.201001010000.gz 14 | - filename: ssl-access_log.201001020000.gz 15 | - filename: ssl-access_log.201001030000.gz 16 | - filename: ssl-access_log.201001040000.gz 17 | - filename: ssl-access_log.201001050000.gz 18 | - filename: ssl-access_log.201001060000.gz 19 | - filename: ssl-access_log.201001070000.gz 20 | - filename: ssl-access_log.201001080000.gz 21 | - filename: ssl-access_log.201001090000.gz 22 | - filename: ssl-access_log.201001100000.gz 23 | -------------------------------------------------------------------------------- /cmd/jenkins-usage-stats/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "os" 8 | 9 | sq "github.com/Masterminds/squirrel" 10 | 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func main() { 15 | if err := run(context.Background()); err != nil { 16 | fmt.Println(err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | func run(ctx context.Context) error { 22 | rootCmd := &cobra.Command{ 23 | Use: "jenkins-usage-stats", 24 | Short: "Command for running the Jenkins usage stats import and report generation", 25 | Run: func(cmd *cobra.Command, args []string) { 26 | _ = cmd.Help() 27 | }, 28 | DisableAutoGenTag: true, 29 | } 30 | 31 | rootCmd.AddCommand(NewImportCmd()) 32 | rootCmd.AddCommand(NewReportCmd()) 33 | rootCmd.AddCommand(NewFetchCmd(ctx)) 34 | 35 | return rootCmd.Execute() 36 | } 37 | 38 | func getDatabase(dbURL string) (sq.DBProxyBeginner, func(), error) { 39 | rawDB, err := sql.Open("postgres", dbURL) 40 | if err != nil { 41 | return nil, nil, err 42 | } 43 | 44 | return sq.NewStmtCacheProxy(rawDB), func() { 45 | _ = rawDB.Close() 46 | }, nil 47 | } 48 | -------------------------------------------------------------------------------- /input_test.go: -------------------------------------------------------------------------------- 1 | package stats_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | stats "github.com/jenkins-infra/jenkins-usage-stats" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestTimestampFuncs(t *testing.T) { 12 | testCases := []struct { 13 | orig string 14 | reorganized string 15 | timestamp time.Time 16 | err error 17 | }{ 18 | { 19 | orig: "14/Feb/2008:03:44:55 +0000", 20 | reorganized: "2008-02-14T03:44:55+00:00", 21 | timestamp: time.Date(2008, time.February, 14, 3, 44, 55, 0, time.UTC), 22 | }, 23 | { 24 | orig: "02/Mar/2014:21:23:59 -0700", 25 | reorganized: "2014-03-02T21:23:59-07:00", 26 | timestamp: time.Date(2014, time.March, 3, 4, 23, 59, 0, time.UTC), 27 | }, 28 | } 29 | 30 | for _, tc := range testCases { 31 | t.Run(tc.orig, func(t *testing.T) { 32 | assert.Equal(t, tc.reorganized, stats.JSONTimestampToRFC3339(tc.orig)) 33 | r := &stats.JSONReport{TimestampString: tc.orig} 34 | ts, err := r.Timestamp() 35 | if tc.err != nil { 36 | assert.Equal(t, tc.err, err) 37 | } else { 38 | assert.Equal(t, tc.timestamp, ts) 39 | } 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /testdata/reports/OSCountsForMonth.json: -------------------------------------------------------------------------------- 1 | { 2 | "AIX (ppc)": 44, 3 | "AIX (ppc64)": 37, 4 | "Darwin (i386)": 1, 5 | "FreeBSD (amd64)": 13, 6 | "FreeBSD (i386)": 15, 7 | "HP-UX (IA64N)": 52, 8 | "HP-UX (IA64W)": 1, 9 | "HP-UX (PA_RISC2.0)": 30, 10 | "Linux (amd64)": 2166, 11 | "Linux (arm)": 2, 12 | "Linux (i386)": 3706, 13 | "Linux (ia64)": 10, 14 | "Linux (ppc)": 9, 15 | "Linux (ppc64)": 4, 16 | "Linux (s390x)": 1, 17 | "Linux (sparc)": 1, 18 | "Linux (x86)": 46, 19 | "Mac OS X (i386)": 130, 20 | "Mac OS X (ppc)": 23, 21 | "Mac OS X (x86_64)": 98, 22 | "N/A": 3450, 23 | "OpenBSD (i386)": 3, 24 | "SunOS (amd64)": 9, 25 | "SunOS (sparc)": 410, 26 | "SunOS (sparcv9)": 15, 27 | "SunOS (x86)": 386, 28 | "Windows 2000 (x86)": 86, 29 | "Windows 2003 (amd64)": 139, 30 | "Windows 2003 (x86)": 1610, 31 | "Windows 7 (amd64)": 14, 32 | "Windows 7 (x86)": 36, 33 | "Windows NT (unknown) (x86)": 3, 34 | "Windows Server 2003 (amd64)": 3, 35 | "Windows Server 2003 (x86)": 30, 36 | "Windows Server 2008 (amd64)": 58, 37 | "Windows Server 2008 (ia64)": 1, 38 | "Windows Server 2008 (x86)": 136, 39 | "Windows Server 2008 R2 (amd64)": 9, 40 | "Windows Server 2008 R2 (x86)": 6, 41 | "Windows Vista (amd64)": 21, 42 | "Windows Vista (x86)": 151, 43 | "Windows XP (amd64)": 28, 44 | "Windows XP (x86)": 1739, 45 | "linux (amd64)": 1, 46 | "z/OS (s390)": 2 47 | } -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # timeout for analysis, e.g. 30s, 5m, default is 1m 3 | deadline: 10m 4 | 5 | #build-tags: 6 | 7 | # which dirs to skip: they won't be analyzed; 8 | # can use regexp here: generated.*, regexp is applied on full path; 9 | # default value is empty list, but next dirs are always skipped independently 10 | # from this option's value: 11 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 12 | #skip-dirs: 13 | 14 | # which files to skip: they will be analyzed, but issues from them 15 | # won't be reported. Default value is empty list, but there is 16 | # no need to include all autogenerated files, we confidently recognize 17 | # autogenerated files. If it's not please let us know. 18 | #skip-files: 19 | 20 | linters: 21 | enable: 22 | - depguard 23 | - errcheck 24 | - goconst 25 | - gofmt 26 | - goimports 27 | - gosec 28 | - govet 29 | - ineffassign 30 | - megacheck 31 | - misspell 32 | - revive 33 | - structcheck 34 | - varcheck 35 | include: 36 | # Enable off-by-default rules for revive requiring that all exported elements have a properly formatted comment. 37 | - EXC0012 38 | - EXC0014 39 | 40 | issues: 41 | exclude-use-default: false 42 | # List of regexps of issue texts to exclude, empty list by default. 43 | # But independently from this option we use default exclude patterns, 44 | # it can be disabled by `exclude-use-default: false`. To list all 45 | # excluded by default patterns execute `golangci-lint run --help` 46 | # exclude: [] 47 | -------------------------------------------------------------------------------- /testdata/reports/GetInstallCountsForVersions.json: -------------------------------------------------------------------------------- 1 | { 2 | "installations": { 3 | "1.264": 8, 4 | "1.265": 5, 5 | "1.266": 8, 6 | "1.267": 2, 7 | "1.268": 14, 8 | "1.270": 15, 9 | "1.272": 2, 10 | "1.273": 4, 11 | "1.274": 18, 12 | "1.275": 4, 13 | "1.276": 15, 14 | "1.277": 17, 15 | "1.278": 31, 16 | "1.279": 15, 17 | "1.280": 20, 18 | "1.281": 5, 19 | "1.282": 20, 20 | "1.283": 3, 21 | "1.284": 8, 22 | "1.285": 39, 23 | "1.286": 19, 24 | "1.287": 6, 25 | "1.288": 19, 26 | "1.289": 7, 27 | "1.290": 31, 28 | "1.291": 29, 29 | "1.292": 45, 30 | "1.292.17": 1, 31 | "1.293": 49, 32 | "1.294": 8, 33 | "1.295": 29, 34 | "1.296": 13, 35 | "1.297": 24, 36 | "1.299": 45, 37 | "1.300": 66, 38 | "1.301": 41, 39 | "1.302": 2, 40 | "1.303": 117, 41 | "1.304": 65, 42 | "1.305": 1, 43 | "1.306": 64, 44 | "1.307": 70, 45 | "1.308": 18, 46 | "1.309": 138, 47 | "1.310": 85, 48 | "1.311": 48, 49 | "1.312": 150, 50 | "1.313": 69, 51 | "1.314": 107, 52 | "1.315": 74, 53 | "1.316": 102, 54 | "1.317": 105, 55 | "1.318": 121, 56 | "1.319": 111, 57 | "1.320": 114, 58 | "1.321": 131, 59 | "1.322": 154, 60 | "1.323": 341, 61 | "1.324": 205, 62 | "1.325": 24, 63 | "1.326": 103, 64 | "1.327": 217, 65 | "1.328": 251, 66 | "1.329": 208, 67 | "1.330": 225, 68 | "1.331": 78, 69 | "1.332": 292, 70 | "1.333": 341, 71 | "1.334": 259, 72 | "1.335": 382, 73 | "1.336": 938, 74 | "1.337": 878, 75 | "1.338": 877, 76 | "1.339": 688 77 | } 78 | } -------------------------------------------------------------------------------- /parser_test.go: -------------------------------------------------------------------------------- 1 | package stats_test 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | "time" 7 | 8 | stats "github.com/jenkins-infra/jenkins-usage-stats" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestParseDailyJSON(t *testing.T) { 14 | fooFile := filepath.Join("testdata", "base.json.gz") 15 | 16 | reports, err := stats.ParseDailyJSON(fooFile) 17 | require.NoError(t, err) 18 | assert.Len(t, reports, 2) 19 | assert.Equal(t, "32b68faa8644852c4ad79540b4bfeb1caf63284811f4f9d6c2bc511f797218c8", reports[0].Install) 20 | assert.Equal(t, uint64(50), reports[0].Jobs["hudson-maven-MavenModuleSet"]) 21 | assert.Len(t, reports[0].Plugins, 75) 22 | assert.Equal(t, "1.8", reports[0].Nodes[0].JVMVersion) 23 | assert.Equal(t, "1.6", reports[1].Nodes[0].JVMVersion) 24 | 25 | ts, err := reports[0].Timestamp() 26 | require.NoError(t, err) 27 | assert.Equal(t, time.Date(2021, time.October, 30, 23, 59, 54, 0, time.UTC), ts) 28 | } 29 | 30 | func TestFilterPrivateFromReport(t *testing.T) { 31 | report := &stats.JSONReport{ 32 | Plugins: []stats.JSONPlugin{ 33 | { 34 | Name: "legit-plugin", 35 | Version: "1.2.3", 36 | }, 37 | { 38 | Name: "privateplugin-something", 39 | Version: "1.2.3", 40 | }, 41 | { 42 | Name: "other-legit-plugin", 43 | Version: "2.3.4 (private)", 44 | }, 45 | { 46 | Name: "final-legit-plugin", 47 | Version: "4.5.6", 48 | }, 49 | }, 50 | } 51 | 52 | stats.FilterPrivateFromReport(report) 53 | 54 | assert.Len(t, report.Plugins, 2) 55 | assert.Equal(t, report.Plugins[0].Name, "legit-plugin") 56 | assert.Equal(t, report.Plugins[1].Name, "final-legit-plugin") 57 | } 58 | -------------------------------------------------------------------------------- /testdata/reports/GetCapabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "installations": { 3 | "1.264": 8838, 4 | "1.265": 8830, 5 | "1.266": 8825, 6 | "1.267": 8817, 7 | "1.268": 8815, 8 | "1.270": 8801, 9 | "1.272": 8786, 10 | "1.273": 8784, 11 | "1.274": 8780, 12 | "1.275": 8762, 13 | "1.276": 8758, 14 | "1.277": 8743, 15 | "1.278": 8726, 16 | "1.279": 8695, 17 | "1.280": 8680, 18 | "1.281": 8660, 19 | "1.282": 8655, 20 | "1.283": 8635, 21 | "1.284": 8632, 22 | "1.285": 8624, 23 | "1.286": 8585, 24 | "1.287": 8566, 25 | "1.288": 8560, 26 | "1.289": 8541, 27 | "1.290": 8534, 28 | "1.291": 8503, 29 | "1.292": 8474, 30 | "1.292.17": 8429, 31 | "1.293": 8428, 32 | "1.294": 8379, 33 | "1.295": 8371, 34 | "1.296": 8342, 35 | "1.297": 8329, 36 | "1.299": 8305, 37 | "1.300": 8260, 38 | "1.301": 8194, 39 | "1.302": 8153, 40 | "1.303": 8151, 41 | "1.304": 8034, 42 | "1.305": 7969, 43 | "1.306": 7968, 44 | "1.307": 7904, 45 | "1.308": 7834, 46 | "1.309": 7816, 47 | "1.310": 7678, 48 | "1.311": 7593, 49 | "1.312": 7545, 50 | "1.313": 7395, 51 | "1.314": 7326, 52 | "1.315": 7219, 53 | "1.316": 7145, 54 | "1.317": 7043, 55 | "1.318": 6938, 56 | "1.319": 6817, 57 | "1.320": 6706, 58 | "1.321": 6592, 59 | "1.322": 6461, 60 | "1.323": 6307, 61 | "1.324": 5966, 62 | "1.325": 5761, 63 | "1.326": 5737, 64 | "1.327": 5634, 65 | "1.328": 5417, 66 | "1.329": 5166, 67 | "1.330": 4958, 68 | "1.331": 4733, 69 | "1.332": 4655, 70 | "1.333": 4363, 71 | "1.334": 4022, 72 | "1.335": 3763, 73 | "1.336": 3381, 74 | "1.337": 2443, 75 | "1.338": 1565, 76 | "1.339": 688 77 | } 78 | } -------------------------------------------------------------------------------- /etc/migrations/000001_initial_setup.up.sql: -------------------------------------------------------------------------------- 1 | create table if not exists jenkins_versions ( 2 | id int generated by default as identity primary key, 3 | version varchar(32) 4 | ); 5 | 6 | create unique index jenkins_version_version on jenkins_versions(version); 7 | 8 | create table if not exists jvm_versions ( 9 | id int generated by default as identity primary key, 10 | name text NOT NULL 11 | ); 12 | 13 | create unique index jvm_version_name on jvm_versions(name); 14 | 15 | CREATE TABLE IF NOT EXISTS instance_reports ( 16 | id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 17 | instance_id varchar(64) NOT NULL, 18 | year smallint NOT NULL, 19 | month smallint NOT NULL, 20 | count_for_month int default 0, 21 | report_time timestamptz NOT NULL, 22 | version int references jenkins_versions, 23 | jvm_version_id int references jvm_versions, 24 | executors int default 0, 25 | plugins int[], 26 | jobs jsonb, 27 | nodes jsonb 28 | ); 29 | 30 | create index instance_reports_year_month on instance_reports using btree(year, month); 31 | create unique index instance_reports_instance_id_year_month on instance_reports using btree(instance_id, year, month); 32 | 33 | CREATE TABLE IF NOT EXISTS plugins ( 34 | id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 35 | name text NOT NULL, 36 | version text NOT NULL 37 | ); 38 | 39 | create unique index plugins_name_and_version on plugins using btree(name, version); 40 | create index plugins_name on plugins(name); 41 | 42 | CREATE TABLE IF NOT EXISTS job_types ( 43 | id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 44 | name text NOT NULL 45 | ); 46 | 47 | create unique index job_type_name on job_types(name); 48 | 49 | CREATE TABLE IF NOT EXISTS os_types ( 50 | id int GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 51 | name text NOT NULL 52 | ); 53 | 54 | create unique index os_type_name on os_types(name); 55 | 56 | create table if not exists report_files ( 57 | filename text primary key 58 | ) 59 | -------------------------------------------------------------------------------- /cmd/jenkins-usage-stats/report.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | stats "github.com/jenkins-infra/jenkins-usage-stats" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | // ReportOptions contains the configuration for actually outputting reports 13 | type ReportOptions struct { 14 | Directory string 15 | Database string 16 | LatestYear int 17 | LatestMonth int 18 | } 19 | 20 | // NewReportCmd returns the report command 21 | func NewReportCmd() *cobra.Command { 22 | options := &ReportOptions{} 23 | 24 | cobraCmd := &cobra.Command{ 25 | Use: "report", 26 | Short: "Generate stats.jenkins.io reports", 27 | Run: func(_ *cobra.Command, args []string) { 28 | if err := options.runReport(); err != nil { 29 | fmt.Println(err) 30 | os.Exit(1) 31 | } 32 | }, 33 | DisableAutoGenTag: true, 34 | } 35 | 36 | cobraCmd.Flags().StringVar(&options.Database, "database", "", "Database URL to import to") 37 | _ = cobraCmd.MarkFlagRequired("database") 38 | cobraCmd.Flags().StringVar(&options.Directory, "directory", "", "Directory to output to") 39 | _ = cobraCmd.MarkFlagRequired("directory") 40 | cobraCmd.Flags().IntVar(&options.LatestYear, "latest-year", 0, "Year of latest data to include. Defaults to the year of the previous month of when this is running.") 41 | cobraCmd.Flags().IntVar(&options.LatestMonth, "latest-month", 0, "Month of latest data to include. Defaults the previous month of when this is running.") 42 | cobraCmd.MarkFlagsRequiredTogether("latest-year", "latest-month") 43 | 44 | return cobraCmd 45 | } 46 | 47 | func (ro *ReportOptions) runReport() error { 48 | db, closeFunc, err := getDatabase(ro.Database) 49 | if err != nil { 50 | return err 51 | } 52 | defer closeFunc() 53 | 54 | startTime := time.Now() 55 | err = stats.GenerateReport(db, ro.LatestYear, ro.LatestMonth, ro.Directory) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | fmt.Printf("Reports generated to %s, in %s\n", ro.Directory, time.Since(startTime)) 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /testdata/fixtures/os_types.yml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | name: Linux (i386) 3 | - id: 2 4 | name: N/A 5 | - id: 3 6 | name: Linux (amd64) 7 | - id: 4 8 | name: Windows XP (x86) 9 | - id: 5 10 | name: Windows 2003 (x86) 11 | - id: 6 12 | name: Windows XP (amd64) 13 | - id: 7 14 | name: Windows Server 2008 (x86) 15 | - id: 8 16 | name: SunOS (sparc) 17 | - id: 9 18 | name: SunOS (x86) 19 | - id: 10 20 | name: Windows 2003 (amd64) 21 | - id: 11 22 | name: Windows 7 (x86) 23 | - id: 12 24 | name: SunOS (sparcv9) 25 | - id: 13 26 | name: Windows 2000 (x86) 27 | - id: 14 28 | name: Linux (ppc64) 29 | - id: 15 30 | name: Windows Vista (x86) 31 | - id: 16 32 | name: Linux (x86) 33 | - id: 17 34 | name: Mac OS X (x86_64) 35 | - id: 18 36 | name: HP-UX (PA_RISC2.0) 37 | - id: 19 38 | name: AIX (ppc64) 39 | - id: 20 40 | name: HP-UX (IA64N) 41 | - id: 21 42 | name: Windows 7 (amd64) 43 | - id: 22 44 | name: Mac OS X (ppc) 45 | - id: 23 46 | name: Mac OS X (i386) 47 | - id: 24 48 | name: Windows Vista (amd64) 49 | - id: 25 50 | name: FreeBSD (i386) 51 | - id: 26 52 | name: Windows Server 2003 (amd64) 53 | - id: 27 54 | name: Windows Server 2008 (amd64) 55 | - id: 28 56 | name: AIX (ppc) 57 | - id: 29 58 | name: Linux (ia64) 59 | - id: 30 60 | name: Windows Server 2008 (ia64) 61 | - id: 31 62 | name: Windows Server 2008 R2 (amd64) 63 | - id: 32 64 | name: FreeBSD (amd64) 65 | - id: 33 66 | name: OpenBSD (i386) 67 | - id: 34 68 | name: Linux (ppc) 69 | - id: 35 70 | name: Linux (sparc) 71 | - id: 36 72 | name: SunOS (amd64) 73 | - id: 37 74 | name: Linux (arm) 75 | - id: 38 76 | name: HP-UX (IA64W) 77 | - id: 39 78 | name: Darwin (i386) 79 | - id: 40 80 | name: Windows NT (unknown) (x86) 81 | - id: 41 82 | name: Windows Server 2003 (x86) 83 | - id: 42 84 | name: Windows Server 2008 R2 (x86) 85 | - id: 43 86 | name: Linux (s390x) 87 | - id: 44 88 | name: z/OS (s390) 89 | - id: 45 90 | name: linux (amd64) 91 | - id: 46 92 | name: HP-UX (PA_RISC2.0W) 93 | - id: 47 94 | name: Linux (zArch_64) 95 | - id: 48 96 | name: OS/400 (PowerPC) 97 | - id: 49 98 | name: Linux (s390) 99 | -------------------------------------------------------------------------------- /input.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "time" 7 | ) 8 | 9 | const ( 10 | jsonReportDateRE = `(\d\d)\/(\w\w\w)\/(\d\d\d\d)\:(\d\d\:\d\d\:\d\d) ([\+\-]\d\d)(\d\d)` 11 | ) 12 | 13 | var ( 14 | shortMonthToNumber = map[string]string{ 15 | "Jan": "01", 16 | "Feb": "02", 17 | "Mar": "03", 18 | "Apr": "04", 19 | "May": "05", 20 | "Jun": "06", 21 | "Jul": "07", 22 | "Aug": "08", 23 | "Sep": "09", 24 | "Oct": "10", 25 | "Nov": "11", 26 | "Dec": "12", 27 | } 28 | ) 29 | 30 | // JSONNode is how a node report is represented in the JSON 31 | type JSONNode struct { 32 | Executors uint64 `json:"executors,omitempty"` 33 | JVMName string `json:"jvm-name,omitempty"` 34 | JVMVendor string `json:"jvm-vendor,omitempty"` 35 | JVMVersion string `json:"jvm-version,omitempty"` 36 | IsController bool `json:"master"` 37 | OS string `json:"os,omitempty"` 38 | } 39 | 40 | // JSONPlugin is how a plugin report is represented in the JSON 41 | type JSONPlugin struct { 42 | Name string `json:"name"` 43 | Version string `json:"version"` 44 | } 45 | 46 | // JSONReport is how an instance report is represented in the JSON 47 | type JSONReport struct { 48 | Install string `json:"install"` 49 | Jobs map[string]uint64 `json:"jobs"` 50 | Nodes []JSONNode `json:"nodes"` 51 | Plugins []JSONPlugin `json:"plugins"` 52 | ServletContainer string `json:"servletContainer,omitempty"` 53 | TimestampString string `json:"timestamp"` 54 | Version string `json:"version"` 55 | } 56 | 57 | // Timestamp parses the raw timestamp string on a report 58 | func (j *JSONReport) Timestamp() (time.Time, error) { 59 | rawTime, err := time.Parse(time.RFC3339, JSONTimestampToRFC3339(j.TimestampString)) 60 | if err != nil { 61 | return rawTime, err 62 | } 63 | return rawTime.In(time.UTC), nil 64 | } 65 | 66 | // JSONTimestampToRFC3339 converts the timestamp string in the raw reports into a form Go can parse 67 | func JSONTimestampToRFC3339(ts string) string { 68 | re := regexp.MustCompile(jsonReportDateRE) 69 | matches := re.FindAllStringSubmatch(ts, -1) 70 | return fmt.Sprintf("%s-%s-%sT%s%s:%s", matches[0][3], shortMonthToNumber[matches[0][2]], matches[0][1], matches[0][4], matches[0][5], matches[0][6]) 71 | /* withoutZone := strings.TrimSuffix(ts, " +0000") 72 | splitDateAndTime := strings.SplitN(withoutZone, ":", 2) 73 | dayMonthYear := strings.Split(splitDateAndTime[0], "/") 74 | return fmt.Sprintf("%s-%s-%sT%sZ", dayMonthYear[2], shortMonthToNumber[dayMonthYear[1]], dayMonthYear[0], splitDateAndTime[1])*/ 75 | } 76 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | 3 | package stats_test 4 | 5 | import ( 6 | "database/sql" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "regexp" 11 | "sort" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | sq "github.com/Masterminds/squirrel" 17 | stats "github.com/jenkins-infra/jenkins-usage-stats" 18 | "github.com/jenkins-infra/jenkins-usage-stats/testutil" 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestDBIntegration(t *testing.T) { 23 | db, closeFunc := DBForIntTest(t) 24 | defer closeFunc() 25 | 26 | cache := stats.NewStatsCache() 27 | 28 | sampleStatsDir := filepath.Join("testdata", "report-stats") 29 | files, err := ioutil.ReadDir(sampleStatsDir) 30 | require.NoError(t, err) 31 | 32 | totalReports := 0 33 | 34 | dateRe := regexp.MustCompile(`.*\.(\d\d\d\d\d\d\d\d).*`) 35 | 36 | sort.Slice(files, func(i, j int) bool { 37 | if !files[i].IsDir() && strings.HasSuffix(files[i].Name(), ".gz") && !files[j].IsDir() && strings.HasSuffix(files[j].Name(), ".gz") { 38 | iMatch := dateRe.FindStringSubmatch(files[i].Name()) 39 | if len(iMatch) > 1 { 40 | iDate := iMatch[1] 41 | if iDate == "" { 42 | return true 43 | } 44 | jMatch := dateRe.FindStringSubmatch(files[j].Name()) 45 | if len(jMatch) > 1 { 46 | jDate := jMatch[1] 47 | if jDate == "" { 48 | return true 49 | } 50 | return iDate < jDate 51 | } 52 | } 53 | } 54 | return true 55 | }) 56 | 57 | for _, fi := range files { 58 | if !fi.IsDir() && !strings.Contains(fi.Name(), "fudged") && strings.HasSuffix(fi.Name(), ".gz") { // && strings.Contains(fi.Name(), ".201001") { 59 | startedAt := time.Now() 60 | alreadyRead, err := stats.ReportAlreadyRead(db, fi.Name()) 61 | require.NoError(t, err) 62 | if alreadyRead { 63 | t.Logf("file %s already read\n", fi.Name()) 64 | continue 65 | } 66 | fn := filepath.Join(sampleStatsDir, fi.Name()) 67 | jsonReports, err := stats.ParseDailyJSON(fn) 68 | require.NoError(t, err) 69 | t.Logf("adding %d reports from file %s\n", len(jsonReports), fi.Name()) 70 | totalReports += len(jsonReports) 71 | for _, jr := range jsonReports { 72 | require.NoError(t, stats.AddIndividualReport(db, cache, jr)) 73 | } 74 | require.NoError(t, stats.MarkReportRead(db, fi.Name())) 75 | t.Logf("imported in %s", time.Since(startedAt)) 76 | } 77 | } 78 | 79 | t.Log(cache.ReportTimes()) 80 | t.Logf("total reports: %d\n", totalReports) 81 | } 82 | 83 | // DBForIntTest connects to a local database for testing 84 | func DBForIntTest(f testutil.Fataler) (sq.BaseRunner, func()) { 85 | databaseURL := os.Getenv("DATABASE_URL") 86 | if databaseURL == "" { 87 | databaseURL = "postgres://postgres@localhost/jenkins_usage_stats?sslmode=disable&timezone=UTC" 88 | } 89 | 90 | db, err := sql.Open("postgres", databaseURL) 91 | if err != nil { 92 | f.Fatal(err) 93 | } 94 | 95 | closeFunc := func() { 96 | if err := db.Close(); err != nil { 97 | f.Fatal(err) 98 | } 99 | } 100 | 101 | return sq.NewStmtCacheProxy(db), closeFunc 102 | } 103 | -------------------------------------------------------------------------------- /cmd/jenkins-usage-stats/import.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | "sort" 9 | "strings" 10 | "time" 11 | 12 | stats "github.com/jenkins-infra/jenkins-usage-stats" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | // ImportOptions is the configuration for the import command 17 | type ImportOptions struct { 18 | Database string 19 | Directory string 20 | } 21 | 22 | // NewImportCmd returns the import command 23 | func NewImportCmd() *cobra.Command { 24 | options := &ImportOptions{} 25 | 26 | cobraCmd := &cobra.Command{ 27 | Use: "import", 28 | Short: "Import instance reports", 29 | Run: func(cmd *cobra.Command, args []string) { 30 | if err := options.runImport(); err != nil { 31 | fmt.Println(err) 32 | os.Exit(1) 33 | } 34 | }, 35 | DisableAutoGenTag: true, 36 | } 37 | 38 | cobraCmd.Flags().StringVar(&options.Database, "database", "", "Database URL to import to") 39 | _ = cobraCmd.MarkFlagRequired("database") 40 | cobraCmd.Flags().StringVar(&options.Directory, "directory", "", "Directory to import from") 41 | _ = cobraCmd.MarkFlagRequired("directory") 42 | 43 | return cobraCmd 44 | } 45 | 46 | func (io *ImportOptions) runImport() error { 47 | db, closeFunc, err := getDatabase(io.Database) 48 | if err != nil { 49 | return err 50 | } 51 | defer closeFunc() 52 | 53 | files, err := os.ReadDir(io.Directory) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | totalReports := 0 59 | 60 | dateRe := regexp.MustCompile(`.*\.(\d\d\d\d\d\d\d\d).*`) 61 | 62 | sort.Slice(files, func(i, j int) bool { 63 | if !files[i].IsDir() && strings.HasSuffix(files[i].Name(), ".gz") && !files[j].IsDir() && strings.HasSuffix(files[j].Name(), ".gz") { 64 | iMatch := dateRe.FindStringSubmatch(files[i].Name()) 65 | if len(iMatch) > 1 { 66 | iDate := iMatch[1] 67 | if iDate == "" { 68 | return true 69 | } 70 | jMatch := dateRe.FindStringSubmatch(files[j].Name()) 71 | if len(jMatch) > 1 { 72 | jDate := jMatch[1] 73 | if jDate == "" { 74 | return true 75 | } 76 | return iDate < jDate 77 | } 78 | } 79 | } 80 | return true 81 | }) 82 | 83 | cache := stats.NewStatsCache() 84 | 85 | importStart := time.Now() 86 | 87 | for _, fi := range files { 88 | if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".gz") { 89 | startedAt := time.Now() 90 | alreadyRead, err := stats.ReportAlreadyRead(db, fi.Name()) 91 | if err != nil { 92 | return err 93 | } 94 | if alreadyRead { 95 | fmt.Printf("file %s already read\n", fi.Name()) 96 | continue 97 | } 98 | fn := filepath.Join(io.Directory, fi.Name()) 99 | jsonReports, err := stats.ParseDailyJSON(fn) 100 | if err != nil { 101 | return err 102 | } 103 | fmt.Printf("adding %d reports from file %s\n", len(jsonReports), fi.Name()) 104 | totalReports += len(jsonReports) 105 | for _, jr := range jsonReports { 106 | if err := stats.AddIndividualReport(db, cache, jr); err != nil { 107 | return err 108 | } 109 | } 110 | if err := stats.MarkReportRead(db, fi.Name()); err != nil { 111 | return err 112 | } 113 | fmt.Printf("imported in %s\n", time.Since(startedAt)) 114 | } 115 | } 116 | 117 | fmt.Println(cache.ReportTimes()) 118 | fmt.Printf("total reports: %d (time to import: %s)\n", totalReports, time.Since(importStart)) 119 | 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /testutil/db.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "log" 8 | "path/filepath" 9 | "runtime" 10 | 11 | "github.com/docker/go-connections/nat" 12 | "github.com/golang-migrate/migrate/v4" 13 | "github.com/golang-migrate/migrate/v4/database/postgres" 14 | testcontainers "github.com/testcontainers/testcontainers-go" 15 | "github.com/testcontainers/testcontainers-go/wait" 16 | ) 17 | 18 | const ( 19 | dbName = "jenkins_usage_stats_test" 20 | ) 21 | 22 | // Fataler interface has a single method Fatal, which takes 23 | // a slice of arguments and is expected to panic. 24 | type Fataler interface { 25 | Fatal(args ...interface{}) 26 | } 27 | 28 | // DBForTest spins up a postgres container, creates the test database on it, migrates it, and returns the db and a close function 29 | func DBForTest(f Fataler) (*sql.DB, func()) { 30 | ctx := context.Background() 31 | // container and database 32 | container, db, err := CreateTestContainer(ctx) 33 | if err != nil { 34 | f.Fatal(err) 35 | } 36 | 37 | closeFunc := func() { 38 | _ = db.Close() 39 | _ = container.Terminate(ctx) 40 | } 41 | 42 | // migration 43 | mig, err := NewPgMigrator(db) 44 | if err != nil { 45 | closeFunc() 46 | f.Fatal(err) 47 | } 48 | 49 | err = mig.Up() 50 | if err != nil { 51 | closeFunc() 52 | f.Fatal(err) 53 | } 54 | 55 | return db, closeFunc 56 | } 57 | 58 | // CreateTestContainer spins up a Postgres database container 59 | func CreateTestContainer(ctx context.Context) (testcontainers.Container, *sql.DB, error) { 60 | env := map[string]string{ 61 | "POSTGRES_PASSWORD": "password", 62 | "POSTGRES_USER": "postgres", 63 | "POSTGRES_DB": dbName, 64 | } 65 | dockerPort := "5432/tcp" 66 | dbURL := func(_ string, port nat.Port) string { 67 | return fmt.Sprintf("postgres://postgres:password@localhost:%s/%s?sslmode=disable", port.Port(), dbName) 68 | } 69 | 70 | req := testcontainers.GenericContainerRequest{ 71 | ContainerRequest: testcontainers.ContainerRequest{ 72 | Image: "postgres:12", 73 | ExposedPorts: []string{dockerPort}, 74 | Cmd: []string{"postgres", "-c", "fsync=off"}, 75 | Env: env, 76 | WaitingFor: wait.ForSQL(nat.Port(dockerPort), "postgres", dbURL), 77 | }, 78 | Started: true, 79 | } 80 | container, err := testcontainers.GenericContainer(ctx, req) 81 | if err != nil { 82 | return container, nil, fmt.Errorf("failed to start container: %s", err) 83 | } 84 | 85 | mappedPort, err := container.MappedPort(ctx, nat.Port(dockerPort)) 86 | if err != nil { 87 | return container, nil, fmt.Errorf("failed to get container external port: %s", err) 88 | } 89 | 90 | log.Println("postgres container ready and running at dockerPort: ", mappedPort) 91 | 92 | url := fmt.Sprintf("postgres://postgres:password@localhost:%s/%s?sslmode=disable", mappedPort.Port(), dbName) 93 | db, err := sql.Open("postgres", url) 94 | if err != nil { 95 | return container, db, fmt.Errorf("failed to establish database connection: %s", err) 96 | } 97 | 98 | return container, db, nil 99 | } 100 | 101 | // NewPgMigrator creates a migrator instance 102 | func NewPgMigrator(db *sql.DB) (*migrate.Migrate, error) { 103 | _, path, _, ok := runtime.Caller(0) 104 | if !ok { 105 | log.Fatalf("failed to get path") 106 | } 107 | 108 | sourceURL := "file://" + filepath.Dir(path) + "/../etc/migrations" 109 | 110 | driver, err := postgres.WithInstance(db, &postgres.Config{}) 111 | 112 | if err != nil { 113 | log.Fatalf("failed to create migrator driver: %s", err) 114 | } 115 | 116 | m, err := migrate.NewWithDatabaseInstance(sourceURL, "postgres", driver) 117 | 118 | return m, err 119 | } 120 | -------------------------------------------------------------------------------- /testdata/day-later.json: -------------------------------------------------------------------------------- 1 | {"install":"32b68faa8644852c4ad79540b4bfeb1caf63284811f4f9d6c2bc511f797218c8","jobs":{"com-cloudbees-hudson-plugins-folder-Folder":0,"com-tikal-jenkins-plugins-multijob-MultiJobProject":0,"hudson-matrix-MatrixProject":10,"hudson-maven-MavenModuleSet":50,"hudson-model-ExternalJob":0,"hudson-model-FreeStyleProject":48,"jenkins-branch-OrganizationFolder":0,"org-jenkinsci-plugins-workflow-job-WorkflowJob":0,"org-jenkinsci-plugins-workflow-multibranch-WorkflowMultiBranchProject":0,"private-2ab0d9d28b1b071788f80194f45c5e49f915bb58d05b8cf3100db4da2930f63bdc7e38a1f80b85a216b8d781a0f5a4a7af2f01a07f19c60ffd41f78f7d4a859116ba65011b4aec78e794ade7fb8eb77479e4d675d0164a2f337ca04f6249f0f0":0},"nodes":[{"executors":5,"jvm-name":"Java HotSpot(TM) 64-Bit Server VM","jvm-vendor":"Oracle Corporation","jvm-version":"1.8.0_191","master":true,"os":"Windows Server 2016 (amd64)"}],"plugins":[{"name":"ace-editor","version":"1.2"},{"name":"ant","version":"1.11"},{"name":"antisamy-markup-formatter","version":"2.1"},{"name":"apache-httpcomponents-client-4-api","version":"4.5.10-2.0"},{"name":"bouncycastle-api","version":"2.18"},{"name":"branch-api","version":"2.5.5"},{"name":"built-on-column","version":"1.1"},{"name":"cloudbees-folder","version":"6.11.1"},{"name":"command-launcher","version":"1.4"},{"name":"conditional-buildstep","version":"1.3.6"},{"name":"config-file-provider","version":"3.6.3"},{"name":"configurationslicing","version":"1.51"},{"name":"credentials","version":"2.3.11"},{"name":"deploy","version":"1.15"},{"name":"dingding-notifications","version":"1.9"},{"name":"display-url-api","version":"2.3.2"},{"name":"echarts-api","version":"4.8.0-2"},{"name":"email-ext","version":"2.80"},{"name":"envinject-api","version":"1.7"},{"name":"envinject","version":"2.3.0"},{"name":"external-monitor-job","version":"1.7"},{"name":"git-client","version":"3.2.0"},{"name":"git-parameter","version":"0.9.13"},{"name":"git-server","version":"1.9"},{"name":"git","version":"4.2.0"},{"name":"github-api","version":"1.106"},{"name":"github-branch-source","version":"2.6.0"},{"name":"github-oauth","version":"0.33"},{"name":"github-organization-folder","version":"1.6"},{"name":"github","version":"1.29.5"},{"name":"jackson2-api","version":"2.11.1"},{"name":"javadoc","version":"1.5"},{"name":"jdk-tool","version":"1.4"},{"name":"jenkins-multijob-plugin","version":"1.33"},{"name":"jobConfigHistory","version":"2.25"},{"name":"jquery-detached","version":"1.2.1"},{"name":"jquery","version":"1.12.4-1"},{"name":"jquery3-api","version":"3.5.1-1"},{"name":"jsch","version":"0.1.55.2"},{"name":"junit","version":"1.31"},{"name":"ldap","version":"1.25"},{"name":"localization-support","version":"1.1"},{"name":"localization-zh-cn","version":"1.0.24"},{"name":"mailer","version":"1.32.1"},{"name":"mapdb-api","version":"1.0.9.0"},{"name":"matrix-auth","version":"2.5.1"},{"name":"matrix-project","version":"1.14"},{"name":"maven-plugin","version":"3.7"},{"name":"msbuild","version":"1.29"},{"name":"nodejs","version":"1.3.5"},{"name":"pam-auth","version":"1.6"},{"name":"parameterized-trigger","version":"2.39"},{"name":"pipeline-github-lib","version":"1.0"},{"name":"plain-credentials","version":"1.7"},{"name":"plugin-util-api","version":"1.2.2"},{"name":"role-strategy","version":"2.16"},{"name":"run-condition","version":"1.3"},{"name":"scm-api","version":"2.6.3"},{"name":"script-security","version":"1.74"},{"name":"snakeyaml-api","version":"1.26.3"},{"name":"ssh-credentials","version":"1.18.1"},{"name":"structs","version":"1.20"},{"name":"subversion","version":"2.13.1"},{"name":"thinBackup","version":"1.10"},{"name":"token-macro","version":"2.12"},{"name":"trilead-api","version":"1.0.5"},{"name":"windows-slaves","version":"1.6"},{"name":"workflow-api","version":"2.40"},{"name":"workflow-cps-global-lib","version":"2.15"},{"name":"workflow-cps","version":"2.80"},{"name":"workflow-job","version":"2.40"},{"name":"workflow-multibranch","version":"2.21"},{"name":"workflow-scm-step","version":"2.10"},{"name":"workflow-step-api","version":"2.22"}],"servletContainer":"jetty/9.4.25.v20191220","stat":1,"timestamp":"31/Oct/2021:23:59:54 +0000","version":"2.204.5"} 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jenkins-infra/jenkins-usage-stats 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 7 | github.com/Masterminds/semver v1.5.0 8 | github.com/Masterminds/squirrel v1.5.1 9 | github.com/beevik/etree v1.1.0 10 | github.com/docker/go-connections v0.5.0 11 | github.com/go-testfixtures/testfixtures/v3 v3.6.1 12 | github.com/golang-migrate/migrate/v4 v4.15.1 13 | github.com/lib/pq v1.10.3 14 | github.com/spf13/cobra v1.8.1 15 | github.com/stretchr/testify v1.9.0 16 | github.com/testcontainers/testcontainers-go v0.29.1 17 | gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a 18 | gopkg.in/yaml.v2 v2.4.0 19 | ) 20 | 21 | require ( 22 | dario.cat/mergo v1.0.0 // indirect 23 | github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect 24 | github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect 25 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 26 | github.com/Microsoft/go-winio v0.6.1 // indirect 27 | github.com/Microsoft/hcsshim v0.11.4 // indirect 28 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 29 | github.com/containerd/containerd v1.7.12 // indirect 30 | github.com/containerd/log v0.1.0 // indirect 31 | github.com/cpuguy83/dockercfg v0.3.1 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/distribution/reference v0.5.0 // indirect 34 | github.com/docker/docker v25.0.3+incompatible // indirect 35 | github.com/docker/go-units v0.5.0 // indirect 36 | github.com/felixge/httpsnoop v1.0.3 // indirect 37 | github.com/go-logr/logr v1.2.4 // indirect 38 | github.com/go-logr/stdr v1.2.2 // indirect 39 | github.com/go-ole/go-ole v1.2.6 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/golang/protobuf v1.5.3 // indirect 42 | github.com/google/uuid v1.6.0 // indirect 43 | github.com/hashicorp/errwrap v1.1.0 // indirect 44 | github.com/hashicorp/go-multierror v1.1.1 // indirect 45 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 46 | github.com/klauspost/compress v1.16.0 // indirect 47 | github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect 48 | github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect 49 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 50 | github.com/magiconair/properties v1.8.7 // indirect 51 | github.com/moby/patternmatcher v0.6.0 // indirect 52 | github.com/moby/sys/sequential v0.5.0 // indirect 53 | github.com/moby/sys/user v0.1.0 // indirect 54 | github.com/moby/term v0.5.0 // indirect 55 | github.com/morikuni/aec v1.0.0 // indirect 56 | github.com/opencontainers/go-digest v1.0.0 // indirect 57 | github.com/opencontainers/image-spec v1.1.0 // indirect 58 | github.com/pkg/errors v0.9.1 // indirect 59 | github.com/pmezard/go-difflib v1.0.0 // indirect 60 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 61 | github.com/shirou/gopsutil/v3 v3.23.12 // indirect 62 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 63 | github.com/sirupsen/logrus v1.9.3 // indirect 64 | github.com/spf13/pflag v1.0.5 // indirect 65 | github.com/tklauser/go-sysconf v0.3.12 // indirect 66 | github.com/tklauser/numcpus v0.6.1 // indirect 67 | github.com/yusufpapurcu/wmi v1.2.3 // indirect 68 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect 69 | go.opentelemetry.io/otel v1.19.0 // indirect 70 | go.opentelemetry.io/otel/metric v1.19.0 // indirect 71 | go.opentelemetry.io/otel/trace v1.19.0 // indirect 72 | go.uber.org/atomic v1.7.0 // indirect 73 | golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect 74 | golang.org/x/mod v0.16.0 // indirect 75 | golang.org/x/net v0.17.0 // indirect 76 | golang.org/x/sys v0.16.0 // indirect 77 | golang.org/x/text v0.13.0 // indirect 78 | golang.org/x/tools v0.13.0 // indirect 79 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect 80 | google.golang.org/grpc v1.58.3 // indirect 81 | google.golang.org/protobuf v1.31.0 // indirect 82 | gopkg.in/yaml.v3 v3.0.1 // indirect 83 | ) 84 | -------------------------------------------------------------------------------- /testdata/fixtures/jenkins_versions.yml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | version: "1.320" 3 | - id: 2 4 | version: "1.337" 5 | - id: 3 6 | version: "1.329" 7 | - id: 4 8 | version: "1.336" 9 | - id: 5 10 | version: "1.335" 11 | - id: 6 12 | version: "1.331" 13 | - id: 7 14 | version: "1.323" 15 | - id: 8 16 | version: "1.310" 17 | - id: 9 18 | version: "1.309" 19 | - id: 10 20 | version: "1.314" 21 | - id: 11 22 | version: "1.330" 23 | - id: 12 24 | version: "1.324" 25 | - id: 13 26 | version: "1.332" 27 | - id: 14 28 | version: "1.292" 29 | - id: 15 30 | version: "1.333" 31 | - id: 16 32 | version: "1.319" 33 | - id: 17 34 | version: "1.286" 35 | - id: 18 36 | version: "1.334" 37 | - id: 19 38 | version: "1.327" 39 | - id: 20 40 | version: "1.328" 41 | - id: 21 42 | version: "1.311" 43 | - id: 22 44 | version: "1.297" 45 | - id: 23 46 | version: "1.316" 47 | - id: 24 48 | version: "1.321" 49 | - id: 25 50 | version: "1.266" 51 | - id: 26 52 | version: "1.306" 53 | - id: 27 54 | version: "1.299" 55 | - id: 28 56 | version: "1.313" 57 | - id: 29 58 | version: "1.264" 59 | - id: 30 60 | version: "1.322" 61 | - id: 31 62 | version: "1.267" 63 | - id: 32 64 | version: "1.293" 65 | - id: 33 66 | version: "1.318" 67 | - id: 34 68 | version: "1.312" 69 | - id: 35 70 | version: "1.304" 71 | - id: 36 72 | version: "1.317" 73 | - id: 37 74 | version: "1.281" 75 | - id: 38 76 | version: "1.338" 77 | - id: 39 78 | version: "1.326" 79 | - id: 40 80 | version: "1.325" 81 | - id: 41 82 | version: "1.303" 83 | - id: 42 84 | version: "1.302" 85 | - id: 43 86 | version: 1.301 (private) 87 | - id: 44 88 | version: "1.296" 89 | - id: 45 90 | version: "1.285" 91 | - id: 46 92 | version: "1.300" 93 | - id: 47 94 | version: 1.293 (private) 95 | - id: 48 96 | version: 1.336 (private) 97 | - id: 49 98 | version: "1.291" 99 | - id: 50 100 | version: "1.279" 101 | - id: 51 102 | version: "1.287" 103 | - id: 52 104 | version: "1.290" 105 | - id: 53 106 | version: "1.294" 107 | - id: 54 108 | version: "1.315" 109 | - id: 55 110 | version: "1.278" 111 | - id: 56 112 | version: "1.307" 113 | - id: 57 114 | version: "1.276" 115 | - id: 58 116 | version: "1.301" 117 | - id: 59 118 | version: "1.270" 119 | - id: 60 120 | version: "1.265" 121 | - id: 61 122 | version: "1.308" 123 | - id: 62 124 | version: "1.274" 125 | - id: 63 126 | version: "1.288" 127 | - id: 64 128 | version: "1.289" 129 | - id: 65 130 | version: "1.282" 131 | - id: 66 132 | version: 1.334 (private) 133 | - id: 67 134 | version: 1.337 (private) 135 | - id: 68 136 | version: 1.303 (private) 137 | - id: 69 138 | version: 1.292.17 139 | - id: 70 140 | version: "1.295" 141 | - id: 71 142 | version: 1.278 (private) 143 | - id: 72 144 | version: 1.324 (private) 145 | - id: 73 146 | version: 1.300 (private) 147 | - id: 74 148 | version: "1.273" 149 | - id: 75 150 | version: "1.268" 151 | - id: 76 152 | version: "1.277" 153 | - id: 77 154 | version: "1.275" 155 | - id: 78 156 | version: "1.280" 157 | - id: 79 158 | version: 1.330 (private) 159 | - id: 80 160 | version: 1.328 (private) 161 | - id: 81 162 | version: 1.292 (private) 163 | - id: 82 164 | version: 1.338 (private) 165 | - id: 83 166 | version: 1.299 (private) 167 | - id: 84 168 | version: 1.309-CGU (private) 169 | - id: 85 170 | version: "1.272" 171 | - id: 86 172 | version: 1.313 (private) 173 | - id: 87 174 | version: "1.284" 175 | - id: 88 176 | version: 1.320 (private) 177 | - id: 89 178 | version: 1.274 (private) 179 | - id: 90 180 | version: 1.295 (private) 181 | - id: 91 182 | version: 1.335 (private) 183 | - id: 92 184 | version: 1.306 (private) 185 | - id: 93 186 | version: "1.283" 187 | - id: 94 188 | version: 1.312 (private) 189 | - id: 95 190 | version: 1.329 (private) 191 | - id: 96 192 | version: "1.305" 193 | - id: 97 194 | version: "1.339" 195 | - id: 98 196 | version: 1.318 (private) 197 | - id: 99 198 | version: 1.339 (private) 199 | - id: 100 200 | version: "1.271" 201 | - id: 101 202 | version: 1.315 (private) 203 | - id: 102 204 | version: 1.327 (private) 205 | -------------------------------------------------------------------------------- /cmd/jenkins-usage-stats/fetch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" 11 | stats "github.com/jenkins-infra/jenkins-usage-stats" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | // FetchOptions is the configuration for the fetch command 16 | type FetchOptions struct { 17 | Database string 18 | Directory string 19 | AzureAccount string 20 | AzureKey string 21 | AzureContainer string 22 | } 23 | 24 | // NewFetchCmd returns the fetch command 25 | func NewFetchCmd(ctx context.Context) *cobra.Command { 26 | options := &FetchOptions{} 27 | 28 | cobraCmd := &cobra.Command{ 29 | Use: "fetch", 30 | Short: "Fetch raw usage data from Azure", 31 | Run: func(cmd *cobra.Command, args []string) { 32 | if err := options.runFetch(ctx); err != nil { 33 | fmt.Println(err) 34 | os.Exit(1) 35 | } 36 | }, 37 | DisableAutoGenTag: true, 38 | } 39 | 40 | cobraCmd.Flags().StringVar(&options.Database, "database", "", "Database URL") 41 | _ = cobraCmd.MarkFlagRequired("database") 42 | cobraCmd.Flags().StringVar(&options.Directory, "directory", "", "Directory to write raw usage gz files to") 43 | _ = cobraCmd.MarkFlagRequired("directory") 44 | cobraCmd.Flags().StringVar(&options.AzureAccount, "account", "", "Azure account") 45 | _ = cobraCmd.MarkFlagRequired("account") 46 | cobraCmd.Flags().StringVar(&options.AzureKey, "key", "", "Azure key") 47 | _ = cobraCmd.MarkFlagRequired("key") 48 | cobraCmd.Flags().StringVar(&options.AzureContainer, "container", "", "Azure blob container") 49 | _ = cobraCmd.MarkFlagRequired("container") 50 | 51 | return cobraCmd 52 | } 53 | 54 | func (fo *FetchOptions) runFetch(ctx context.Context) error { 55 | db, closeFunc, err := getDatabase(fo.Database) 56 | if err != nil { 57 | return err 58 | } 59 | defer closeFunc() 60 | 61 | fmt.Printf("creating raw usage directory %s if it doesn't exist\n", fo.Directory) 62 | err = os.MkdirAll(fo.Directory, 0755) //nolint:gosec 63 | if err != nil { 64 | return err 65 | } 66 | 67 | azCred, err := azblob.NewSharedKeyCredential(fo.AzureAccount, fo.AzureKey) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | azClient, err := azblob.NewServiceClientWithSharedKey(fmt.Sprintf("https://%s.blob.core.windows.net/", fo.AzureAccount), azCred, nil) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | var toDownload []string 78 | 79 | azCtr := azClient.NewContainerClient(fo.AzureContainer) 80 | 81 | fmt.Printf("checking container %s for new raw usage files\n", fo.AzureContainer) 82 | pager := azCtr.ListBlobsFlat(nil) 83 | 84 | for pager.NextPage(ctx) { 85 | resp := pager.PageResponse() 86 | 87 | for _, v := range resp.ContainerListBlobFlatSegmentResult.Segment.BlobItems { 88 | if v.Name != nil { 89 | // Check if we've already recorded this file in the database. 90 | alreadyRecorded, err := stats.ReportAlreadyRead(db, *v.Name) 91 | if err != nil { 92 | return err 93 | } 94 | if !alreadyRecorded { 95 | fmt.Printf("%s - new raw usage file, queuing for download\n", *v.Name) 96 | toDownload = append(toDownload, *v.Name) 97 | } 98 | } 99 | } 100 | } 101 | 102 | if err := pager.Err(); err != nil { 103 | return err 104 | } 105 | 106 | if len(toDownload) == 0 { 107 | fmt.Println("no new raw usage files to download, finishing") 108 | return nil 109 | } 110 | 111 | fmt.Printf("%d new raw usage files to download\n", len(toDownload)) 112 | 113 | for _, fn := range toDownload { 114 | blockBlob := azCtr.NewBlobClient(fn) 115 | 116 | fmt.Printf(" - downloading %s\n", fn) 117 | dlResp, err := blockBlob.Download(ctx, nil) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | dlData, err := ioutil.ReadAll(dlResp.Body(azblob.RetryReaderOptions{})) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | destFile := filepath.Join(fo.Directory, fn) 128 | err = ioutil.WriteFile(destFile, dlData, 0644) //nolint:gosec 129 | if err != nil { 130 | return err 131 | } 132 | } 133 | 134 | fmt.Println("fetch complete") 135 | return nil 136 | } 137 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/gzip" 7 | "encoding/json" 8 | "io/ioutil" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | hotspotAndMisc16Versions = []string{ 14 | // Until sometime in 2010, HotSpot and some other JVMs reported _their_ version number, not the Java version. 15 | // So for those cases, we're just going to map them all to 1.6. 16 | 17 | // HotSpot versions 18 | "10.0-b19", 19 | "10.0-b22", 20 | "10.0-b23", 21 | "10.0-b25", 22 | "11.0-b11", 23 | "11.0-b12", 24 | "11.0-b15", 25 | "11.0-b16", 26 | "11.0-b17", 27 | "11.2-b01", 28 | "11.3-b02", 29 | "13.0-b04", 30 | "14.0-b01", 31 | "14.0-b05", 32 | "14.0-b08", 33 | "14.0-b09", 34 | "14.0-b10", 35 | "14.0-b12", 36 | "14.0-b15", 37 | "14.0-b16", 38 | "14.1-b02", 39 | "14.2-b01", 40 | "14.3-b01", 41 | "16.0-b03", 42 | "16.0-b08", 43 | "16.0-b13", 44 | "16.2-b04", 45 | "16.3-b01", 46 | "17.0-b14", 47 | "17.0-b15", 48 | "17.0-b16", 49 | "17.0-b17", 50 | "17.1-b03", 51 | // IBM JVM 52 | "2.3", 53 | "2.4", 54 | // SAP JVM 55 | "5.1.0844", 56 | "5.1.0909", 57 | } 58 | ) 59 | 60 | // ParseDailyJSON parses an individual day's gzipped JSON reports 61 | func ParseDailyJSON(filename string) ([]*JSONReport, error) { 62 | gzippedJSON, err := ioutil.ReadFile(filename) // #nosec 63 | if err != nil { 64 | return nil, err 65 | } 66 | zReader, err := gzip.NewReader(bytes.NewReader(gzippedJSON)) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | var reports []*JSONReport 72 | 73 | scanner := bufio.NewScanner(zReader) 74 | sBuffer := make([]byte, 0, bufio.MaxScanTokenSize) 75 | scanner.Buffer(sBuffer, bufio.MaxScanTokenSize*50) // Otherwise long lines crash the scanner. 76 | 77 | for scanner.Scan() { 78 | var r *JSONReport 79 | err = json.Unmarshal(scanner.Bytes(), &r) 80 | if err != nil { 81 | // If the error is a "cannot unmarshal number...", just skip this record. This is to deal with the range of 82 | // possible weird executor count values we see, ranging from -4 to 2147483655 - i.e., 8 more than the max 32 83 | // bit number. We're opting to just pay attention to positive values, and we don't really want to deal with 84 | // bad data anyway. 85 | if strings.Contains(err.Error(), "cannot unmarshal number") { 86 | continue 87 | } 88 | // If the error is "cannot unmarshal array into Go struct field JSONReport.jobs of type uint64", we hit a 89 | // weird case of the value for a job count being an array, so let's just ignore that record. 90 | if strings.Contains(err.Error(), "cannot unmarshal array into Go struct field JSONReport.jobs of type uint64") { 91 | continue 92 | } 93 | return nil, err 94 | } 95 | FilterPrivateFromReport(r) 96 | standardizeJVMVersions(r) 97 | reports = append(reports, r) 98 | } 99 | if err := scanner.Err(); err != nil { 100 | return nil, err 101 | } 102 | 103 | return reports, nil 104 | } 105 | 106 | // FilterPrivateFromReport removes private plugins from the report 107 | func FilterPrivateFromReport(r *JSONReport) { 108 | var plugins []JSONPlugin 109 | for _, p := range r.Plugins { 110 | if !strings.HasPrefix(p.Name, "privateplugin-") && !strings.Contains(p.Version, "(private)") { 111 | plugins = append(plugins, p) 112 | } 113 | } 114 | r.Plugins = plugins 115 | } 116 | 117 | func standardizeJVMVersions(r *JSONReport) { 118 | var nodes []JSONNode 119 | for _, n := range r.Nodes { 120 | fullVersion := hotspotJVMVersionToJavaVersion(n.JVMVersion) 121 | if fullVersion == "" { 122 | n.JVMVersion = "N/A" 123 | } else if fullVersion == "8" { 124 | n.JVMVersion = "1.8" 125 | } else if strings.HasPrefix(fullVersion, "1.") { 126 | n.JVMVersion = fullVersion[0:3] 127 | if n.JVMVersion == "1.9" { 128 | n.JVMVersion = "9" 129 | } 130 | } else { 131 | splitVersion := strings.Split(fullVersion, ".") 132 | n.JVMVersion = splitVersion[0] 133 | } 134 | nodes = append(nodes, n) 135 | } 136 | r.Nodes = nodes 137 | } 138 | 139 | func hotspotJVMVersionToJavaVersion(input string) string { 140 | for _, hv := range hotspotAndMisc16Versions { 141 | if input == hv { 142 | return "1.6" 143 | } 144 | } 145 | return input 146 | } 147 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## jenkins-usage-stats generator 2 | 3 | `jenkins-usage-stats` handles importing daily Jenkins usage reports into a database, and generating monthly reports from that data. It replaces https://github.com/jenkins-infra/infra-statistics, and is run on Jenkins project infrastructure. 4 | 5 | ### Differences from infra-statistics report generator 6 | 7 | * Much, much faster. 8 | * Uses a persistent Postgres database rather than a weird hodge-podge of Mongo, giant JSON files, and sqlite. This means `jenkins-usage-stats` can be run on dynamically provisioned compute resources using a hosted Postgres database, rather than the whole thing needing to live on a single persistent machine using >500GB of disk. 9 | * The "start time" for months is midnight UTC, rather than midnight PST/-0800. Data is very slightly different as a result, but not in a meaningful way. 10 | * Months which have no data but do have report gzip files (i.e., April 2007 until December 2008) will not be included in the generated reports, SVGs, etc. 11 | * Input data is filtered a little more aggressively when it comes to weird/non-standard Jenkins and plugin versions. This doesn't seem to make a statistically significant difference in the generated reports. 12 | * With the infra-statistics tooling, JVM versions were limited to explicitly specified patterns. That's no longer the case, so modern JVM versions are going to be much more accurately represented. 13 | 14 | ### Running 15 | 16 | #### Database 17 | 18 | You will need to have a URL for your Postgres database, like `postgres://postgres@localhost/jenkins_usage_stats?sslmode=disable&timezone=UTC`. This will be used when running both `jenkins-usage-stats import` and `jenkins-usage-stats report`. 19 | 20 | #### Import 21 | 22 | Run `jenkins-usage-stats import --database "(database URL from above)" --directory (location containing daily report gzip files from usage.jenkins.io)`. Any gzip report file which hasn't already been imported will be read, line by line, into JSON, filtered for reports which should be excluded due to non-standard or SNAPSHOT Jenkins versions, not having any jobs defined, and some other filtering criteria. 23 | 24 | Each report will then be added to the database specified. If there is already a report present in the database for the year/month, and its report time is earlier than the new report, the new report will overwrite the previous report, incrementing the monthly count. If the new report is earlier than the existing report, the existing report's monthly count is incremented but no other changes are made - we only care about the _last_ report of the month for each instance ID. 25 | 26 | #### Report 27 | 28 | Run `jenkins-usage-stats report --database "(database URL from above)" --directory (output directory to write the generated reports to)`. The various reports used on https://stats.jenkins.io will be written to that output directory in the same layout as is used on the `gh-pages` branch of this repo, and its predecessor, https://github.com/jenkins-infra/infra-statistics. Data will be considered for every month _before_ the current one, so that we don't include incomplete data for this month. 29 | 30 | Note that the "start time" for months is midnight UTC. For example, 1654041600000 is June 1, 2022, 00:00:00 UTC, identifying the data gathered in June: 31 | 32 | ```sh 33 | $ curl https://stats.jenkins.io/plugin-installation-trend/jvms.json | jq '.jvmStatsPerMonth | to_entries | last' 34 | { 35 | "key": "1654041600000", 36 | "value": { 37 | "1.5": 1, 38 | "1.6": 588, 39 | "1.7": 2156, 40 | "1.8": 182865, 41 | "10": 44, 42 | "11": 117976, 43 | "12": 45, 44 | "13": 42, 45 | "14": 64, 46 | "15": 126, 47 | "16": 63, 48 | "17": 1331, 49 | "18": 216, 50 | "9": 26 51 | } 52 | } 53 | $ python -c 'from datetime import datetime; print(datetime.utcfromtimestamp(1654041600000 / 1000.0))' 54 | 2022-06-01 00:00:00 55 | ``` 56 | 57 | ### Development 58 | 59 | #### Setup 60 | 61 | You will need to have `make`, `go` (1.17 or later), and, if you want to run the unit tests, `docker`, installed. 62 | 63 | #### Testing 64 | 65 | Make sure you have Docker running, and run `make test` to execute the unit tests. 66 | 67 | #### Format and linting 68 | 69 | Run `make fmt lint` to format the Go code and report on any linting/static analysis problems. 70 | 71 | #### Building 72 | 73 | Run `make build` to generate `build/jenkins-usage-stats`, compiled for your current platform. 74 | 75 | #### Local usage 76 | 77 | Run `make migrate` - by default, this will expect to be able to use the `postgres` user against localhost without authentication, and that the `jenkins_usage_stats` database already exists. You can use `testdata/report-stats` as a small (22 days) dataset to import. 78 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME := jenkins-usage-stats 2 | BINARY_NAME := jenkins-usage-stats 3 | 4 | DATABASE_URL ?= postgres://postgres@localhost/jenkins_usage_stats?sslmode=disable&timezone=UTC 5 | 6 | MIGRATE_VERSION := v4.15.1 7 | TESTFIXTURES_VERSION := v3.6.1 8 | 9 | GO := GO111MODULE=on GO15VENDOREXPERIMENT=1 go 10 | GO_NOMOD := GO111MODULE=off go 11 | GOTEST := $(GO) test 12 | 13 | GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) 14 | GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) 15 | 16 | REV := $(shell git rev-parse --short HEAD 2> /dev/null || echo 'unknown') 17 | SHA1 := $(shell git rev-parse HEAD 2> /dev/null || echo 'unknown') 18 | BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null || echo 'unknown') 19 | BUILD_DATE := $(shell date +%Y%m%d-%H:%M:%S) 20 | 21 | # set dev version unless VERSION is explicitly set via environment 22 | VERSION ?= $(shell echo "$$(git describe --abbrev=0 --tags 2>/dev/null)-dev+$(REV)" | sed 's/^v//') 23 | GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') 24 | 25 | ORG := jenkins-infra 26 | ORG_REPO := $(ORG)/$(NAME) 27 | RELEASE_ORG_REPO := $(ORG_REPO) 28 | ROOT_PACKAGE := github.com/$(ORG_REPO) 29 | 30 | BUILD_TARGET=build 31 | REPORTS_DIR=$(BUILD_TARGET)/reports 32 | 33 | COVER_OUT:=$(REPORTS_DIR)/cover.out 34 | COVERFLAGS=-coverprofile=$(COVER_OUT) --covermode=atomic --coverpkg=./... 35 | 36 | # If available, use gotestsum which provides more comprehensive output 37 | # This is used in the CI builds 38 | ifneq (, $(shell which gotestsum 2> /dev/null)) 39 | GOTESTSUM_FORMAT ?= standard-quiet 40 | GOTEST := GO111MODULE=on gotestsum --junitfile $(REPORTS_DIR)/integration.junit.xml --format $(GOTESTSUM_FORMAT) -- 41 | endif 42 | 43 | GOLANGCI_LINT := 44 | GOLANGCI_LINT_OPTS ?= 45 | GOLANGCI_LINT_VERSION ?= v1.41.1 46 | # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. 47 | # windows isn't included here because of the path separator being different. 48 | ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) 49 | ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) 50 | GOLANGCI_LINT := $(GOPATH)/bin/golangci-lint 51 | endif 52 | endif 53 | 54 | BUILDFLAGS := -ldflags \ 55 | " -X $(ROOT_PACKAGE)/pkg/version.Version=$(VERSION)\ 56 | -X $(ROOT_PACKAGE)/pkg/version.Revision=$(REV)\ 57 | -X $(ROOT_PACKAGE)/pkg/version.Sha1=$(SHA1)\ 58 | -X $(ROOT_PACKAGE)/pkg/version.Branch='$(BRANCH)'\ 59 | -X $(ROOT_PACKAGE)/pkg/version.BuildDate='$(BUILD_DATE)'\ 60 | -X $(ROOT_PACKAGE)/pkg/version.GoVersion='$(GO_VERSION)'" 61 | 62 | ifdef DEBUG 63 | BUILDFLAGS += -gcflags "all=-N -l" 64 | endif 65 | 66 | .PHONY: build 67 | build: 68 | CGO_ENABLED=$(CGO_ENABLED) $(GO) build $(BUILDFLAGS) -o $(BUILD_TARGET)/$(BINARY_NAME) ./cmd/jenkins-usage-stats/... 69 | 70 | get-migrate-deps: 71 | $(GO) install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@$(MIGRATE_VERSION) 72 | 73 | .PHONY: migrate 74 | migrate: get-migrate-deps 75 | @echo "MIGRATING DB" 76 | migrate -database "$(DATABASE_URL)" -source file://etc/migrations up 77 | 78 | get-testfixture-deps: 79 | $(GO) install github.com/go-testfixtures/testfixtures/v3/cmd/testfixtures@$(TESTFIXTURES_VERSION) 80 | 81 | .PHONY: dump-fixtures 82 | dump-fixtures: get-testfixture-deps 83 | @echo "DUMPING FIXTURES FROM DATABASE" 84 | testfixtures --dump -d postgres -c "$(DATABASE_URL)" -D testdata/fixtures --files os_types,job_types,plugins,instance_reports,jenkins_versions,report_files,jvm_versions 85 | 86 | get-fmt-deps: 87 | $(GO) install golang.org/x/tools/cmd/goimports@latest 88 | 89 | .PHONY: importfmt 90 | importfmt: get-fmt-deps ## Checks the import format of the Go source files 91 | @echo "FORMATTING IMPORTS" 92 | @goimports -l -e -w . 93 | 94 | .PHONY: fmt ## Checks Go source files are formatted properly 95 | fmt: importfmt 96 | @echo "FORMATTING SOURCE" 97 | FORMATTED=`$(GO) fmt ./...` 98 | @([ ! -z "$(FORMATTED)" ] && printf "Fixed un-formatted files:\n$(FORMATTED)") || true 99 | 100 | ifdef GOLANGCI_LINT 101 | $(GOLANGCI_LINT): 102 | mkdir -p $(GOPATH)/bin 103 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \ 104 | | sed -e '/install -d/d' \ 105 | | sh -s -- -b $(GOPATH)/bin $(GOLANGCI_LINT_VERSION) 106 | endif 107 | 108 | .PHONY: lint 109 | lint: $(GOLANGCI_LINT) 110 | @echo "--> Running golangci-lint" 111 | golangci-lint run 112 | 113 | .PHONY: make-reports-dir 114 | make-reports-dir: 115 | mkdir -p $(REPORTS_DIR) 116 | 117 | .PHONY: test 118 | test: make-reports-dir ## Runs the unit tests 119 | CGO_ENABLED=$(CGO_ENABLED) DATABASE_URL=$(DATABASE_URL) $(GOTEST) -p 1 $(COVERFLAGS) $(BUILDFLAGS) -race -timeout=600s -v -short ./... 120 | -------------------------------------------------------------------------------- /testdata/reports/GetLatestPluginNumbers.json: -------------------------------------------------------------------------------- 1 | { 2 | "month": 1259625600000, 3 | "plugins": { 4 | "SCTMExecutor": 10, 5 | "Schmant": 3, 6 | "URLSCM": 234, 7 | "accurev": 52, 8 | "active-directory": 940, 9 | "analysis-core": 864, 10 | "artifactory": 43, 11 | "audit-trail": 659, 12 | "backlog": 44, 13 | "backup": 651, 14 | "bamboo-notifier": 1, 15 | "batch-task": 1026, 16 | "bazaar": 43, 17 | "bitkeeper": 10, 18 | "blame-upstream-commiters": 2, 19 | "bruceschneier": 32, 20 | "buckminster": 32, 21 | "bugzilla": 344, 22 | "build-publisher": 262, 23 | "build-timeout": 1075, 24 | "campfire": 17, 25 | "cccc": 128, 26 | "changelog-history": 397, 27 | "checkstyle": 1618, 28 | "chucknorris": 757, 29 | "ci-game": 931, 30 | "claim": 304, 31 | "clearcase": 390, 32 | "clearcase-release": 16, 33 | "clover": 422, 34 | "cmakebuilder": 64, 35 | "cmvc": 2, 36 | "cobertura": 1761, 37 | "codescanner": 10, 38 | "collabnet": 15, 39 | "configurationslicing": 235, 40 | "copy-to-slave": 69, 41 | "copyarchiver": 164, 42 | "cppcheck": 126, 43 | "cppncss": 12, 44 | "cpptest": 10, 45 | "cppunit": 166, 46 | "crap4j": 268, 47 | "createjobadvanced": 70, 48 | "crowd": 40, 49 | "cvs": 7, 50 | "cvs-tag": 205, 51 | "cygpath": 22, 52 | "dashboard-view": 445, 53 | "dbCharts": 21, 54 | "dependencyanalyzer": 399, 55 | "deploy": 952, 56 | "description-setter": 363, 57 | "dimensionsscm": 11, 58 | "disk-usage": 1459, 59 | "distfork": 45, 60 | "doclinks": 250, 61 | "downstream-buildview": 3, 62 | "downstream-ext": 56, 63 | "doxygen": 265, 64 | "drools": 16, 65 | "dry": 575, 66 | "easyant": 37, 67 | "ec2": 69, 68 | "email-ext": 2014, 69 | "emma": 956, 70 | "emotional-hudson": 1196, 71 | "extended-read-permission": 51, 72 | "filesystem_scm": 85, 73 | "findbugs": 1967, 74 | "fitnesse": 1, 75 | "fortify360": 2, 76 | "ftppublisher": 353, 77 | "gallio": 40, 78 | "gant": 114, 79 | "gcal": 161, 80 | "git": 602, 81 | "github": 109, 82 | "gnat": 6, 83 | "googleanalytics": 57, 84 | "googlecode": 70, 85 | "gradle": 58, 86 | "grails": 198, 87 | "greenballs": 1459, 88 | "grinder": 85, 89 | "groovy": 319, 90 | "hadoop": 17, 91 | "harvest": 14, 92 | "hgca": 41, 93 | "htmlpublisher": 201, 94 | "hudson-logaction-plugin": 78, 95 | "hudson-pview-plugin": 153, 96 | "hudson-wsclean-plugin": 108, 97 | "hudsontrayapp": 138, 98 | "instant-messaging": 352, 99 | "ircbot": 173, 100 | "ivy": 177, 101 | "jabber": 382, 102 | "japex": 101, 103 | "javancss": 263, 104 | "javanet": 24, 105 | "javanet-uploader": 22, 106 | "javatest-report": 77, 107 | "jdepend": 240, 108 | "jira": 1038, 109 | "job-exporter": 22, 110 | "jobrevision": 32, 111 | "jobtype-column": 36, 112 | "join": 251, 113 | "jsunit": 37, 114 | "jswidgets": 21, 115 | "junit-attachments": 49, 116 | "jython": 43, 117 | "kagemai": 10, 118 | "klaros-testmanagement": 4, 119 | "kundo": 8, 120 | "lastfailureversioncolumn": 57, 121 | "lastsuccessversioncolumn": 54, 122 | "ldapemail": 345, 123 | "locale": 281, 124 | "locked-files-report": 39, 125 | "locks-and-latches": 511, 126 | "m2-extra-steps": 491, 127 | "m2-repo-reaper": 47, 128 | "m2release": 660, 129 | "mantis": 224, 130 | "maven-info": 169, 131 | "maven-plugin": 8107, 132 | "mercurial": 290, 133 | "monitoring": 185, 134 | "mozmill": 2, 135 | "msbuild": 633, 136 | "mstest": 189, 137 | "nabaztag": 32, 138 | "naginator": 147, 139 | "nant": 278, 140 | "ncover": 122, 141 | "next-build-number": 100, 142 | "nunit": 443, 143 | "parameterized-trigger": 444, 144 | "perforce": 346, 145 | "phing": 69, 146 | "piwikanalytics": 5, 147 | "platformlabeler": 29, 148 | "plot": 473, 149 | "pmd": 1494, 150 | "polarion": 191, 151 | "port-allocator": 130, 152 | "postbuild-task": 294, 153 | "powershell": 84, 154 | "promoted-builds": 356, 155 | "promoted-builds-simple": 96, 156 | "pvcs_scm": 13, 157 | "pxe": 31, 158 | "python": 228, 159 | "rad-builder": 8, 160 | "radiatorviewplugin": 345, 161 | "rake": 281, 162 | "redmine": 249, 163 | "regexemail": 125, 164 | "release": 465, 165 | "ruby": 272, 166 | "rubyMetrics": 152, 167 | "schedule-failed-builds": 91, 168 | "scis-ad": 6618, 169 | "scons": 37, 170 | "scp": 812, 171 | "screenshot": 68, 172 | "script-realm": 6, 173 | "secret": 28, 174 | "sectioned-view": 312, 175 | "selenium": 162, 176 | "selenium-aes": 172, 177 | "seleniumhq": 244, 178 | "serenity": 2, 179 | "setenv": 297, 180 | "sfee": 4, 181 | "sidebar-link": 457, 182 | "slave-status": 222, 183 | "sloccount": 311, 184 | "sonar": 1009, 185 | "sounds": 29, 186 | "speaks": 30, 187 | "ssh-slaves": 7999, 188 | "starteam": 43, 189 | "statusmonitor": 390, 190 | "subversion": 7550, 191 | "svn-release-mgr": 548, 192 | "svn-tag": 932, 193 | "svncompat13": 92, 194 | "svncompat14": 61, 195 | "swarm": 66, 196 | "synergy": 35, 197 | "tasks": 1238, 198 | "template-project": 211, 199 | "testabilityexplorer": 152, 200 | "text-finder": 495, 201 | "tfs": 66, 202 | "trac": 643, 203 | "tracking-svn": 31, 204 | "translation": 28, 205 | "tuxdroid": 7, 206 | "twitter": 209, 207 | "url-change-trigger": 245, 208 | "validating-string-parameter": 46, 209 | "versionnumber": 168, 210 | "viewVC": 220, 211 | "violations": 1191, 212 | "vmware": 181, 213 | "vss": 106, 214 | "warnings": 1462, 215 | "was-builder": 4, 216 | "webtestpresenter": 93, 217 | "windmill": 23, 218 | "xfpanel": 140, 219 | "xunit": 242, 220 | "xvnc": 250, 221 | "zentimestamp": 128 222 | } 223 | } -------------------------------------------------------------------------------- /templates/svgs.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 14 | 15 | 16 |
17 |
18 |
19 |

Some statistics on the usage of Jenkins

20 |
21 | 22 | 23 | {{range $totalFile := .totalFiles}} 24 | 29 | {{end}} 30 | 31 |
25 | {{$totalFile}}.svg 26 | CSV 27 | 28 |
32 |
33 |
34 |
35 |

Statistics by months

36 |
37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | {{range $monthData := .months}} 51 | 52 | 53 | 58 | 63 | 68 | 73 | 78 | 83 | 88 | 93 | 98 | 99 | {{end}} 100 |
MonthjenkinsjobsnodesnodesPiepluginstop-plugins1000top-plugins2500top-plugins500total-executors
{{$monthData.Year}}-{{$monthData.Num}} ({{$monthData.Name}}) 54 | SVG 55 | / 56 | CSV 57 | 59 | SVG 60 | / 61 | CSV 62 | 64 | SVG 65 | / 66 | CSV 67 | 69 | SVG 70 | / 71 | CSV 72 | 74 | SVG 75 | / 76 | CSV 77 | 79 | SVG 80 | / 81 | CSV 82 | 84 | SVG 85 | / 86 | CSV 87 | 89 | SVG 90 | / 91 | CSV 92 | 94 | SVG 95 | / 96 | CSV 97 |
101 | 110 |
111 |
112 | 113 | 114 | -------------------------------------------------------------------------------- /report_test.go: -------------------------------------------------------------------------------- 1 | package stats_test 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | sq "github.com/Masterminds/squirrel" 15 | testfixtures "github.com/go-testfixtures/testfixtures/v3" 16 | _ "github.com/golang-migrate/migrate/v4/source/file" 17 | stats "github.com/jenkins-infra/jenkins-usage-stats" 18 | "github.com/jenkins-infra/jenkins-usage-stats/testutil" 19 | "github.com/stretchr/testify/assert" 20 | "github.com/stretchr/testify/require" 21 | "gopkg.in/yaml.v2" 22 | ) 23 | 24 | func TestReportFuncs(t *testing.T) { 25 | var getDBFunc func(*testing.T) (sq.BaseRunner, func()) 26 | 27 | getDBFunc = dbWithFixtures 28 | // If USE_JENKINS_STATS_IT_DB is set, use the integration test DB rather than testcontainers-go. 29 | if os.Getenv("USE_JENKINS_STATS_IT_DB") != "" { 30 | getDBFunc = useITDB 31 | } 32 | 33 | // Pre-load the database with the fixtures we'll be using for the tests. We only do this once for all report func 34 | // tests because it takes ~45s to spin up the postgres container and load the fixtures into it on my beefy MBP. 35 | db, closeFunc := getDBFunc(t) 36 | defer closeFunc() 37 | 38 | // Make sure we have the same number of instance reports in the database that we do in the fixtures. 39 | var c int 40 | require.NoError(t, stats.PSQL(db).Select("count(*)").From(stats.InstanceReportsTable).QueryRow().Scan(&c)) 41 | rawYaml, err := ioutil.ReadFile(filepath.Join("testdata", "fixtures", "instance_reports.yml")) 42 | require.NoError(t, err) 43 | var allYamlReports []interface{} 44 | require.NoError(t, yaml.Unmarshal(rawYaml, &allYamlReports)) 45 | assert.Equal(t, len(allYamlReports), c) 46 | 47 | t.Run("GetInstallCountsForVersions", func(t *testing.T) { 48 | ir, err := stats.GetInstallCountForVersions(db, 2009, 12) 49 | require.NoError(t, err) 50 | 51 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, ir) 52 | 53 | var goldenIR stats.InstallationReport 54 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenIR)) 55 | 56 | assert.Equal(t, goldenIR, ir) 57 | }) 58 | 59 | t.Run("GetLatestPluginNumbers", func(t *testing.T) { 60 | pn, err := stats.GetLatestPluginNumbers(db, 2009, 12) 61 | require.NoError(t, err) 62 | 63 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 64 | 65 | var goldenPN stats.LatestPluginNumbersReport 66 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 67 | 68 | assert.Equal(t, goldenPN, pn) 69 | }) 70 | 71 | t.Run("GetCapabilities", func(t *testing.T) { 72 | pn, err := stats.GetCapabilities(db, 2009, 12) 73 | require.NoError(t, err) 74 | 75 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 76 | 77 | var goldenPN stats.CapabilitiesReport 78 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 79 | 80 | assert.Equal(t, goldenPN, pn) 81 | }) 82 | 83 | t.Run("JobCountsForMonth", func(t *testing.T) { 84 | pn, err := stats.JobCountsForMonth(db, 2009, 12) 85 | require.NoError(t, err) 86 | 87 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 88 | 89 | var goldenPN map[string]uint64 90 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 91 | 92 | assert.Equal(t, goldenPN, pn) 93 | }) 94 | 95 | t.Run("OSCountsForMonth", func(t *testing.T) { 96 | pn, err := stats.OSCountsForMonth(db, 2009, 12) 97 | require.NoError(t, err) 98 | 99 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 100 | 101 | var goldenPN map[string]uint64 102 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 103 | 104 | assert.Equal(t, goldenPN, pn) 105 | }) 106 | 107 | t.Run("GetJVMReports", func(t *testing.T) { 108 | pn, err := stats.GetJVMsReport(db, 2010, 2) 109 | require.NoError(t, err) 110 | 111 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 112 | 113 | var goldenPN stats.JVMReport 114 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 115 | 116 | assert.Equal(t, goldenPN, pn) 117 | }) 118 | 119 | t.Run("GetPluginReports", func(t *testing.T) { 120 | pn, err := stats.GetPluginReports(db, 2010, 2) 121 | require.NoError(t, err) 122 | 123 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 124 | 125 | var goldenPN []stats.PluginReport 126 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 127 | 128 | assert.Equal(t, goldenPN, pn) 129 | }) 130 | 131 | t.Run("JenkinsVersionsForPluginVersions", func(t *testing.T) { 132 | orderedPN, err := stats.JenkinsVersionsForPluginVersions(db, 2010, 1) 133 | require.NoError(t, err) 134 | 135 | // Need to jump through some hoops to compare the map[string]*stats.PVDPluginVersionMap we get from stats.JenkinsVersionsForPluginVersions 136 | // to the map[string]map[string]map[string]uint64 we get from unmarshalling the JSON. 137 | pn := make(map[string]map[string]map[string]uint64) 138 | for k, v := range orderedPN { 139 | pn[k] = make(map[string]map[string]uint64) 140 | pvIter := v.EntriesIter() 141 | for { 142 | pvPair, ok := pvIter() 143 | if !ok { 144 | break 145 | } 146 | pn[k][pvPair.Key] = make(map[string]uint64) 147 | 148 | asJVMap := pvPair.Value.(*stats.PVDJenkinsVersionMap) 149 | 150 | jvIter := asJVMap.EntriesIter() 151 | for { 152 | jvPair, ok := jvIter() 153 | if !ok { 154 | break 155 | } 156 | pn[k][pvPair.Key][jvPair.Key] = jvPair.Value.(uint64) 157 | } 158 | } 159 | } 160 | goldenBytes := jsonReadGoldenAndUpdateIfDesired(t, pn) 161 | 162 | var goldenPN map[string]map[string]map[string]uint64 163 | require.NoError(t, json.Unmarshal(goldenBytes, &goldenPN)) 164 | 165 | assert.Equal(t, goldenPN, pn) 166 | }) 167 | 168 | t.Run("ExecutorCountsForMonth", func(t *testing.T) { 169 | pn, err := stats.ExecutorCountsForMonth(db, 2010, 1) 170 | require.NoError(t, err) 171 | 172 | execSVG, execCSV, err := stats.CreateBarSVG(fmt.Sprintf("Executors per install (total: %d)", 5), pn, 25, false, false, true, stats.DefaultFilter) 173 | require.NoError(t, err) 174 | 175 | goldenSVG := rawReadGoldenAndUpdateIfDesired(t, execSVG, "svg") 176 | assert.Equal(t, string(goldenSVG), string(execSVG)) 177 | 178 | goldenCSV := rawReadGoldenAndUpdateIfDesired(t, execCSV, "csv") 179 | assert.Equal(t, string(goldenCSV), string(execCSV)) 180 | }) 181 | 182 | t.Run("GenerateReport", func(t *testing.T) { 183 | tmpOut, err := os.MkdirTemp("", "out-dir-") 184 | require.NoError(t, err) 185 | defer func() { 186 | _ = os.RemoveAll(tmpOut) 187 | }() 188 | 189 | require.NoError(t, stats.GenerateReport(db, 2010, 1, tmpOut)) 190 | }) 191 | } 192 | 193 | func jsonReadGoldenAndUpdateIfDesired(t *testing.T, input interface{}) []byte { 194 | testName := strings.Split(t.Name(), "/")[1] 195 | 196 | goldenFile := filepath.Join("testdata", "reports", fmt.Sprintf("%s.json", testName)) 197 | 198 | if os.Getenv("UPDATE_GOLDEN") == "" { 199 | jb, err := json.MarshalIndent(input, "", " ") 200 | require.NoError(t, err) 201 | require.NoError(t, ioutil.WriteFile(goldenFile, jb, 0644)) //nolint:gosec 202 | } 203 | 204 | goldenBytes, err := ioutil.ReadFile(goldenFile) //nolint:gosec 205 | require.NoError(t, err) 206 | 207 | return goldenBytes 208 | } 209 | 210 | func rawReadGoldenAndUpdateIfDesired(t *testing.T, input []byte, suffix string) []byte { 211 | testName := strings.Split(t.Name(), "/")[1] 212 | 213 | goldenFile := filepath.Join("testdata", "reports", fmt.Sprintf("%s.%s", testName, suffix)) 214 | 215 | if os.Getenv("UPDATE_GOLDEN") != "" { 216 | require.NoError(t, ioutil.WriteFile(goldenFile, input, 0644)) //nolint:gosec 217 | } 218 | 219 | goldenBytes, err := ioutil.ReadFile(goldenFile) //nolint:gosec 220 | require.NoError(t, err) 221 | 222 | return goldenBytes 223 | } 224 | 225 | func dbWithFixtures(t *testing.T) (sq.BaseRunner, func()) { 226 | db, closeFunc := testutil.DBForTest(t) 227 | fixtures, err := testfixtures.New( 228 | testfixtures.Database(db), 229 | testfixtures.Dialect("postgres"), 230 | testfixtures.Directory(filepath.Join("testdata", "fixtures")), 231 | // Make sure we don't inadvertently bork sequences 232 | testfixtures.ResetSequencesTo(30000), 233 | // We store timestamps in UTC 234 | testfixtures.Location(time.UTC)) 235 | if err != nil { 236 | closeFunc() 237 | t.Fatal(err) 238 | } 239 | 240 | err = fixtures.Load() 241 | if err != nil { 242 | closeFunc() 243 | t.Fatal(err) 244 | } 245 | 246 | return sq.NewStmtCacheProxy(db), closeFunc 247 | } 248 | 249 | func useITDB(t *testing.T) (sq.BaseRunner, func()) { 250 | databaseURL := os.Getenv("DATABASE_URL") 251 | if databaseURL == "" { 252 | databaseURL = "postgres://postgres@localhost/jenkins_usage_stats_test?sslmode=disable&timezone=UTC" 253 | } 254 | 255 | db, err := sql.Open("postgres", databaseURL) 256 | if err != nil { 257 | t.Fatal(err) 258 | } 259 | 260 | closeFunc := func() { 261 | if err := db.Close(); err != nil { 262 | t.Fatal(err) 263 | } 264 | } 265 | 266 | return sq.NewStmtCacheProxy(db), closeFunc 267 | } 268 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package stats_test 2 | 3 | import ( 4 | "database/sql" 5 | "path/filepath" 6 | "testing" 7 | 8 | sq "github.com/Masterminds/squirrel" 9 | _ "github.com/golang-migrate/migrate/v4/source/file" 10 | stats "github.com/jenkins-infra/jenkins-usage-stats" 11 | "github.com/jenkins-infra/jenkins-usage-stats/testutil" 12 | _ "github.com/lib/pq" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestGetJVMVersionID(t *testing.T) { 18 | db, closeFunc := testutil.DBForTest(t) 19 | defer closeFunc() 20 | 21 | cache := stats.NewStatsCache() 22 | firstVer := "1.7" 23 | secondVer := "13" 24 | 25 | var fetchedVersion stats.JVMVersion 26 | err := stats.PSQL(db).Select("id", "name").From(stats.JVMVersionsTable).Where(sq.Eq{"name": firstVer}). 27 | QueryRow().Scan(&fetchedVersion.ID, &fetchedVersion.Name) 28 | require.Equal(t, sql.ErrNoRows, err) 29 | 30 | firstID, err := stats.GetJVMVersionID(db, cache, firstVer) 31 | require.NoError(t, err) 32 | require.NoError(t, stats.PSQL(db).Select("id", "name").From(stats.JVMVersionsTable).Where(sq.Eq{"name": firstVer}). 33 | QueryRow().Scan(&fetchedVersion.ID, &fetchedVersion.Name)) 34 | assert.Equal(t, firstID, fetchedVersion.ID) 35 | 36 | secondID, err := stats.GetJVMVersionID(db, cache, secondVer) 37 | require.NoError(t, err) 38 | assert.NotEqual(t, firstID, secondID) 39 | } 40 | 41 | func TestGetOSTypeID(t *testing.T) { 42 | db, closeFunc := testutil.DBForTest(t) 43 | defer closeFunc() 44 | 45 | cache := stats.NewStatsCache() 46 | 47 | firstVer := "Windows 11" 48 | secondVer := "Ubuntu something" 49 | 50 | var fetchedOS stats.OSType 51 | err := stats.PSQL(db).Select("id", "name").From(stats.OSTypesTable).Where(sq.Eq{"name": firstVer}). 52 | QueryRow().Scan(&fetchedOS.ID, &fetchedOS.Name) 53 | require.Equal(t, sql.ErrNoRows, err) 54 | 55 | firstID, err := stats.GetOSTypeID(db, cache, firstVer) 56 | require.NoError(t, err) 57 | require.NoError(t, stats.PSQL(db).Select("id", "name").From(stats.OSTypesTable).Where(sq.Eq{"name": firstVer}). 58 | QueryRow().Scan(&fetchedOS.ID, &fetchedOS.Name)) 59 | assert.Equal(t, firstID, fetchedOS.ID) 60 | 61 | secondID, err := stats.GetOSTypeID(db, cache, secondVer) 62 | require.NoError(t, err) 63 | assert.NotEqual(t, firstID, secondID) 64 | } 65 | 66 | func TestGetJobTypeID(t *testing.T) { 67 | db, closeFunc := testutil.DBForTest(t) 68 | defer closeFunc() 69 | 70 | cache := stats.NewStatsCache() 71 | 72 | firstVer := "hudson-maven-MavenModuleSet" 73 | secondVer := "org-jenkinsci-plugins-workflow-job-WorkflowJob" 74 | 75 | var fetchedJobType stats.JobType 76 | err := stats.PSQL(db).Select("id", "name").From(stats.JobTypesTable).Where(sq.Eq{"name": firstVer}). 77 | QueryRow().Scan(&fetchedJobType.ID, &fetchedJobType.Name) 78 | require.Equal(t, sql.ErrNoRows, err) 79 | 80 | firstID, err := stats.GetJobTypeID(db, cache, firstVer) 81 | require.NoError(t, err) 82 | require.NoError(t, stats.PSQL(db).Select("id", "name").From(stats.JobTypesTable).Where(sq.Eq{"name": firstVer}). 83 | QueryRow().Scan(&fetchedJobType.ID, &fetchedJobType.Name)) 84 | assert.Equal(t, firstID, fetchedJobType.ID) 85 | 86 | secondID, err := stats.GetJobTypeID(db, cache, secondVer) 87 | require.NoError(t, err) 88 | assert.NotEqual(t, firstID, secondID) 89 | } 90 | 91 | func TestGetJenkinsVersionID(t *testing.T) { 92 | db, closeFunc := testutil.DBForTest(t) 93 | defer closeFunc() 94 | 95 | cache := stats.NewStatsCache() 96 | 97 | firstVer := "1.500" 98 | secondVer := "2.201.1" 99 | 100 | var fetchedJV stats.JenkinsVersion 101 | err := stats.PSQL(db).Select("id", "version").From(stats.JenkinsVersionsTable).Where(sq.Eq{"version": firstVer}). 102 | QueryRow().Scan(&fetchedJV.ID, &fetchedJV.Version) 103 | require.Equal(t, sql.ErrNoRows, err) 104 | 105 | firstID, err := stats.GetJenkinsVersionID(db, cache, firstVer) 106 | require.NoError(t, err) 107 | require.NoError(t, stats.PSQL(db).Select("id", "version").From(stats.JenkinsVersionsTable).Where(sq.Eq{"version": firstVer}). 108 | QueryRow().Scan(&fetchedJV.ID, &fetchedJV.Version)) 109 | assert.Equal(t, firstID, fetchedJV.ID) 110 | 111 | secondID, err := stats.GetJenkinsVersionID(db, cache, secondVer) 112 | require.NoError(t, err) 113 | assert.NotEqual(t, firstID, secondID) 114 | } 115 | 116 | func TestGetPluginID(t *testing.T) { 117 | db, closeFunc := testutil.DBForTest(t) 118 | defer closeFunc() 119 | 120 | cache := stats.NewStatsCache() 121 | 122 | firstName := "first-plugin" 123 | firstVer := "1.0" 124 | secondVer := "2.0" 125 | secondName := "second-plugin" 126 | 127 | var fetchedPlugin stats.Plugin 128 | err := stats.PSQL(db).Select("id", "name", "version").From(stats.PluginsTable).Where(sq.Eq{"name": firstName}).Where(sq.Eq{"version": firstVer}). 129 | QueryRow().Scan(&fetchedPlugin.ID, &fetchedPlugin.Name, &fetchedPlugin.Version) 130 | require.Equal(t, sql.ErrNoRows, err) 131 | 132 | firstID, err := stats.GetPluginID(db, cache, firstName, firstVer) 133 | require.NoError(t, err) 134 | require.NoError(t, stats.PSQL(db).Select("id", "name", "version").From(stats.PluginsTable).Where(sq.Eq{"name": firstName}).Where(sq.Eq{"version": firstVer}). 135 | QueryRow().Scan(&fetchedPlugin.ID, &fetchedPlugin.Name, &fetchedPlugin.Version)) 136 | assert.Equal(t, firstID, fetchedPlugin.ID) 137 | 138 | secondID, err := stats.GetPluginID(db, cache, firstName, secondVer) 139 | require.NoError(t, err) 140 | assert.NotEqual(t, firstID, secondID) 141 | 142 | otherPluginID, err := stats.GetPluginID(db, cache, secondName, firstVer) 143 | require.NoError(t, err) 144 | assert.NotEqual(t, firstID, otherPluginID) 145 | } 146 | 147 | func TestAddIndividualReport(t *testing.T) { 148 | db, closeFunc := testutil.DBForTest(t) 149 | defer closeFunc() 150 | 151 | cache := stats.NewStatsCache() 152 | 153 | initialFile := filepath.Join("testdata", "base.json.gz") 154 | jsonReports, err := stats.ParseDailyJSON(initialFile) 155 | require.NoError(t, err) 156 | 157 | for _, jr := range jsonReports { 158 | require.NoError(t, stats.AddIndividualReport(db, cache, jr)) 159 | } 160 | 161 | result, err := stats.PSQL(db).Select("count(*)"). 162 | From(stats.InstanceReportsTable). 163 | Query() 164 | require.NoError(t, err) 165 | 166 | var counts []int 167 | for result.Next() { 168 | var c int 169 | require.NoError(t, result.Scan(&c)) 170 | counts = append(counts, c) 171 | } 172 | require.Len(t, counts, 1) 173 | require.Equal(t, 2, counts[0]) 174 | 175 | unchangedInstanceID := "b072fa1e15fa4529001bb1ab81a7c2f2af63284811f4f9d6c2bc511f797218c8" 176 | updatedInstanceID := "32b68faa8644852c4ad79540b4bfeb1caf63284811f4f9d6c2bc511f797218c8" 177 | 178 | // Get the job type IDs for "com-tikal-jenkins-plugins-multijob-MultiJobProject" and "hudson-matrix-MatrixProject" 179 | multiJobID, err := stats.GetJobTypeID(db, cache, "com-tikal-jenkins-plugins-multijob-MultiJobProject") 180 | require.NoError(t, err) 181 | matrixJobID, err := stats.GetJobTypeID(db, cache, "hudson-matrix-MatrixProject") 182 | require.NoError(t, err) 183 | 184 | var firstReports []stats.InstanceReport 185 | reportsQuery := stats.PSQL(db).Select("id", "instance_id", "report_time", "year", "month", "version", "jvm_version_id", 186 | "executors", "count_for_month", "plugins", "jobs", "nodes"). 187 | From(stats.InstanceReportsTable). 188 | OrderBy("instance_id asc") 189 | 190 | rows, err := reportsQuery.Query() 191 | require.NoError(t, err) 192 | for rows.Next() { 193 | var ir stats.InstanceReport 194 | require.NoError(t, rows.Scan(&ir.ID, &ir.InstanceID, &ir.ReportTime, &ir.Year, &ir.Month, &ir.Version, &ir.JVMVersionID, &ir.Executors, &ir.CountForMonth, &ir.Plugins, &ir.Jobs, &ir.Nodes)) 195 | firstReports = append(firstReports, ir) 196 | } 197 | assert.Len(t, firstReports, 2) 198 | 199 | var unchangedFirstReport stats.InstanceReport 200 | var updatedFirstReport stats.InstanceReport 201 | for _, r := range firstReports { 202 | switch r.InstanceID { 203 | case unchangedInstanceID: 204 | unchangedFirstReport = r 205 | case updatedInstanceID: 206 | updatedFirstReport = r 207 | } 208 | } 209 | 210 | jobMap := *updatedFirstReport.Jobs 211 | // There should be 11 MultiJobs in the initial report 212 | assert.Equal(t, 11, int(jobMap[multiJobID])) 213 | // There should be 0 MatrixProjects in the initial report 214 | assert.Equal(t, 0, int(jobMap[matrixJobID])) 215 | 216 | secondFile := filepath.Join("testdata", "day-later.json.gz") 217 | dayLaterReports, err := stats.ParseDailyJSON(secondFile) 218 | require.NoError(t, err) 219 | 220 | for _, jr := range dayLaterReports { 221 | require.NoError(t, stats.AddIndividualReport(db, cache, jr)) 222 | } 223 | 224 | var secondReports []stats.InstanceReport 225 | rows, err = reportsQuery.Query() 226 | require.NoError(t, err) 227 | for rows.Next() { 228 | var ir stats.InstanceReport 229 | require.NoError(t, rows.Scan(&ir.ID, &ir.InstanceID, &ir.ReportTime, &ir.Year, &ir.Month, &ir.Version, &ir.JVMVersionID, &ir.Executors, &ir.CountForMonth, &ir.Plugins, &ir.Jobs, &ir.Nodes)) 230 | secondReports = append(secondReports, ir) 231 | } 232 | 233 | // Make sure there are only two reports, since the second run should just overwrite the updatedInstanceID's report from the first run. 234 | assert.Len(t, secondReports, 2) 235 | 236 | var unchangedSecondReport stats.InstanceReport 237 | var updatedSecondReport stats.InstanceReport 238 | for _, r := range secondReports { 239 | switch r.InstanceID { 240 | case unchangedInstanceID: 241 | unchangedSecondReport = r 242 | case updatedInstanceID: 243 | updatedSecondReport = r 244 | } 245 | } 246 | 247 | assert.Equal(t, unchangedFirstReport, unchangedSecondReport) 248 | 249 | assert.NotEqual(t, updatedFirstReport, updatedSecondReport) 250 | // CountForMonth should be one higher 251 | assert.Equal(t, updatedFirstReport.CountForMonth+1, updatedSecondReport.CountForMonth) 252 | // There should be once less plugin in the second report 253 | assert.Len(t, updatedSecondReport.Plugins, len(updatedFirstReport.Plugins)-1) 254 | 255 | secondJobMap := *updatedSecondReport.Jobs 256 | // There should be 0 MultiJobs 257 | assert.Equal(t, 0, int(secondJobMap[multiJobID])) 258 | // There should be 10 MatrixProjects 259 | assert.Equal(t, 10, int(secondJobMap[matrixJobID])) 260 | } 261 | -------------------------------------------------------------------------------- /testdata/base.json: -------------------------------------------------------------------------------- 1 | {"install":"32b68faa8644852c4ad79540b4bfeb1caf63284811f4f9d6c2bc511f797218c8","jobs":{"com-cloudbees-hudson-plugins-folder-Folder":0,"com-tikal-jenkins-plugins-multijob-MultiJobProject":11,"hudson-matrix-MatrixProject":0,"hudson-maven-MavenModuleSet":50,"hudson-model-ExternalJob":0,"hudson-model-FreeStyleProject":48,"jenkins-branch-OrganizationFolder":0,"org-jenkinsci-plugins-workflow-job-WorkflowJob":0,"org-jenkinsci-plugins-workflow-multibranch-WorkflowMultiBranchProject":0,"private-2ab0d9d28b1b071788f80194f45c5e49f915bb58d05b8cf3100db4da2930f63bdc7e38a1f80b85a216b8d781a0f5a4a7af2f01a07f19c60ffd41f78f7d4a859116ba65011b4aec78e794ade7fb8eb77479e4d675d0164a2f337ca04f6249f0f0":0},"nodes":[{"executors":5,"jvm-name":"Java HotSpot(TM) 64-Bit Server VM","jvm-vendor":"Oracle Corporation","jvm-version":"1.8.0_191","master":true,"os":"Windows Server 2016 (amd64)"}],"plugins":[{"name":"ace-editor","version":"1.1"},{"name":"ant","version":"1.11"},{"name":"antisamy-markup-formatter","version":"2.1"},{"name":"apache-httpcomponents-client-4-api","version":"4.5.10-2.0"},{"name":"bouncycastle-api","version":"2.18"},{"name":"branch-api","version":"2.5.5"},{"name":"built-on-column","version":"1.1"},{"name":"cloudbees-folder","version":"6.11.1"},{"name":"command-launcher","version":"1.4"},{"name":"conditional-buildstep","version":"1.3.6"},{"name":"config-file-provider","version":"3.6.3"},{"name":"configurationslicing","version":"1.51"},{"name":"credentials","version":"2.3.11"},{"name":"deploy","version":"1.15"},{"name":"dingding-notifications","version":"1.9"},{"name":"display-url-api","version":"2.3.2"},{"name":"echarts-api","version":"4.8.0-2"},{"name":"email-ext","version":"2.80"},{"name":"envinject-api","version":"1.7"},{"name":"envinject","version":"2.3.0"},{"name":"external-monitor-job","version":"1.7"},{"name":"git-client","version":"3.2.0"},{"name":"git-parameter","version":"0.9.13"},{"name":"git-server","version":"1.9"},{"name":"git","version":"4.2.0"},{"name":"github-api","version":"1.106"},{"name":"github-branch-source","version":"2.6.0"},{"name":"github-oauth","version":"0.33"},{"name":"github-organization-folder","version":"1.6"},{"name":"github","version":"1.29.5"},{"name":"jackson2-api","version":"2.11.1"},{"name":"javadoc","version":"1.5"},{"name":"jdk-tool","version":"1.4"},{"name":"jenkins-multijob-plugin","version":"1.33"},{"name":"jobConfigHistory","version":"2.25"},{"name":"jquery-detached","version":"1.2.1"},{"name":"jquery","version":"1.12.4-1"},{"name":"jquery3-api","version":"3.5.1-1"},{"name":"jsch","version":"0.1.55.2"},{"name":"junit","version":"1.31"},{"name":"ldap","version":"1.25"},{"name":"localization-support","version":"1.1"},{"name":"localization-zh-cn","version":"1.0.24"},{"name":"mailer","version":"1.32.1"},{"name":"mapdb-api","version":"1.0.9.0"},{"name":"matrix-auth","version":"2.5.1"},{"name":"matrix-project","version":"1.14"},{"name":"maven-plugin","version":"3.7"},{"name":"msbuild","version":"1.29"},{"name":"nodejs","version":"1.3.5"},{"name":"pam-auth","version":"1.6"},{"name":"parameterized-trigger","version":"2.39"},{"name":"pipeline-github-lib","version":"1.0"},{"name":"plain-credentials","version":"1.7"},{"name":"plugin-util-api","version":"1.2.2"},{"name":"role-strategy","version":"2.16"},{"name":"run-condition","version":"1.3"},{"name":"scm-api","version":"2.6.3"},{"name":"script-security","version":"1.74"},{"name":"snakeyaml-api","version":"1.26.3"},{"name":"ssh-credentials","version":"1.18.1"},{"name":"structs","version":"1.20"},{"name":"subversion","version":"2.13.1"},{"name":"thinBackup","version":"1.10"},{"name":"token-macro","version":"2.12"},{"name":"trilead-api","version":"1.0.5"},{"name":"windows-slaves","version":"1.6"},{"name":"workflow-api","version":"2.40"},{"name":"workflow-cps-global-lib","version":"2.15"},{"name":"workflow-cps","version":"2.80"},{"name":"workflow-job","version":"2.40"},{"name":"workflow-multibranch","version":"2.21"},{"name":"workflow-scm-step","version":"2.10"},{"name":"workflow-step-api","version":"2.22"},{"name":"workflow-support","version":"3.4"}],"servletContainer":"jetty/9.4.25.v20191220","stat":1,"timestamp":"30/Oct/2021:23:59:54 +0000","version":"2.204.4"} 2 | {"install":"b072fa1e15fa4529001bb1ab81a7c2f2af63284811f4f9d6c2bc511f797218c8","jobs":{"com-cloudbees-hudson-plugins-folder-Folder":0,"hudson-matrix-MatrixProject":0,"hudson-maven-MavenModuleSet":0,"hudson-model-ExternalJob":0,"hudson-model-FreeStyleProject":0,"jenkins-branch-OrganizationFolder":0,"org-jenkinsci-plugins-workflow-job-WorkflowJob":11,"org-jenkinsci-plugins-workflow-multibranch-WorkflowMultiBranchProject":0},"nodes":[{"executors":2,"jvm-name":"OpenJDK 64-Bit Server VM","jvm-vendor":"Ubuntu","jvm-version":"10.0-b25","master":true,"os":"Linux (amd64)"},{"executors":1}],"plugins":[{"name":"blueocean-jwt","version":"1.24.8"},{"name":"lockable-resources","version":"2.12"},{"name":"email-ext","version":"2.84"},{"name":"blueocean-rest-impl","version":"1.24.8"},{"name":"ssh-slaves","version":"1.31.5"},{"name":"structs","version":"1.23"},{"name":"gradle","version":"1.37.1"},{"name":"chroot","version":"0.1.4"},{"name":"cloud-stats","version":"0.27"},{"name":"github-api","version":"1.133"},{"name":"blueocean-personalization","version":"1.24.8"},{"name":"blueocean-autofavorite","version":"1.2.4"},{"name":"github-branch-source","version":"2.10.2"},{"name":"git","version":"4.8.3"},{"name":"build-timeout","version":"1.20"},{"name":"jjwt-api","version":"0.11.2-9.c8b45b8bb173"},{"name":"blueocean-web","version":"1.24.8"},{"name":"git-client","version":"3.9.0"},{"name":"external-monitor-job","version":"1.7"},{"name":"copyartifact","version":"1.46.2"},{"name":"javadoc","version":"1.6"},{"name":"blueocean-config","version":"1.24.8"},{"name":"config-file-provider","version":"3.8.1"},{"name":"jenkins-design-language","version":"1.24.8"},{"name":"virtualbox","version":"0.7"},{"name":"branch-api","version":"2.7.0"},{"name":"caffeine-api","version":"2.9.2-29.v717aac953ff3"},{"name":"conditional-buildstep","version":"1.4.1"},{"name":"aws-java-sdk-minimal","version":"1.12.89-292.v2712528e879c"},{"name":"handy-uri-templates-2-api","version":"2.1.8-1.0"},{"name":"durable-task","version":"1.39"},{"name":"bootstrap5-api","version":"5.1.1-1"},{"name":"plain-credentials","version":"1.7"},{"name":"script-security","version":"1.78"},{"name":"blueocean","version":"1.24.8"},{"name":"blueocean-git-pipeline","version":"1.24.8"},{"name":"mapdb-api","version":"1.0.9.0"},{"name":"token-macro","version":"267.vcdaea6462991"},{"name":"windows-slaves","version":"1.8"},{"name":"extended-read-permission","version":"3.2"},{"name":"parameterized-trigger","version":"2.41"},{"name":"handlebars","version":"3.0.8"},{"name":"matrix-project","version":"1.18"},{"name":"workflow-scm-step","version":"2.13"},{"name":"pipeline-stage-step","version":"2.5"},{"name":"jquery-detached","version":"1.2.1"},{"name":"pubsub-light","version":"1.16"},{"name":"ssh2easy","version":"1.4"},{"name":"pam-auth","version":"1.6"},{"name":"jira","version":"3.3"},{"name":"aws-java-sdk-ec2","version":"1.12.89-292.v2712528e879c"},{"name":"pipeline-graph-analysis","version":"1.11"},{"name":"batch-task","version":"1.19"},{"name":"apache-httpcomponents-client-4-api","version":"4.5.13-1.0"},{"name":"pipeline-model-api","version":"1.9.2"},{"name":"github","version":"1.34.1"},{"name":"azure-commons","version":"1.1.3"},{"name":"periodicbackup","version":"1.7"},{"name":"bootstrap4-api","version":"4.6.0-3"},{"name":"powershell","version":"1.7"},{"name":"text-file-operations","version":"1.3.2"},{"name":"publish-over","version":"0.22"},{"name":"blueocean-github-pipeline","version":"1.24.8"},{"name":"cygpath","version":"1.5"},{"name":"workflow-step-api","version":"2.24"},{"name":"pipeline-input-step","version":"2.12"},{"name":"promoted-builds","version":"3.10"},{"name":"cloudbees-bitbucket-branch-source","version":"2.9.11"},{"name":"trilead-api","version":"1.0.13"},{"name":"workflow-multibranch","version":"2.24"},{"name":"docker-commons","version":"1.17"},{"name":"disk-usage","version":"0.28"},{"name":"display-url-api","version":"2.3.5"},{"name":"next-build-number","version":"1.7"},{"name":"cygwin-process-killer","version":"0.2"},{"name":"jdk-tool","version":"1.5"},{"name":"docker-workflow","version":"1.26"},{"name":"workflow-basic-steps","version":"2.24"},{"name":"blueocean-events","version":"1.24.8"},{"name":"popper2-api","version":"2.10.2-1"},{"name":"junit","version":"1.53"},{"name":"cmakebuilder","version":"4.1.0"},{"name":"ant","version":"1.12"},{"name":"blueocean-display-url","version":"2.4.1"},{"name":"mailer","version":"1.34"},{"name":"pipeline-milestone-step","version":"1.3.2"},{"name":"blueocean-core-js","version":"1.24.8"},{"name":"azure-credentials","version":"198.vf9c2fdfde55c"},{"name":"mercurial","version":"2.15"},{"name":"jackson2-api","version":"2.13.0-230.v59243c64b0a5"},{"name":"scm-api","version":"2.6.5"},{"name":"ws-cleanup","version":"0.39"},{"name":"echarts-api","version":"5.2.1-2"},{"name":"variant","version":"1.4"},{"name":"pipeline-rest-api","version":"2.19"},{"name":"blueocean-bitbucket-pipeline","version":"1.24.8"},{"name":"pipeline-model-declarative-agent","version":"1.1.1"},{"name":"backup","version":"1.6.1"},{"name":"snakeyaml-api","version":"1.29.1"},{"name":"blueocean-pipeline-api-impl","version":"1.24.8"},{"name":"mstestrunner","version":"1.3.0"},{"name":"blueocean-pipeline-scm-api","version":"1.24.8"},{"name":"blueocean-rest","version":"1.24.8"},{"name":"credentials","version":"2.6.1"},{"name":"pipeline-stage-view","version":"2.19"},{"name":"jsch","version":"0.1.55.2"},{"name":"authentication-tokens","version":"1.4"},{"name":"subversion","version":"2.15.0"},{"name":"git-server","version":"1.9"},{"name":"azure-vm-agents","version":"797.v31f530348574"},{"name":"rocketchatnotifier","version":"1.4.10"},{"name":"popper-api","version":"1.16.1-2"},{"name":"credentials-binding","version":"1.27"},{"name":"command-launcher","version":"1.6"},{"name":"buildgraph-view","version":"1.8"},{"name":"okhttp-api","version":"3.14.9"},{"name":"momentjs","version":"1.1.1"},{"name":"blueocean-i18n","version":"1.24.8"},{"name":"pipeline-model-definition","version":"1.9.2"},{"name":"blueocean-jira","version":"1.24.8"},{"name":"audit-trail","version":"3.10"},{"name":"bds-plugin","version":"3.1"},{"name":"plugin-util-api","version":"2.5.0"},{"name":"htmlpublisher","version":"1.25"},{"name":"maven-plugin","version":"3.12"},{"name":"font-awesome-api","version":"5.15.4-1"},{"name":"ldap","version":"2.7"},{"name":"pipeline-github-lib","version":"1.0"},{"name":"gitea","version":"1.3.0"},{"name":"pipeline-stage-tags-metadata","version":"1.9.2"},{"name":"additional-metrics","version":"1.3"},{"name":"pipeline-model-extensions","version":"1.9.2"},{"name":"sse-gateway","version":"1.24"},{"name":"favorite","version":"2.3.3"},{"name":"aws-credentials","version":"1.32"},{"name":"workflow-support","version":"3.8"},{"name":"build-publisher","version":"1.22"},{"name":"golang","version":"1.4"},{"name":"debian-package-builder","version":"1.6.11"},{"name":"workflow-durable-task-step","version":"2.40"},{"name":"msbuild","version":"1.30"},{"name":"publish-over-ssh","version":"1.22"},{"name":"nodejs","version":"1.4.1"},{"name":"blueocean-pipeline-editor","version":"1.24.8"},{"name":"jquery","version":"1.12.4-1"},{"name":"resource-disposer","version":"0.16"},{"name":"workflow-aggregator","version":"2.6"},{"name":"checks-api","version":"1.7.2"},{"name":"ace-editor","version":"1.1"},{"name":"blueocean-dashboard","version":"1.24.8"},{"name":"azure-sdk","version":"61.v6a8af1f5f5b6"},{"name":"matrix-auth","version":"2.6.8"},{"name":"workflow-cps-global-lib","version":"2.21"},{"name":"ssh-credentials","version":"1.18.1"},{"name":"ftp-rename","version":"1.2"},{"name":"jaxb","version":"2.3.0.1"},{"name":"bouncycastle-api","version":"2.25"},{"name":"change-assembly-version-plugin","version":"1.10"},{"name":"workflow-cps","version":"2.94"},{"name":"workflow-api","version":"2.47"},{"name":"antisamy-markup-formatter","version":"2.1"},{"name":"workflow-job","version":"2.42"},{"name":"run-condition","version":"1.5"},{"name":"pipeline-build-step","version":"2.15"},{"name":"blueocean-commons","version":"1.24.8"},{"name":"publish-over-ftp","version":"1.16"},{"name":"timestamper","version":"1.13"},{"name":"jquery3-api","version":"3.6.0-2"},{"name":"docker-build-step","version":"2.8"},{"name":"job-direct-mail","version":"1.5"},{"name":"cloudbees-folder","version":"6.16"},{"name":"file-operations","version":"1.11"}],"servletContainer":"jetty/9.4.38.v20210224","stat":1,"timestamp":"30/Oct/2021:23:59:55 +0000","version":"2.277.2"} 3 | -------------------------------------------------------------------------------- /testdata/reports/ExecutorCountsForMonth.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 0 (211) 4 | 5 | 1 (1138) 6 | 7 | 2 (4769) 8 | 9 | 3 (533) 10 | 11 | 4 (508) 12 | 13 | 5 (283) 14 | 15 | 6 (175) 16 | 17 | 7 (85) 18 | 19 | 8 (122) 20 | 21 | 9 (56) 22 | 23 | 10 (101) 24 | 25 | 11 (35) 26 | 27 | 12 (40) 28 | 29 | 13 (26) 30 | 31 | 14 (21) 32 | 33 | 15 (20) 34 | 35 | 16 (28) 36 | 37 | 17 (17) 38 | 39 | 18 (9) 40 | 41 | 19 (12) 42 | 43 | 20 (15) 44 | 45 | 21 (6) 46 | 47 | 22 (7) 48 | 49 | 23 (9) 50 | 51 | 24 (10) 52 | 53 | 25 (11) 54 | 55 | 26 (5) 56 | 57 | 27 (6) 58 | 59 | 28 (5) 60 | 61 | 29 (4) 62 | 63 | 30 (6) 64 | 65 | 32 (3) 66 | 67 | 33 (2) 68 | 69 | 34 (3) 70 | 71 | 35 (1) 72 | 73 | 36 (4) 74 | 75 | 37 (1) 76 | 77 | 38 (2) 78 | 79 | 39 (2) 80 | 81 | 40 (5) 82 | 83 | 41 (2) 84 | 85 | 42 (2) 86 | 87 | 43 (1) 88 | 89 | 44 (2) 90 | 91 | 45 (1) 92 | 93 | 47 (3) 94 | 95 | 48 (1) 96 | 97 | 49 (1) 98 | 99 | 50 (2) 100 | 101 | 51 (1) 102 | 103 | 52 (2) 104 | 105 | 53 (1) 106 | 107 | 55 (1) 108 | 109 | 56 (1) 110 | 111 | 58 (1) 112 | 113 | 59 (1) 114 | 115 | 61 (2) 116 | 117 | 63 (1) 118 | 119 | 66 (1) 120 | 121 | 67 (2) 122 | 123 | 69 (1) 124 | 125 | 76 (2) 126 | 127 | 77 (1) 128 | 129 | 79 (1) 130 | 131 | 108 (1) 132 | 133 | 114 (1) 134 | 135 | 138 (1) 136 | 137 | 152 (1) 138 | 139 | 192 (1) 140 | Executors per install (total: 5) 141 | 142 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "database/sql" 5 | "database/sql/driver" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "strings" 10 | "time" 11 | 12 | sq "github.com/Masterminds/squirrel" 13 | "github.com/lib/pq" 14 | ) 15 | 16 | const ( 17 | // JVMVersionsTable is the jvm_versions table name 18 | JVMVersionsTable = "jvm_versions" 19 | // OSTypesTable is the os_types table name 20 | OSTypesTable = "os_types" 21 | // JobTypesTable is the job_types table name 22 | JobTypesTable = "job_types" 23 | // PluginsTable is the plugins table name 24 | PluginsTable = "plugins" 25 | // JenkinsVersionsTable is the jenkins_versions table name 26 | JenkinsVersionsTable = "jenkins_versions" 27 | // InstanceReportsTable is the instance_reports table name 28 | InstanceReportsTable = "instance_reports" 29 | 30 | questionVersion = "???" 31 | ) 32 | 33 | // ReportFile records a daily report file which has been imported. 34 | type ReportFile struct { 35 | Filename string `db:"filename"` 36 | } 37 | 38 | // JVMVersion represents a row in the jvm_versions table 39 | type JVMVersion struct { 40 | ID uint64 `db:"id"` 41 | Name string `db:"name"` 42 | } 43 | 44 | // OSType represents a row in the os_types table 45 | type OSType struct { 46 | ID uint64 `db:"id"` 47 | Name string `db:"name"` 48 | } 49 | 50 | // JobType represents a row in the job_types table 51 | type JobType struct { 52 | ID uint64 `db:"id"` 53 | Name string `db:"name"` 54 | } 55 | 56 | // Plugin represents a row in the plugins table 57 | type Plugin struct { 58 | ID uint64 `db:"id"` 59 | Name string `db:"name"` 60 | Version string `db:"version"` 61 | } 62 | 63 | // JenkinsVersion represents a row in the jenkins_versions table 64 | type JenkinsVersion struct { 65 | ID uint64 `db:"id"` 66 | Version string `db:"version"` 67 | } 68 | 69 | // InstanceReport is a record of an individual instance's most recent report in a given month 70 | type InstanceReport struct { 71 | ID uint64 `db:"id"` 72 | InstanceID string `db:"instance_id"` 73 | ReportTime time.Time `db:"report_time"` 74 | Year int `db:"year"` 75 | Month int `db:"month"` 76 | Version uint64 `db:"version"` 77 | JVMVersionID uint64 `db:"jvm_version_id"` 78 | Executors uint64 `db:"executors"` 79 | CountForMonth uint64 `db:"count_for_month"` 80 | Plugins pq.Int64Array `db:"plugins"` 81 | Jobs *JobsForReport `db:"jobs"` 82 | Nodes *NodesForReport `db:"nodes"` 83 | } 84 | 85 | // PluginsForReport is a map of IDs from the "plugins" table seen on an instance report 86 | type PluginsForReport []uint64 87 | 88 | // Value is used for marshalling to JSON 89 | func (p *PluginsForReport) Value() (driver.Value, error) { 90 | return json.Marshal(p) 91 | } 92 | 93 | // Scan is used for unmarshalling from JSON 94 | func (p *PluginsForReport) Scan(value interface{}) error { 95 | b, ok := value.([]byte) 96 | if !ok { 97 | return errors.New("type assertion to []byte failed") 98 | } 99 | 100 | return json.Unmarshal(b, &p) 101 | } 102 | 103 | // NodesForReport is a map of IDs from the "os_types" table to counts seen on an instance report 104 | type NodesForReport map[uint64]uint64 105 | 106 | // Value is used for marshalling to JSON 107 | func (n *NodesForReport) Value() (driver.Value, error) { 108 | return json.Marshal(n) 109 | } 110 | 111 | // Scan is used for unmarshalling from JSON 112 | func (n *NodesForReport) Scan(value interface{}) error { 113 | b, ok := value.([]byte) 114 | if !ok { 115 | return errors.New("type assertion to []byte failed") 116 | } 117 | 118 | return json.Unmarshal(b, &n) 119 | } 120 | 121 | // JobsForReport is a map of IDs from the "job_types" table to counts seen on an instance report 122 | type JobsForReport map[uint64]uint64 123 | 124 | // Value is used for marshalling to JSON 125 | func (j *JobsForReport) Value() (driver.Value, error) { 126 | return json.Marshal(j) 127 | } 128 | 129 | // Scan is used for unmarshalling from JSON 130 | func (j *JobsForReport) Scan(value interface{}) error { 131 | b, ok := value.([]byte) 132 | if !ok { 133 | return errors.New("type assertion to []byte failed") 134 | } 135 | 136 | return json.Unmarshal(b, &j) 137 | } 138 | 139 | // DBCache contains caching for the stats db 140 | type DBCache struct { 141 | jvmVersions map[string]uint64 142 | osTypes map[string]uint64 143 | jobTypes map[string]uint64 144 | jenkinsVersions map[string]uint64 145 | plugins map[string]map[string]uint64 146 | 147 | getJVMVersionTime time.Duration 148 | getOSTypeTime time.Duration 149 | getJobTypeTime time.Duration 150 | getJenkinsVersionTime time.Duration 151 | getPluginTime time.Duration 152 | 153 | getInstanceReportTime time.Duration 154 | insertInstanceReportTime time.Duration 155 | updateInstanceReportTime time.Duration 156 | insertNewReportsTime time.Duration 157 | 158 | skippedForInstall int 159 | skippedForVersion int 160 | skippedForTime int 161 | skippedForJobs int 162 | } 163 | 164 | // ReportTimes returns a string with function times 165 | func (sc *DBCache) ReportTimes() string { 166 | return fmt.Sprintf(`GetJVMVersion: %s 167 | GetOSType: %s 168 | GetJobType: %s 169 | GetJenkinsVersion: %s 170 | GetPlugin: %s 171 | GetInstanceReport: %s 172 | InsertInstanceReport: %s 173 | UpdateInstanceReport: %s 174 | InsertNewReports: %s 175 | SkippedForInstall: %d 176 | SkippedForVersion: %d 177 | SkippedForTime: %d 178 | SkippedForJobs: %d 179 | `, sc.getJVMVersionTime.String(), sc.getOSTypeTime.String(), sc.getJobTypeTime.String(), sc.getJenkinsVersionTime.String(), 180 | sc.getPluginTime.String(), sc.getInstanceReportTime.String(), sc.insertInstanceReportTime.String(), sc.updateInstanceReportTime.String(), sc.insertNewReportsTime.String(), 181 | sc.skippedForInstall, sc.skippedForVersion, sc.skippedForTime, sc.skippedForJobs) 182 | } 183 | 184 | // NewStatsCache initializes a cache 185 | func NewStatsCache() *DBCache { 186 | return &DBCache{ 187 | jvmVersions: map[string]uint64{}, 188 | osTypes: map[string]uint64{}, 189 | jobTypes: map[string]uint64{}, 190 | jenkinsVersions: map[string]uint64{}, 191 | plugins: map[string]map[string]uint64{}, 192 | getJVMVersionTime: 0, 193 | getOSTypeTime: 0, 194 | getJobTypeTime: 0, 195 | getJenkinsVersionTime: 0, 196 | getPluginTime: 0, 197 | getInstanceReportTime: 0, 198 | insertInstanceReportTime: 0, 199 | updateInstanceReportTime: 0, 200 | insertNewReportsTime: 0, 201 | } 202 | } 203 | 204 | // GetJVMVersionID gets the ID for the row of this version if it exists, and creates it and returns the ID if not 205 | func GetJVMVersionID(db sq.BaseRunner, cache *DBCache, name string) (uint64, error) { 206 | start := time.Now() 207 | defer func() { 208 | cache.getJVMVersionTime += time.Since(start) 209 | }() 210 | if cached, ok := cache.jvmVersions[name]; ok { 211 | return cached, nil 212 | } 213 | var row JVMVersion 214 | err := PSQL(db).Select("id").From(JVMVersionsTable). 215 | Where(sq.Eq{"name": name}). 216 | QueryRow(). 217 | Scan(&row.ID) 218 | if errors.Is(err, sql.ErrNoRows) { 219 | var id uint64 220 | q := PSQL(db).Insert(JVMVersionsTable).Columns("name").Values(name).Suffix(`RETURNING "id"`) 221 | err = q.QueryRow().Scan(&id) 222 | if err != nil { 223 | return 0, err 224 | } 225 | cache.jvmVersions[name] = id 226 | return id, nil 227 | } 228 | if err == nil { 229 | cache.jvmVersions[name] = row.ID 230 | return row.ID, nil 231 | } 232 | return 0, err 233 | } 234 | 235 | // GetOSTypeID gets the ID for the row of this OS if it exists, and creates it and returns the ID if not 236 | func GetOSTypeID(db sq.BaseRunner, cache *DBCache, name string) (uint64, error) { 237 | start := time.Now() 238 | defer func() { 239 | cache.getOSTypeTime += time.Since(start) 240 | }() 241 | if name == "" { 242 | name = "N/A" 243 | } 244 | if cached, ok := cache.osTypes[name]; ok { 245 | return cached, nil 246 | } 247 | var row OSType 248 | err := PSQL(db).Select("id").From(OSTypesTable). 249 | Where(sq.Eq{"name": name}). 250 | QueryRow(). 251 | Scan(&row.ID) 252 | if errors.Is(err, sql.ErrNoRows) { 253 | var id uint64 254 | q := PSQL(db).Insert(OSTypesTable).Columns("name").Values(name).Suffix(`RETURNING "id"`) 255 | err = q.QueryRow().Scan(&id) 256 | if err != nil { 257 | return 0, err 258 | } 259 | cache.osTypes[name] = id 260 | return id, nil 261 | } 262 | if err == nil { 263 | cache.osTypes[name] = row.ID 264 | return row.ID, nil 265 | } 266 | return 0, err 267 | } 268 | 269 | // GetJobTypeID gets the ID for the row of this job type if it exists, and creates it and returns the ID if not 270 | func GetJobTypeID(db sq.BaseRunner, cache *DBCache, name string) (uint64, error) { 271 | start := time.Now() 272 | defer func() { 273 | cache.getJobTypeTime += time.Since(start) 274 | }() 275 | if cached, ok := cache.jobTypes[name]; ok { 276 | return cached, nil 277 | } 278 | var row JobType 279 | err := PSQL(db).Select("id").From(JobTypesTable). 280 | Where(sq.Eq{"name": name}). 281 | QueryRow(). 282 | Scan(&row.ID) 283 | if errors.Is(err, sql.ErrNoRows) { 284 | var id uint64 285 | q := PSQL(db).Insert(JobTypesTable).Columns("name").Values(name).Suffix(`RETURNING "id"`) 286 | err = q.QueryRow().Scan(&id) 287 | if err != nil { 288 | return 0, err 289 | } 290 | cache.jobTypes[name] = id 291 | return id, nil 292 | } 293 | if err == nil { 294 | cache.jobTypes[name] = row.ID 295 | return row.ID, nil 296 | } 297 | return 0, err 298 | } 299 | 300 | // GetJenkinsVersionID gets the ID for the row of this version if it exists, and creates it and returns the ID if not 301 | func GetJenkinsVersionID(db sq.BaseRunner, cache *DBCache, version string) (uint64, error) { 302 | start := time.Now() 303 | defer func() { 304 | cache.getJenkinsVersionTime += time.Since(start) 305 | }() 306 | if cached, ok := cache.jenkinsVersions[version]; ok { 307 | return cached, nil 308 | } 309 | var row JenkinsVersion 310 | err := PSQL(db).Select("id").From(JenkinsVersionsTable). 311 | Where(sq.Eq{"version": version}). 312 | QueryRow(). 313 | Scan(&row.ID) 314 | if errors.Is(err, sql.ErrNoRows) { 315 | var id uint64 316 | q := PSQL(db).Insert(JenkinsVersionsTable).Columns("version").Values(version).Suffix(`RETURNING "id"`) 317 | err = q.QueryRow().Scan(&id) 318 | if err != nil { 319 | return 0, err 320 | } 321 | cache.jenkinsVersions[version] = id 322 | return id, nil 323 | } 324 | if err == nil { 325 | cache.jenkinsVersions[version] = row.ID 326 | return row.ID, nil 327 | } 328 | return 0, err 329 | } 330 | 331 | // GetPluginID gets the ID for the row of this plugin/version if it exists, and creates it and returns the ID if not 332 | func GetPluginID(db sq.BaseRunner, cache *DBCache, name, version string) (uint64, error) { 333 | start := time.Now() 334 | defer func() { 335 | cache.getPluginTime += time.Since(start) 336 | }() 337 | if cachedPlugin, ok := cache.plugins[name]; ok { 338 | if cachedVersion, ok := cachedPlugin[version]; ok { 339 | return cachedVersion, nil 340 | } 341 | } else { 342 | cache.plugins[name] = make(map[string]uint64) 343 | } 344 | var row Plugin 345 | err := PSQL(db).Select("id").From(PluginsTable). 346 | Where(sq.Eq{"name": name}). 347 | Where(sq.Eq{"version": version}). 348 | QueryRow(). 349 | Scan(&row.ID) 350 | if errors.Is(err, sql.ErrNoRows) { 351 | var id uint64 352 | q := PSQL(db).Insert(PluginsTable).Columns("name", "version").Values(name, version).Suffix(`RETURNING "id"`) 353 | err = q.QueryRow().Scan(&id) 354 | if err != nil { 355 | return 0, err 356 | } 357 | cache.plugins[name][version] = id 358 | return id, nil 359 | } 360 | if err == nil { 361 | cache.plugins[name][version] = row.ID 362 | return row.ID, nil 363 | } 364 | return 0, err 365 | } 366 | 367 | // AddIndividualReport adds/updates the JSON report to the database, along with all related tables. 368 | func AddIndividualReport(db sq.BaseRunner, cache *DBCache, jsonReport *JSONReport) error { 369 | // Short-circuit for a few weird cases where the instance ID is >64 characters or the Jenkins version is >32 characters 370 | if len(jsonReport.Install) > 64 { 371 | cache.skippedForInstall++ 372 | return nil 373 | } 374 | if len(jsonReport.Version) > 32 { 375 | cache.skippedForVersion++ 376 | return nil 377 | } 378 | // Skip SNAPSHOT and weird ***/? Jenkins versions 379 | if strings.Contains(jsonReport.Version, "SNAPSHOT") || strings.Contains(jsonReport.Version, "***") || strings.Contains(jsonReport.Version, "?") { 380 | cache.skippedForVersion++ 381 | return nil 382 | } 383 | 384 | ts, err := jsonReport.Timestamp() 385 | if err != nil { 386 | return err 387 | } 388 | 389 | insertRow := false 390 | 391 | // Check if there's an existing report. 392 | var report InstanceReport 393 | 394 | getReportStart := time.Now() 395 | var prevReport InstanceReport 396 | rows, err := PSQL(db). 397 | Select("id", "count_for_month, report_time"). 398 | From(InstanceReportsTable). 399 | Where(sq.Eq{"instance_id": jsonReport.Install}). 400 | Where(sq.Eq{"year": ts.Year()}). 401 | Where(sq.Eq{"month": ts.Month()}). 402 | Query() 403 | defer func() { 404 | _ = rows.Close() 405 | }() 406 | if errors.Is(err, sql.ErrNoRows) { 407 | insertRow = true 408 | } else if err != nil { 409 | return err 410 | } else { 411 | for rows.Next() { 412 | err = rows.Scan(&prevReport.ID, &prevReport.CountForMonth, &prevReport.ReportTime) 413 | if err != nil { 414 | return err 415 | } 416 | } 417 | } 418 | cache.getInstanceReportTime += time.Since(getReportStart) 419 | 420 | if prevReport.CountForMonth == 0 { 421 | insertRow = true 422 | } 423 | 424 | report.CountForMonth = prevReport.CountForMonth + 1 425 | report.InstanceID = jsonReport.Install 426 | report.Year = ts.Year() 427 | report.Month = int(ts.Month()) 428 | 429 | // If we already have a report for this install at this time, skip it. 430 | if prevReport.ReportTime == ts || ts.Before(prevReport.ReportTime) { 431 | cache.skippedForTime++ 432 | 433 | if prevReport.CountForMonth == 1 { 434 | q := PSQL(db).Update(InstanceReportsTable). 435 | Where(sq.Eq{"id": prevReport.ID}). 436 | Set("count_for_month", report.CountForMonth) 437 | 438 | _, err = q.Exec() 439 | if err != nil { 440 | return err 441 | } 442 | 443 | } 444 | return nil 445 | } 446 | 447 | newReportsStart := time.Now() 448 | 449 | nodes := NodesForReport{} 450 | for _, jsonNode := range jsonReport.Nodes { 451 | if jsonNode.IsController { 452 | jvmVersionID, err := GetJVMVersionID(db, cache, jsonNode.JVMVersion) 453 | if err != nil { 454 | return err 455 | } 456 | report.JVMVersionID = jvmVersionID 457 | } 458 | // At least one report somehow screwed up and claims to have 32-bit max executors, so ignore that. 459 | if jsonNode.Executors != 2147483647 { 460 | report.Executors += jsonNode.Executors 461 | } 462 | 463 | osTypeID, err := GetOSTypeID(db, cache, jsonNode.OS) 464 | if err != nil { 465 | return err 466 | } 467 | if _, ok := nodes[osTypeID]; !ok { 468 | nodes[osTypeID] = 0 469 | } 470 | nodes[osTypeID]++ 471 | } 472 | report.Nodes = &nodes 473 | 474 | if report.JVMVersionID == 0 { 475 | jvmVersionID, err := GetJVMVersionID(db, cache, "N/A") 476 | if err != nil { 477 | return err 478 | } 479 | report.JVMVersionID = jvmVersionID 480 | } 481 | 482 | var pluginIDs pq.Int64Array 483 | for _, jsonPlugin := range jsonReport.Plugins { 484 | // Exclude weird cases where there's no real version for the plugin 485 | if jsonPlugin.Version != questionVersion { 486 | pluginID, err := GetPluginID(db, cache, jsonPlugin.Name, jsonPlugin.Version) 487 | if err != nil { 488 | return err 489 | } 490 | pluginIDs = append(pluginIDs, int64(pluginID)) 491 | } 492 | } 493 | report.Plugins = pluginIDs 494 | 495 | jobs := JobsForReport{} 496 | jobCount := uint64(0) 497 | for jobType, count := range jsonReport.Jobs { 498 | if count != 0 && !strings.HasPrefix(jobType, "private") { 499 | jobTypeID, err := GetJobTypeID(db, cache, jobType) 500 | if err != nil { 501 | return err 502 | } 503 | jobs[jobTypeID] = count 504 | jobCount += count 505 | } 506 | } 507 | if jobCount == 0 { 508 | cache.skippedForJobs++ 509 | return nil 510 | } 511 | report.Jobs = &jobs 512 | cache.insertNewReportsTime += time.Since(newReportsStart) 513 | 514 | report.ReportTime = ts 515 | 516 | jvID, err := GetJenkinsVersionID(db, cache, jsonReport.Version) 517 | if err != nil { 518 | return err 519 | } 520 | report.Version = jvID 521 | 522 | if insertRow { 523 | insertStart := time.Now() 524 | _, err = PSQL(db).Insert(InstanceReportsTable). 525 | Columns("instance_id", "report_time", "year", "month", "version", "jvm_version_id", 526 | "executors", "count_for_month", "plugins", "jobs", "nodes"). 527 | Values(report.InstanceID, 528 | report.ReportTime, 529 | report.Year, 530 | report.Month, 531 | report.Version, 532 | report.JVMVersionID, 533 | report.Executors, 534 | report.CountForMonth, 535 | report.Plugins, 536 | report.Jobs, 537 | report.Nodes). 538 | Exec() 539 | if err != nil { 540 | return err 541 | } 542 | cache.insertInstanceReportTime += time.Since(insertStart) 543 | } else { 544 | updateStart := time.Now() 545 | q := PSQL(db).Update(InstanceReportsTable). 546 | Where(sq.Eq{"id": prevReport.ID}). 547 | Set("count_for_month", report.CountForMonth). 548 | Set("report_time", report.ReportTime). 549 | Set("version", report.Version). 550 | Set("jvm_version_id", report.JVMVersionID). 551 | Set("executors", report.Executors). 552 | Set("plugins", report.Plugins). 553 | Set("jobs", report.Jobs). 554 | Set("nodes", report.Nodes) 555 | 556 | _, err = q.Exec() 557 | cache.updateInstanceReportTime += time.Since(updateStart) 558 | if err != nil { 559 | return err 560 | } 561 | } 562 | 563 | return nil 564 | } 565 | 566 | // ReportAlreadyRead checks if a filename has already been read and processed 567 | func ReportAlreadyRead(db sq.BaseRunner, filename string) (bool, error) { 568 | rows, err := PSQL(db).Select("count(*)"). 569 | From("report_files"). 570 | Where(sq.Eq{"filename": filename}). 571 | Query() 572 | defer func() { 573 | if err != nil { 574 | panic(err) 575 | } 576 | _ = rows.Close() 577 | }() 578 | if err != nil { 579 | if errors.Is(err, sql.ErrNoRows) { 580 | return false, nil 581 | } 582 | return false, err 583 | } 584 | 585 | for rows.Next() { 586 | var c int 587 | err := rows.Scan(&c) 588 | if err != nil { 589 | return false, err 590 | } 591 | if c > 0 { 592 | return true, nil 593 | } 594 | } 595 | return false, nil 596 | } 597 | 598 | // MarkReportRead records that we've read and processed a filename. 599 | func MarkReportRead(db sq.BaseRunner, filename string) error { 600 | _, err := PSQL(db).Insert("report_files").Columns("filename").Values(filename).Exec() 601 | return err 602 | } 603 | 604 | // PSQL is a postgresql squirrel statement builder 605 | func PSQL(db sq.BaseRunner) sq.StatementBuilderType { 606 | return sq.StatementBuilder.PlaceholderFormat(sq.Dollar).RunWith(db) 607 | } 608 | 609 | func startDateForYearMonth(year int, month int) time.Time { 610 | return time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.FixedZone("", 0)) 611 | } 612 | -------------------------------------------------------------------------------- /report.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "encoding/json" 7 | "fmt" 8 | "html/template" 9 | "io/ioutil" 10 | "math" 11 | "os" 12 | "path/filepath" 13 | "sort" 14 | "strconv" 15 | "strings" 16 | "time" 17 | 18 | "gitlab.com/c0b/go-ordered-json" 19 | 20 | "github.com/Masterminds/semver" 21 | sq "github.com/Masterminds/squirrel" 22 | "github.com/beevik/etree" 23 | ) 24 | 25 | var ( 26 | // PieColors is the ordered list of strings to be used for coloring pie wedges. 27 | PieColors = []string{ 28 | "BurlyWood", 29 | "CadetBlue", 30 | "red", 31 | "blue", 32 | "yellow", 33 | "green", 34 | "gold", 35 | "brown", 36 | "Azure", 37 | "pink", 38 | "khaki", 39 | "gray", 40 | "Aqua", 41 | "Aquamarine", 42 | "beige", 43 | "blueviolet", 44 | "Bisque", 45 | "coral", 46 | "darkblue", 47 | "crimson", 48 | "cyan", 49 | "darkred", 50 | "ivory", 51 | "lime", 52 | "maroon", 53 | "navy", 54 | "olive", 55 | "plum", 56 | "peru", 57 | "silver", 58 | "tan", 59 | "teal", 60 | "violet", 61 | ///// 62 | "AliceBlue", 63 | "DarkOliveGreen", 64 | "Indigo", 65 | "MediumPurple", 66 | "Purple", 67 | "AntiqueWhite", 68 | "DarkOrange", 69 | "Ivory", 70 | "MediumSeaGreen", 71 | "Red", 72 | "Aqua", 73 | "DarkOrchid", 74 | "Khaki", 75 | "MediumSlateBlue", 76 | "RosyBrown", 77 | "AquaMarine", 78 | "DarkRed", 79 | "Lavender", 80 | "MediumSpringGreen", 81 | "RoyalBlue", 82 | "Azure", 83 | "DarkSalmon", 84 | "LavenderBlush", 85 | "MediumTurquoise", 86 | "SaddleBrown", 87 | "Beige", 88 | "DarkSeaGreen", 89 | "LawnGreen", 90 | "MediumVioletRed", 91 | "Salmon", 92 | "Bisque", 93 | "DarkSlateBlue", 94 | "LemonChiffon", 95 | "MidnightBlue", 96 | "SandyBrown", 97 | "Black", 98 | "DarkSlateGray", 99 | "LightBlue", 100 | "MintCream", 101 | "SeaGreen", 102 | "BlanchedAlmond", 103 | "DarkTurquoise", 104 | "LightCoral", 105 | "MistyRose", 106 | "SeaShell", 107 | "Blue", 108 | "DarkViolet", 109 | "LightCyan", 110 | "Moccasin", 111 | "Sienna", 112 | "BlueViolet", 113 | "DeepPink", 114 | "LightGoldenrodYellow", 115 | "NavajoWhite", 116 | "Silver", 117 | "Brown", 118 | "DeepSkyBlue", 119 | "LightGray", 120 | "Navy", 121 | "SkyBlue", 122 | "BurlyWood", 123 | "DimGray", 124 | "LightGreen", 125 | "OldLace", 126 | "SlateBlue", 127 | "CadetBlue", 128 | "DodgerBlue", 129 | "LightPink", 130 | "Olive", 131 | "SlateGray", 132 | "Chartreuse", 133 | "FireBrick", 134 | "LightSalmon", 135 | "OliveDrab", 136 | "Snow", 137 | "Chocolate", 138 | "FloralWhite", 139 | "LightSeaGreen", 140 | "Orange", 141 | "SpringGreen", 142 | "Coral", 143 | "ForestGreen", 144 | "LightSkyBlue", 145 | "OrangeRed", 146 | "SteelBlue", 147 | "CornFlowerBlue", 148 | "Fuchsia", 149 | "LightSlateGray", 150 | "Orchid", 151 | "Tan", 152 | "Cornsilk", 153 | "Gainsboro", 154 | "LightSteelBlue", 155 | "PaleGoldenRod", 156 | "Teal", 157 | "Crimson", 158 | "GhostWhite", 159 | "LightYellow", 160 | "PaleGreen", 161 | "Thistle", 162 | "Cyan", 163 | "Gold", 164 | "Lime", 165 | "PaleTurquoise", 166 | "Tomato", 167 | "DarkBlue", 168 | "GoldenRod", 169 | "LimeGreen", 170 | "PaleVioletRed", 171 | "Turquoise", 172 | "DarkCyan", 173 | "Gray", 174 | "Linen", 175 | "PapayaWhip", 176 | "Violet", 177 | "DarkGoldenRod", 178 | "Green", 179 | "Magenta", 180 | "PeachPuff", 181 | "Wheat", 182 | "DarkGray", 183 | "GreenYellow", 184 | "Maroon", 185 | "Peru", 186 | "White", 187 | "DarkGreen", 188 | "HoneyDew", 189 | "MediumAquaMarine", 190 | "Pink", 191 | "WhiteSmoke", 192 | "DarkKhaki", 193 | "HotPink", 194 | "MediumBlue", 195 | "Plum", 196 | "Yellow", 197 | "DarkMagenta", 198 | "IndianRed", 199 | "MediumOrchid", 200 | "PowderBlue", 201 | "YellowGreen", 202 | } 203 | 204 | //go:embed templates/versionDistroPlugin.html.tmpl 205 | // VersionDistributionTemplate is the Golang template used for generating plugin version distribution HTML. 206 | VersionDistributionTemplate string 207 | 208 | //go:embed templates/versionDistroIndex.html.tmpl 209 | // VersionDistributionIndexTemplate is the Golang template used for generating the plugin version distribution index. 210 | VersionDistributionIndexTemplate string 211 | 212 | //go:embed templates/svgs.html.tmpl 213 | // SVGsIndexTemplate is the Golang template used for generating the svg directory HTML index. 214 | SVGsIndexTemplate string 215 | 216 | //go:embed templates/pitIndex.html.tmpl 217 | // PITIndexTemplate is the Golang template used for generating the plugin-installation-trend HTML index. 218 | PITIndexTemplate string 219 | ) 220 | 221 | // PluginReport is written out as JSON for reports for each plugin 222 | type PluginReport struct { 223 | Name string `json:"name"` 224 | Installations map[string]uint64 `json:"installations"` 225 | MonthPercentages map[string]float32 `json:"installationsPercentage"` 226 | PerVersion map[string]uint64 `json:"installationsPerVersion"` 227 | VersionPercentages map[string]float32 `json:"installationsPercentagePerVersion"` 228 | } 229 | 230 | // JVMReport is marshalled to create jvms.json 231 | type JVMReport struct { 232 | PerMonth map[string]map[string]uint64 `json:"jvmStatsPerMonth"` 233 | PerMonth2x map[string]map[string]uint64 `json:"jvmStatsPerMonth_2.x"` 234 | } 235 | 236 | // InstallationReport is written out to generate installations.{json,csv} 237 | type InstallationReport struct { 238 | Installations map[string]uint64 `json:"installations"` 239 | } 240 | 241 | // ToCSV returns a CSV representation of the InstallationReport 242 | func (i InstallationReport) ToCSV() (string, error) { 243 | var keys []string 244 | 245 | for k := range i.Installations { 246 | keys = append(keys, k) 247 | } 248 | sort.Strings(keys) 249 | 250 | var builder strings.Builder 251 | 252 | for _, k := range keys { 253 | _, err := builder.Write([]byte(fmt.Sprintf(`"%s","%d"`+"\n", k, i.Installations[k]))) 254 | if err != nil { 255 | return "", err 256 | } 257 | } 258 | 259 | return builder.String(), nil 260 | } 261 | 262 | // LatestPluginNumbersReport is written out to generate latestNumbers.{json,csv} 263 | type LatestPluginNumbersReport struct { 264 | Month int64 `json:"month"` 265 | Plugins map[string]uint64 `json:"plugins"` 266 | } 267 | 268 | // ToCSV returns a CSV representation of the LatestPluginNumbersReport 269 | func (l LatestPluginNumbersReport) ToCSV() (string, error) { 270 | var keys []string 271 | 272 | for k := range l.Plugins { 273 | keys = append(keys, k) 274 | } 275 | sort.Strings(keys) 276 | 277 | var builder strings.Builder 278 | 279 | for _, k := range keys { 280 | _, err := builder.Write([]byte(fmt.Sprintf(`"%s","%d"`+"\n", k, l.Plugins[k]))) 281 | if err != nil { 282 | return "", err 283 | } 284 | } 285 | 286 | return builder.String(), nil 287 | } 288 | 289 | // CapabilitiesReport is written out to generate capabilities.{json,csv} 290 | type CapabilitiesReport struct { 291 | Installations map[string]uint64 `json:"installations"` 292 | } 293 | 294 | // ToCSV returns a CSV representation of the CapabilitiesReport 295 | func (i CapabilitiesReport) ToCSV() (string, error) { 296 | var keys []string 297 | 298 | for k := range i.Installations { 299 | keys = append(keys, k) 300 | } 301 | sort.Strings(keys) 302 | 303 | var builder strings.Builder 304 | 305 | for _, k := range keys { 306 | _, err := builder.Write([]byte(fmt.Sprintf(`"%s","%d"`+"\n", k, i.Installations[k]))) 307 | if err != nil { 308 | return "", err 309 | } 310 | } 311 | 312 | return builder.String(), nil 313 | } 314 | 315 | type yearMonth struct { 316 | year int 317 | month int 318 | } 319 | 320 | type monthForHTML struct { 321 | Year int 322 | Num string 323 | Name string 324 | AsStr string 325 | } 326 | 327 | // GenerateReport creates the JSON, CSV, SVG, and HTML files for a monthly report 328 | func GenerateReport(db sq.BaseRunner, specifiedYear, specifiedMonth int, baseDir string) error { 329 | err := os.MkdirAll(baseDir, 0755) //nolint:gosec 330 | if err != nil { 331 | return err 332 | } 333 | 334 | pitDir := filepath.Join(baseDir, "plugin-installation-trend") 335 | err = os.MkdirAll(pitDir, 0755) //nolint:gosec 336 | if err != nil { 337 | return err 338 | } 339 | 340 | svgDir := filepath.Join(baseDir, "jenkins-stats/svg") 341 | err = os.MkdirAll(svgDir, 0755) //nolint:gosec 342 | if err != nil { 343 | return err 344 | } 345 | 346 | pvDir := filepath.Join(baseDir, "pluginversions") 347 | err = os.MkdirAll(pvDir, 0755) //nolint:gosec 348 | if err != nil { 349 | return err 350 | } 351 | 352 | var latestMonthToReport time.Time 353 | 354 | // If we're given specific year/month, generate through that month. Otherwise, generate through the previous month 355 | // from the time we're running. 356 | if specifiedYear > 0 && specifiedMonth > 0 { 357 | latestMonthToReport = startDateForYearMonth(specifiedYear, specifiedMonth) 358 | } else { 359 | now := time.Now() 360 | latestMonthToReport = startDateForYearMonth(now.Year(), int(now.Month())).AddDate(0, -1, 0) 361 | } 362 | 363 | reportYear := latestMonthToReport.Year() 364 | reportMonth := int(latestMonthToReport.Month()) 365 | 366 | icStart := time.Now() 367 | installCount, err := GetInstallCountForVersions(db, reportYear, reportMonth) 368 | if err != nil { 369 | return err 370 | } 371 | icAsJSON, err := json.MarshalIndent(installCount, "", " ") 372 | if err != nil { 373 | return err 374 | } 375 | 376 | err = writeFile(filepath.Join(pitDir, "installations.json"), icAsJSON) 377 | if err != nil { 378 | return err 379 | } 380 | icAsCSV, err := installCount.ToCSV() 381 | if err != nil { 382 | return err 383 | } 384 | err = writeFile(filepath.Join(pitDir, "installations.csv"), []byte(icAsCSV)) 385 | if err != nil { 386 | return err 387 | } 388 | fmt.Printf("installCount time: %s\n", time.Since(icStart)) 389 | 390 | vdStart := time.Now() 391 | jvpv, err := GenerateVersionDistributions(db, reportYear, reportMonth, pvDir) 392 | if err != nil { 393 | return err 394 | } 395 | fmt.Printf("versionDistribution time: %s\n", time.Since(vdStart)) 396 | 397 | prStart := time.Now() 398 | // GetPluginReports expects to get the _current_ year/month so it can exclude that from its reports. 399 | pluginReports, err := GetPluginReports(db, specifiedYear, specifiedMonth) 400 | if err != nil { 401 | return err 402 | } 403 | for _, pr := range pluginReports { 404 | prAsJSON, err := json.MarshalIndent(pr, "", " ") 405 | if err != nil { 406 | return err 407 | } 408 | err = writeFile(filepath.Join(pitDir, fmt.Sprintf("%s.stats.json", pr.Name)), prAsJSON) 409 | if err != nil { 410 | return err 411 | } 412 | } 413 | fmt.Printf("pluginReport time: %s\n", time.Since(prStart)) 414 | 415 | lnStart := time.Now() 416 | latestNumbers, err := GetLatestPluginNumbers(db, reportYear, reportMonth) 417 | if err != nil { 418 | return err 419 | } 420 | lnAsJSON, err := json.MarshalIndent(latestNumbers, "", " ") 421 | if err != nil { 422 | return err 423 | } 424 | lnAsCSV, err := latestNumbers.ToCSV() 425 | if err != nil { 426 | return err 427 | } 428 | err = writeFile(filepath.Join(pitDir, "latestNumbers.json"), lnAsJSON) 429 | if err != nil { 430 | return err 431 | } 432 | err = writeFile(filepath.Join(pitDir, "latestNumbers.csv"), []byte(lnAsCSV)) 433 | if err != nil { 434 | return err 435 | } 436 | fmt.Printf("latestNumbers time: %s\n", time.Since(lnStart)) 437 | 438 | capStart := time.Now() 439 | capabilities, err := GetCapabilities(db, reportYear, reportMonth) 440 | if err != nil { 441 | return err 442 | } 443 | capAsJSON, err := json.MarshalIndent(capabilities, "", " ") 444 | if err != nil { 445 | return err 446 | } 447 | capAsCSV, err := capabilities.ToCSV() 448 | if err != nil { 449 | return err 450 | } 451 | err = writeFile(filepath.Join(pitDir, "capabilities.json"), capAsJSON) 452 | if err != nil { 453 | return err 454 | } 455 | err = writeFile(filepath.Join(pitDir, "capabilities.csv"), []byte(capAsCSV)) 456 | if err != nil { 457 | return err 458 | } 459 | fmt.Printf("capabilities time: %s\n", time.Since(capStart)) 460 | 461 | jvmStart := time.Now() 462 | // GetJVMsReport expects to get the _current_ year/month so that month can be excluded. 463 | jvms, err := GetJVMsReport(db, specifiedYear, specifiedMonth) 464 | if err != nil { 465 | return err 466 | } 467 | jvmsAsJSON, err := json.MarshalIndent(jvms, "", " ") 468 | if err != nil { 469 | return err 470 | } 471 | err = writeFile(filepath.Join(pitDir, "jvms.json"), jvmsAsJSON) 472 | if err != nil { 473 | return err 474 | } 475 | fmt.Printf("jvms time: %s\n", time.Since(jvmStart)) 476 | 477 | allMonths, err := allOrderedMonths(db, specifiedYear, specifiedMonth) 478 | if err != nil { 479 | return err 480 | } 481 | 482 | var monthsForHTML []monthForHTML 483 | 484 | installCountByMonth := make(map[string]uint64) 485 | jobCountByMonth := make(map[string]uint64) 486 | nodeCountByMonth := make(map[string]uint64) 487 | pluginCountByMonth := make(map[string]uint64) 488 | 489 | svgStart := time.Now() 490 | for _, ym := range allMonths { 491 | monthStr := fmt.Sprintf("%d%02d", ym.year, ym.month) 492 | monthsForHTML = append(monthsForHTML, monthForHTML{ 493 | Year: ym.year, 494 | Num: fmt.Sprintf("%02d", ym.month), 495 | Name: time.Month(ym.month).String(), 496 | AsStr: monthStr, 497 | }) 498 | 499 | installCountByMonth[monthStr] = 0 500 | jobCountByMonth[monthStr] = 0 501 | nodeCountByMonth[monthStr] = 0 502 | pluginCountByMonth[monthStr] = 0 503 | 504 | ir, err := GetInstallCountForVersions(db, ym.year, ym.month) 505 | if err != nil { 506 | return err 507 | } 508 | 509 | for _, c := range ir.Installations { 510 | installCountByMonth[monthStr] += c 511 | } 512 | 513 | irSVG, irCSV, err := CreateBarSVG(fmt.Sprintf("Jenkins installations (total: %d)", installCountByMonth[monthStr]), ir.Installations, 10, false, true, false, DefaultFilter) 514 | if err != nil { 515 | return err 516 | } 517 | 518 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-jenkins.svg", monthStr)), irSVG); err != nil { 519 | return err 520 | } 521 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-jenkins.csv", monthStr)), irCSV); err != nil { 522 | return err 523 | } 524 | 525 | pr, err := GetLatestPluginNumbers(db, ym.year, ym.month) 526 | if err != nil { 527 | return err 528 | } 529 | for _, c := range pr.Plugins { 530 | pluginCountByMonth[monthStr] += c 531 | } 532 | 533 | prSVG, prCSV, err := CreateBarSVG(fmt.Sprintf("Plugin installations (total: %d)", pluginCountByMonth[monthStr]), pr.Plugins, 100, true, false, false, DefaultFilter) 534 | if err != nil { 535 | return err 536 | } 537 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-plugins.svg", monthStr)), prSVG); err != nil { 538 | return err 539 | } 540 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-plugins.csv", monthStr)), prCSV); err != nil { 541 | return err 542 | } 543 | 544 | for _, topNum := range []uint64{500, 1000, 2500} { 545 | topPRSVG, topPRCSV, err := CreateBarSVG(fmt.Sprintf("Plugin installations (installations > %d)", topNum), pr.Plugins, 100, true, false, false, func(s string, u uint64) bool { 546 | return u > topNum 547 | }) 548 | if err != nil { 549 | return err 550 | } 551 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-top-plugins%d.svg", monthStr, topNum)), topPRSVG); err != nil { 552 | return err 553 | } 554 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-top-plugins%d.csv", monthStr, topNum)), topPRCSV); err != nil { 555 | return err 556 | } 557 | } 558 | 559 | osR, err := OSCountsForMonth(db, ym.year, ym.month) 560 | if err != nil { 561 | return err 562 | } 563 | 564 | var osNames []string 565 | var osNumbers []uint64 566 | 567 | for n := range osR { 568 | osNames = append(osNames, n) 569 | } 570 | 571 | sort.Strings(osNames) 572 | 573 | for _, n := range osNames { 574 | nodeCountByMonth[monthStr] += osR[n] 575 | osNumbers = append(osNumbers, osR[n]) 576 | } 577 | 578 | osBarSVG, osBarCSV, err := CreateBarSVG(fmt.Sprintf("Nodes (total: %d)", nodeCountByMonth[monthStr]), osR, 10, true, false, false, DefaultFilter) 579 | if err != nil { 580 | return err 581 | } 582 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-nodes.svg", monthStr)), osBarSVG); err != nil { 583 | return err 584 | } 585 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-nodes.csv", monthStr)), osBarCSV); err != nil { 586 | return err 587 | } 588 | 589 | osPieSVG, osPieCSV, err := CreatePieSVG("Nodes", osNumbers, 200, 300, 150, 370, 20, osNames, PieColors) 590 | if err != nil { 591 | return err 592 | } 593 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-nodesPie.svg", monthStr)), osPieSVG); err != nil { 594 | return err 595 | } 596 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-nodesPie.csv", monthStr)), osPieCSV); err != nil { 597 | return err 598 | } 599 | 600 | jr, err := JobCountsForMonth(db, ym.year, ym.month) 601 | if err != nil { 602 | return err 603 | } 604 | 605 | for _, c := range jr { 606 | jobCountByMonth[monthStr] += c 607 | } 608 | 609 | jobsSVG, jobsCSV, err := CreateBarSVG(fmt.Sprintf("Jobs (total: %d)", jobCountByMonth[monthStr]), jr, 1000, true, false, false, DefaultFilter) 610 | if err != nil { 611 | return err 612 | } 613 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-jobs.svg", monthStr)), jobsSVG); err != nil { 614 | return err 615 | } 616 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-jobs.csv", monthStr)), jobsCSV); err != nil { 617 | return err 618 | } 619 | 620 | execR, err := ExecutorCountsForMonth(db, ym.year, ym.month) 621 | if err != nil { 622 | return err 623 | } 624 | 625 | totalExecs := uint64(0) 626 | for _, c := range execR { 627 | totalExecs += c 628 | } 629 | 630 | execSVG, execCSV, err := CreateBarSVG(fmt.Sprintf("Executors per install (total: %d)", totalExecs), execR, 25, false, false, true, DefaultFilter) 631 | if err != nil { 632 | return err 633 | } 634 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-total-executors.svg", monthStr)), execSVG); err != nil { 635 | return err 636 | } 637 | if err := writeFile(filepath.Join(svgDir, fmt.Sprintf("%s-total-executors.csv", monthStr)), execCSV); err != nil { 638 | return err 639 | } 640 | } 641 | 642 | totalJenkinsSVG, totalJenkinsCSV, err := CreateBarSVG("Total Jenkins installations", installCountByMonth, 100, false, false, false, DefaultFilter) 643 | if err != nil { 644 | return err 645 | } 646 | if err := writeFile(filepath.Join(svgDir, "total-jenkins.svg"), totalJenkinsSVG); err != nil { 647 | return err 648 | } 649 | if err := writeFile(filepath.Join(svgDir, "total-jenkins.csv"), totalJenkinsCSV); err != nil { 650 | return err 651 | } 652 | 653 | totalJobsSVG, totalJobsCSV, err := CreateBarSVG("Total jobs", jobCountByMonth, 1000, false, false, false, DefaultFilter) 654 | if err != nil { 655 | return err 656 | } 657 | if err := writeFile(filepath.Join(svgDir, "total-jobs.svg"), totalJobsSVG); err != nil { 658 | return err 659 | } 660 | if err := writeFile(filepath.Join(svgDir, "total-jobs.csv"), totalJobsCSV); err != nil { 661 | return err 662 | } 663 | 664 | totalNodesSVG, totalNodesCSV, err := CreateBarSVG("Total nodes", nodeCountByMonth, 100, false, false, false, DefaultFilter) 665 | if err != nil { 666 | return err 667 | } 668 | if err := writeFile(filepath.Join(svgDir, "total-nodes.svg"), totalNodesSVG); err != nil { 669 | return err 670 | } 671 | if err := writeFile(filepath.Join(svgDir, "total-nodes.csv"), totalNodesCSV); err != nil { 672 | return err 673 | } 674 | 675 | totalPluginsSVG, totalPluginsCSV, err := CreateBarSVG("Total Plugin installations", pluginCountByMonth, 1000, false, false, false, DefaultFilter) 676 | if err != nil { 677 | return err 678 | } 679 | if err := writeFile(filepath.Join(svgDir, "total-plugins.svg"), totalPluginsSVG); err != nil { 680 | return err 681 | } 682 | if err := writeFile(filepath.Join(svgDir, "total-plugins.csv"), totalPluginsCSV); err != nil { 683 | return err 684 | } 685 | 686 | totalFiles := []string{"total-plugins", "total-jobs", "total-jenkins", "total-nodes"} 687 | 688 | idxTmpl, err := template.New("svgs-index").Parse(SVGsIndexTemplate) 689 | if err != nil { 690 | return err 691 | } 692 | 693 | idxFile, err := os.OpenFile(filepath.Join(svgDir, "svgs.html"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec 694 | if err != nil { 695 | return err 696 | } 697 | defer func() { 698 | _ = idxFile.Close() 699 | }() 700 | 701 | err = idxTmpl.Execute(idxFile, map[string]interface{}{ 702 | "totalFiles": totalFiles, 703 | "months": monthsForHTML, 704 | }) 705 | if err != nil { 706 | return err 707 | } 708 | fmt.Printf("svgs time: %s\n", time.Since(svgStart)) 709 | 710 | jvpvJSON, err := json.MarshalIndent(jvpv, "", " ") 711 | if err != nil { 712 | return err 713 | } 714 | if err := writeFile(filepath.Join(pitDir, "jenkins-version-per-plugin-version.json"), jvpvJSON); err != nil { 715 | return err 716 | } 717 | 718 | var pluginNames []string 719 | for pn := range latestNumbers.Plugins { 720 | pluginNames = append(pluginNames, pn) 721 | } 722 | sort.Strings(pluginNames) 723 | 724 | pitTmpl, err := template.New("pit-index").Parse(PITIndexTemplate) 725 | if err != nil { 726 | return err 727 | } 728 | 729 | pitFile, err := os.OpenFile(filepath.Join(pitDir, "index.html"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec 730 | if err != nil { 731 | return err 732 | } 733 | defer func() { 734 | _ = pitFile.Close() 735 | }() 736 | 737 | return pitTmpl.Execute(pitFile, map[string]interface{}{ 738 | "jsonFiles": []string{"installations", "latestNumbers", "capabilities", "jenkins-version-per-plugin-version", "jvms"}, 739 | "pluginNames": pluginNames, 740 | }) 741 | } 742 | 743 | // GetInstallCountForVersions generates a map of Jenkins versions to install counts 744 | // analogous to Groovy version's generateInstallationsJson 745 | func GetInstallCountForVersions(db sq.BaseRunner, year, month int) (InstallationReport, error) { 746 | report := InstallationReport{Installations: map[string]uint64{}} 747 | rows, err := PSQL(db).Select("jv.version as jvv", "count(*) as number"). 748 | From("instance_reports i"). 749 | Join("jenkins_versions jv on i.version = jv.id"). 750 | Where(sq.Eq{"i.year": year}). 751 | Where(sq.Eq{"i.month": month}). 752 | Where(sq.GtOrEq{"i.count_for_month": 2}). 753 | Where("jv.version ~ '^\\d'"). 754 | Where("jv.version not like '%private%'"). 755 | GroupBy("jvv"). 756 | OrderBy("jvv"). 757 | Query() 758 | if err != nil { 759 | return report, err 760 | } 761 | defer func() { 762 | _ = rows.Close() 763 | }() 764 | 765 | for rows.Next() { 766 | var v string 767 | var c uint64 768 | err := rows.Scan(&v, &c) 769 | if err != nil { 770 | return report, err 771 | } 772 | report.Installations[v] = c 773 | } 774 | 775 | return report, nil 776 | } 777 | 778 | // GetLatestPluginNumbers generates a map of plugin name and install counts 779 | // analogous to Groovy version's generateLatestNumbersJson 780 | func GetLatestPluginNumbers(db sq.BaseRunner, year, month int) (LatestPluginNumbersReport, error) { 781 | report := LatestPluginNumbersReport{ 782 | Month: startDateForYearMonth(year, month).UnixMilli(), 783 | Plugins: map[string]uint64{}, 784 | } 785 | rows, err := PSQL(db).Select("p.name as pn", "count(*) as number"). 786 | From("instance_reports i, unnest(i.plugins) pr(id)"). 787 | Join("plugins p on p.id = pr.id"). 788 | Where(sq.Eq{"i.year": year}). 789 | Where(sq.Eq{"i.month": month}). 790 | Where(sq.GtOrEq{"i.count_for_month": 2}). 791 | GroupBy("pn"). 792 | Query() 793 | if err != nil { 794 | return report, err 795 | } 796 | defer func() { 797 | _ = rows.Close() 798 | }() 799 | 800 | for rows.Next() { 801 | var p string 802 | var c uint64 803 | err := rows.Scan(&p, &c) 804 | if err != nil { 805 | return report, err 806 | } 807 | report.Plugins[p] = c 808 | } 809 | 810 | return report, nil 811 | } 812 | 813 | // GetCapabilities generates a map of Jenkins versions and install counts for that version and all earlier ones 814 | // analogous to Groovy version's generateCapabilitiesJson 815 | func GetCapabilities(db sq.BaseRunner, year, month int) (CapabilitiesReport, error) { 816 | report := CapabilitiesReport{Installations: map[string]uint64{}} 817 | rows, err := PSQL(db).Select("jv.version as jvv", "count(*) as number"). 818 | From("instance_reports i"). 819 | Join("jenkins_versions jv on i.version = jv.id"). 820 | Where(sq.Eq{"i.year": year}). 821 | Where(sq.Eq{"i.month": month}). 822 | Where(sq.GtOrEq{"i.count_for_month": 2}). 823 | Where("jv.version ~ '^\\d'"). 824 | Where("jv.version not like '%private%'"). 825 | GroupBy("jvv"). 826 | OrderBy("jvv DESC"). 827 | Query() 828 | if err != nil { 829 | return report, err 830 | } 831 | defer func() { 832 | _ = rows.Close() 833 | }() 834 | 835 | higherCapabilityCount := uint64(0) 836 | for rows.Next() { 837 | var p string 838 | var c uint64 839 | err := rows.Scan(&p, &c) 840 | if err != nil { 841 | return report, err 842 | } 843 | higherCapabilityCount += c 844 | report.Installations[p] = higherCapabilityCount 845 | } 846 | 847 | return report, nil 848 | } 849 | 850 | // GetJVMsReport returns the JVM install counts for all months 851 | // analogous to Groovy version's generateJvmJson 852 | func GetJVMsReport(db sq.BaseRunner, year, month int) (JVMReport, error) { 853 | jvr := JVMReport{ 854 | PerMonth: map[string]map[string]uint64{}, 855 | PerMonth2x: map[string]map[string]uint64{}, 856 | } 857 | 858 | months, err := allOrderedMonths(db, year, month) 859 | if err != nil { 860 | return jvr, err 861 | } 862 | jvmIDs, err := jvmIDsForJSON(db) 863 | if err != nil { 864 | return jvr, err 865 | } 866 | jenkinsIDs, err := jenkinsVersions2x(db) 867 | if err != nil { 868 | return jvr, err 869 | } 870 | 871 | baseStmt := PSQL(db).Select("jv.name as n", "count(*)"). 872 | From("instance_reports i"). 873 | Join("jvm_versions jv on jv.id = i.jvm_version_id"). 874 | Where(sq.Eq{"jv.id": jvmIDs}). 875 | Where(sq.GtOrEq{"i.count_for_month": 2}). 876 | GroupBy("n"). 877 | OrderBy("n") 878 | 879 | for _, ym := range months { 880 | err = func() error { 881 | ts := startDateForYearMonth(ym.year, ym.month) 882 | tsStr := fmt.Sprintf("%d", ts.UnixMilli()) 883 | 884 | monthStmt := baseStmt.Where(sq.Eq{"i.year": ym.year}).Where(sq.Eq{"i.month": ym.month}) 885 | rows, err := monthStmt.Query() 886 | if err != nil { 887 | return err 888 | } 889 | defer func() { 890 | _ = rows.Close() 891 | }() 892 | for rows.Next() { 893 | var name string 894 | var count uint64 895 | if _, ok := jvr.PerMonth[tsStr]; !ok { 896 | jvr.PerMonth[tsStr] = map[string]uint64{} 897 | } 898 | err = rows.Scan(&name, &count) 899 | if err != nil { 900 | return err 901 | } 902 | 903 | jvr.PerMonth[tsStr][name] = count 904 | } 905 | 906 | rows2x, err := monthStmt.Where(sq.Eq{"i.version": jenkinsIDs}).Query() 907 | if err != nil { 908 | return err 909 | } 910 | defer func() { 911 | _ = rows2x.Close() 912 | }() 913 | for rows2x.Next() { 914 | var name string 915 | var count uint64 916 | if _, ok := jvr.PerMonth2x[tsStr]; !ok { 917 | jvr.PerMonth2x[tsStr] = map[string]uint64{} 918 | } 919 | err = rows2x.Scan(&name, &count) 920 | if err != nil { 921 | return err 922 | } 923 | 924 | jvr.PerMonth2x[tsStr][name] = count 925 | } 926 | 927 | return nil 928 | }() 929 | if err != nil { 930 | return jvr, err 931 | } 932 | } 933 | 934 | return jvr, nil 935 | } 936 | 937 | // GetPluginReports generates reports for each plugin 938 | // analogous to Groovy version's generatePluginsJson 939 | func GetPluginReports(db sq.BaseRunner, currentYear, currentMonth int) ([]PluginReport, error) { 940 | previousMonth := startDateForYearMonth(currentYear, currentMonth).AddDate(0, -1, 0) 941 | prevMonthStr := fmt.Sprintf("%d", previousMonth.UnixMilli()) 942 | 943 | var reports []PluginReport 944 | 945 | pluginNames, err := allPluginNames(db) 946 | if err != nil { 947 | return nil, err 948 | } 949 | 950 | idsToName, err := pluginIDsToPlugin(db) 951 | if err != nil { 952 | return nil, err 953 | } 954 | 955 | totalInstalls, err := installCountsByMonth(db, currentYear, currentMonth) 956 | if err != nil { 957 | return nil, err 958 | } 959 | 960 | installsByMonth, err := pluginInstallsByMonthForName(db, currentYear, currentMonth, idsToName) 961 | if err != nil { 962 | return nil, err 963 | } 964 | 965 | installsByVersion, err := pluginInstallsByVersionForName(db, previousMonth.Year(), int(previousMonth.Month()), idsToName) 966 | if err != nil { 967 | return nil, err 968 | } 969 | 970 | for _, pn := range pluginNames { 971 | report := PluginReport{ 972 | Name: pn, 973 | Installations: map[string]uint64{}, 974 | MonthPercentages: map[string]float32{}, 975 | PerVersion: map[string]uint64{}, 976 | VersionPercentages: map[string]float32{}, 977 | } 978 | 979 | for versionStr, versionCount := range installsByVersion[pn] { 980 | report.PerVersion[versionStr] = versionCount 981 | report.VersionPercentages[versionStr] = float32(versionCount) * 100 / float32(totalInstalls[prevMonthStr]) 982 | } 983 | 984 | for monthStr, monthCount := range installsByMonth[pn] { 985 | report.Installations[monthStr] = monthCount 986 | report.MonthPercentages[monthStr] = float32(monthCount) * 100 / float32(totalInstalls[monthStr]) 987 | } 988 | 989 | if len(report.Installations) > 0 { 990 | reports = append(reports, report) 991 | } 992 | } 993 | 994 | return reports, nil 995 | } 996 | 997 | // GenerateVersionDistributions writes out HTML files for each plugin's version distribution 998 | func GenerateVersionDistributions(db sq.BaseRunner, year, month int, outputDir string) (map[string]*PVDPluginVersionMap, error) { 999 | jvpv, err := JenkinsVersionsForPluginVersions(db, year, month) 1000 | if err != nil { 1001 | return nil, err 1002 | } 1003 | 1004 | tmpl, err := template.New("versionDistribution").Parse(VersionDistributionTemplate) 1005 | if err != nil { 1006 | return nil, err 1007 | } 1008 | 1009 | var pluginNames []string 1010 | 1011 | for k, v := range jvpv { 1012 | versionInfo, err := json.Marshal(v) 1013 | if err != nil { 1014 | return nil, err 1015 | } 1016 | outFile, err := os.OpenFile(filepath.Join(outputDir, fmt.Sprintf("%s.html", k)), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec 1017 | if err != nil { 1018 | return nil, err 1019 | } 1020 | err = tmpl.Execute(outFile, map[string]interface{}{ 1021 | "pluginName": k, 1022 | "pluginVersionData": template.JS(versionInfo), //nolint:gosec 1023 | }) 1024 | if err != nil { 1025 | return nil, err 1026 | } 1027 | err = outFile.Close() 1028 | if err != nil { 1029 | return nil, err 1030 | } 1031 | 1032 | pluginNames = append(pluginNames, k) 1033 | } 1034 | 1035 | sort.Strings(pluginNames) 1036 | 1037 | indexTmpl, err := template.New("versionDistributionIndex").Parse(VersionDistributionIndexTemplate) 1038 | if err != nil { 1039 | return nil, err 1040 | } 1041 | 1042 | indexFile, err := os.OpenFile(filepath.Join(outputDir, "index.html"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec 1043 | if err != nil { 1044 | return nil, err 1045 | } 1046 | 1047 | defer func() { 1048 | _ = indexFile.Close() 1049 | }() 1050 | 1051 | return jvpv, indexTmpl.Execute(indexFile, map[string]interface{}{"pluginNames": pluginNames}) 1052 | } 1053 | 1054 | // PVDJenkinsVersionMap is an ordered map 1055 | type PVDJenkinsVersionMap struct { 1056 | *ordered.OrderedMap 1057 | } 1058 | 1059 | // Incr bumps the count for a particular Jenkins version 1060 | func (pvdj *PVDJenkinsVersionMap) Incr(jv string) { 1061 | rawCount := pvdj.Get(jv) 1062 | if rawCount == nil { 1063 | pvdj.Set(jv, uint64(1)) 1064 | return 1065 | } 1066 | count := rawCount.(uint64) 1067 | pvdj.Set(jv, count+1) 1068 | } 1069 | 1070 | // PVDPluginVersionMap is used with PVDJenkinsVersionMap to have an ordered map for plugin version->Jenkins version distribution 1071 | type PVDPluginVersionMap struct { 1072 | *ordered.OrderedMap 1073 | } 1074 | 1075 | // Version returns a pointer to the PVDJenkinsVersionMap for this plugin version 1076 | func (pvdp *PVDPluginVersionMap) Version(pv string) *PVDJenkinsVersionMap { 1077 | rawVal := pvdp.Get(pv) 1078 | if rawVal == nil { 1079 | m := &PVDJenkinsVersionMap{OrderedMap: ordered.NewOrderedMap()} 1080 | pvdp.Set(pv, m) 1081 | return m 1082 | } 1083 | val := rawVal.(*PVDJenkinsVersionMap) 1084 | return val 1085 | } 1086 | 1087 | // JenkinsVersionsForPluginVersions generates a report for each plugin's version, with a count of installs for each Jenkins version 1088 | // analogous to Groovy version's generateOldestJenkinsPerPlugin 1089 | func JenkinsVersionsForPluginVersions(db sq.BaseRunner, year, month int) (map[string]*PVDPluginVersionMap, error) { 1090 | maxVersionsForInstanceIDs, err := maxInstanceVersionForMonth(db, year, month) 1091 | if err != nil { 1092 | return nil, err 1093 | } 1094 | 1095 | rows, err := PSQL(db).Select("p.name as pn", "p.version as pv", "i.instance_id as iid"). 1096 | From("instance_reports i, unnest(i.plugins) pr(id)"). 1097 | Join("plugins p on p.id = pr.id"). 1098 | Where(sq.Eq{"i.year": year}). 1099 | Where(sq.Eq{"i.month": month}). 1100 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1101 | OrderBy("pn", "pv desc", "iid"). 1102 | Query() 1103 | if err != nil { 1104 | return nil, err 1105 | } 1106 | defer func() { 1107 | _ = rows.Close() 1108 | }() 1109 | 1110 | pluginMap := make(map[string]*PVDPluginVersionMap) 1111 | for rows.Next() { 1112 | var pn, pv, iid string 1113 | 1114 | err := rows.Scan(&pn, &pv, &iid) 1115 | if err != nil { 1116 | return nil, err 1117 | } 1118 | 1119 | // If we don't have a "valid" max Jenkins version for this instance, skip. 1120 | if _, ok := maxVersionsForInstanceIDs[iid]; !ok { 1121 | continue 1122 | } 1123 | 1124 | // If the plugin version is "???", skip. 1125 | if pv == questionVersion { 1126 | continue 1127 | } 1128 | 1129 | maxVer := maxVersionsForInstanceIDs[iid] 1130 | 1131 | if _, ok := pluginMap[pn]; !ok { 1132 | pluginMap[pn] = &PVDPluginVersionMap{OrderedMap: ordered.NewOrderedMap()} 1133 | } 1134 | 1135 | pluginMap[pn].Version(pv).Incr(maxVer) 1136 | } 1137 | 1138 | return pluginMap, nil 1139 | } 1140 | 1141 | // JobCountsForMonth gets the total number of each known job type in a month 1142 | // analogous to jobtype2Number in generateStats.groovy 1143 | func JobCountsForMonth(db sq.BaseRunner, year, month int) (map[string]uint64, error) { 1144 | rows, err := PSQL(db).Select("j.name", "sum(jr.value::int) as total"). 1145 | From("instance_reports i, jsonb_each_text(i.jobs) jr"). 1146 | Join("job_types j on j.id = jr.key::int"). 1147 | Where(sq.Eq{"i.year": year}). 1148 | Where(sq.Eq{"i.month": month}). 1149 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1150 | GroupBy("j.name"). 1151 | OrderBy("total asc"). 1152 | Query() 1153 | if err != nil { 1154 | return nil, err 1155 | } 1156 | defer func() { 1157 | _ = rows.Close() 1158 | }() 1159 | 1160 | jobMap := make(map[string]uint64) 1161 | 1162 | for rows.Next() { 1163 | var name string 1164 | var count uint64 1165 | 1166 | err = rows.Scan(&name, &count) 1167 | if err != nil { 1168 | return nil, err 1169 | } 1170 | jobMap[name] = count 1171 | } 1172 | 1173 | return jobMap, nil 1174 | } 1175 | 1176 | // ExecutorCountsForMonth gets a map of executor count to number of instances with that many executors in a month 1177 | // analogous to executorCount2Number in generateStats.groovy 1178 | func ExecutorCountsForMonth(db sq.BaseRunner, year, month int) (map[string]uint64, error) { 1179 | rows, err := PSQL(db).Select("executors"). 1180 | From("instance_reports"). 1181 | Where(sq.Eq{"year": year}). 1182 | Where(sq.Eq{"month": month}). 1183 | Where(sq.GtOrEq{"count_for_month": 2}). 1184 | Query() 1185 | if err != nil { 1186 | return nil, err 1187 | } 1188 | defer func() { 1189 | _ = rows.Close() 1190 | }() 1191 | 1192 | countMap := make(map[string]uint64) 1193 | 1194 | for rows.Next() { 1195 | var count uint64 1196 | 1197 | err = rows.Scan(&count) 1198 | if err != nil { 1199 | return nil, err 1200 | } 1201 | cStr := fmt.Sprintf("%d", count) 1202 | if _, ok := countMap[cStr]; !ok { 1203 | countMap[cStr] = 0 1204 | } 1205 | countMap[cStr]++ 1206 | } 1207 | 1208 | return countMap, nil 1209 | } 1210 | 1211 | // OSCountsForMonth gets the total number of each known OS type in a month 1212 | // analogous to nodesOnOS2Number in generateStats.groovy 1213 | func OSCountsForMonth(db sq.BaseRunner, year, month int) (map[string]uint64, error) { 1214 | rows, err := PSQL(db).Select("o.name", "sum(nr.value::int) as total"). 1215 | From("instance_reports i, jsonb_each_text(i.nodes) nr"). 1216 | Join("os_types o on o.id = nr.key::int"). 1217 | Where(sq.Eq{"i.year": year}). 1218 | Where(sq.Eq{"i.month": month}). 1219 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1220 | GroupBy("o.name"). 1221 | OrderBy("total asc"). 1222 | Query() 1223 | if err != nil { 1224 | return nil, err 1225 | } 1226 | defer func() { 1227 | _ = rows.Close() 1228 | }() 1229 | 1230 | osMap := make(map[string]uint64) 1231 | 1232 | for rows.Next() { 1233 | var name string 1234 | var count uint64 1235 | 1236 | err = rows.Scan(&name, &count) 1237 | if err != nil { 1238 | return nil, err 1239 | } 1240 | osMap[name] = count 1241 | } 1242 | 1243 | return osMap, nil 1244 | } 1245 | 1246 | // CreateBarSVG takes a dataset and returns byte slices for the corresponding .svg and .csv files 1247 | func CreateBarSVG(title string, data map[string]uint64, scaleReduction int, sortByValue, asVersion, asNumber bool, filterFunc func(string, uint64) bool) ([]byte, []byte, error) { 1248 | sortedData, maxVal := asSortedPairsAndMaxValue(data, sortByValue, asVersion, asNumber, filterFunc) 1249 | 1250 | viewWidth := (len(sortedData) * 15) + 50 1251 | 1252 | doc := etree.NewDocument() 1253 | svg := doc.CreateElement("svg") 1254 | _ = svg.CreateAttr("xmlns", "http://www.w3.org/2000/svg") 1255 | _ = svg.CreateAttr("version", "1.1") 1256 | _ = svg.CreateAttr("preserveAspectRatio", "xMidYMid meet") 1257 | _ = svg.CreateAttr("viewBox", fmt.Sprintf("0 0 %d %.1f", viewWidth, (float32(maxVal)/float32(scaleReduction))+350)) 1258 | 1259 | for idx, kv := range sortedData { 1260 | barHeight := float64(kv.value) / float64(uint64(scaleReduction)) 1261 | xAxis := (idx + 1) * 15 1262 | yAxis := ((float32(maxVal) / float32(scaleReduction)) - float32(barHeight)) + 50 1263 | textY := yAxis + float32(barHeight) + 5 1264 | 1265 | rect := svg.CreateElement("rect") 1266 | _ = rect.CreateAttr("fill", "blue") 1267 | _ = rect.CreateAttr("height", fmt.Sprintf("%.1f", barHeight)) 1268 | _ = rect.CreateAttr("stroke", "black") 1269 | _ = rect.CreateAttr("width", "12") 1270 | _ = rect.CreateAttr("x", fmt.Sprintf("%d", xAxis)) 1271 | _ = rect.CreateAttr("y", fmt.Sprintf("%.1f", yAxis)) 1272 | 1273 | textElem := svg.CreateElement("text") 1274 | _ = textElem.CreateAttr("x", fmt.Sprintf("%d", xAxis)) 1275 | _ = textElem.CreateAttr("y", fmt.Sprintf("%.1f", textY)) 1276 | _ = textElem.CreateAttr("font-family", "Tahoma") 1277 | _ = textElem.CreateAttr("font-size", "12") 1278 | _ = textElem.CreateAttr("transform", fmt.Sprintf("rotate(90 %d,%.1f)", xAxis, textY)) 1279 | _ = textElem.CreateAttr("text-rendering", "optimizeSpeed") 1280 | _ = textElem.CreateAttr("fill", "#000000") 1281 | textElem.SetText(fmt.Sprintf("%s (%d)", kv.key, kv.value)) 1282 | } 1283 | 1284 | titleElem := svg.CreateElement("text") 1285 | _ = titleElem.CreateAttr("x", "10") 1286 | _ = titleElem.CreateAttr("y", "40") 1287 | _ = titleElem.CreateAttr("font-family", "Tahoma") 1288 | _ = titleElem.CreateAttr("font-size", "20") 1289 | _ = titleElem.CreateAttr("text-rendering", "optimizeSpeed") 1290 | _ = titleElem.CreateAttr("fill", "#000000") 1291 | titleElem.SetText(title) 1292 | 1293 | doc.Indent(2) 1294 | body, err := doc.WriteToBytes() 1295 | if err != nil { 1296 | return nil, nil, err 1297 | } 1298 | 1299 | var builder bytes.Buffer 1300 | for _, v := range sortedData { 1301 | _, err = builder.Write([]byte(fmt.Sprintf(`"%s","%d"`+"\n", v.key, v.value))) 1302 | if err != nil { 1303 | return nil, nil, err 1304 | } 1305 | } 1306 | 1307 | return body, builder.Bytes(), nil 1308 | } 1309 | 1310 | // CreatePieSVG takes a dataset and returns byte slices for the corresponding .svg and .csv files 1311 | func CreatePieSVG(title string, data []uint64, centerX, centerY, radius, upperLeftX, upperLeftY int, labels []string, colors []string) ([]byte, []byte, error) { 1312 | 1313 | totalCount := uint64(0) 1314 | for _, v := range data { 1315 | totalCount += v 1316 | } 1317 | 1318 | angles := make([]float64, len(data)) 1319 | for i := range data { 1320 | angles[i] = float64(data[i]) / float64(totalCount) * math.Pi * 2 1321 | } 1322 | 1323 | startAngle := float64(0) 1324 | squareHeight := 30 1325 | 1326 | viewWidth := upperLeftX + 350 1327 | viewHeight := upperLeftY + (len(data) * squareHeight) + 30 1328 | 1329 | doc := etree.NewDocument() 1330 | svg := doc.CreateElement("svg") 1331 | _ = svg.CreateAttr("xmlns", "http://www.w3.org/2000/svg") 1332 | _ = svg.CreateAttr("version", "1.1") 1333 | _ = svg.CreateAttr("preserveAspectRatio", "xMidYMid meet") 1334 | _ = svg.CreateAttr("viewBox", fmt.Sprintf("0 0 %d %d", viewWidth, viewHeight)) 1335 | 1336 | titleElem := svg.CreateElement("text") 1337 | _ = titleElem.CreateAttr("x", "30") 1338 | _ = titleElem.CreateAttr("y", "40") 1339 | _ = titleElem.CreateAttr("font-family", "sans-serif") 1340 | _ = titleElem.CreateAttr("font-size", "16") 1341 | titleElem.SetText(fmt.Sprintf("%s, total: %d", title, totalCount)) 1342 | 1343 | for i, v := range data { 1344 | endAngle := startAngle + angles[i] 1345 | 1346 | x1 := float64(centerX+radius) * math.Sin(startAngle) 1347 | y1 := float64(centerY-radius) * math.Cos(startAngle) 1348 | x2 := float64(centerX+radius) * math.Sin(endAngle) 1349 | y2 := float64(centerY-radius) * math.Cos(endAngle) 1350 | 1351 | greaterThanHalfCircleAdjustment := 0 1352 | if endAngle-startAngle > math.Pi { 1353 | greaterThanHalfCircleAdjustment = 1 1354 | } 1355 | 1356 | pathD := fmt.Sprintf("M %d,%d L %.1f,%.1f A %d,%d 0 %d 1 %.1f,%.1f Z", centerX, centerY, x1, y1, radius, radius, greaterThanHalfCircleAdjustment, x2, y2) 1357 | 1358 | pathElem := svg.CreateElement("path") 1359 | _ = pathElem.CreateAttr("d", pathD) 1360 | _ = pathElem.CreateAttr("fill", colors[i]) 1361 | _ = pathElem.CreateAttr("stroke", "black") 1362 | _ = pathElem.CreateAttr("stroke-width", "1") 1363 | 1364 | // Next wedge starts at the end of this wedge 1365 | startAngle = endAngle 1366 | 1367 | rectElem := svg.CreateElement("rect") 1368 | _ = rectElem.CreateAttr("x", fmt.Sprintf("%d", upperLeftX)) 1369 | _ = rectElem.CreateAttr("y", fmt.Sprintf("%d", upperLeftY+squareHeight+i)) 1370 | _ = rectElem.CreateAttr("width", "20") 1371 | _ = rectElem.CreateAttr("height", fmt.Sprintf("%d", squareHeight)) 1372 | _ = rectElem.CreateAttr("fill", colors[i]) 1373 | _ = rectElem.CreateAttr("stroke", "black") 1374 | _ = rectElem.CreateAttr("stroke-width", "1") 1375 | 1376 | textElem := svg.CreateElement("text") 1377 | _ = textElem.CreateAttr("x", fmt.Sprintf("%d", upperLeftX+30)) 1378 | _ = textElem.CreateAttr("y", fmt.Sprintf("%d", upperLeftY+squareHeight*i+18)) 1379 | _ = textElem.CreateAttr("font-family", "sans-serif") 1380 | _ = textElem.CreateAttr("font-size", "16") 1381 | textElem.SetText(fmt.Sprintf("%s (%d)", labels[i], v)) 1382 | } 1383 | 1384 | doc.Indent(2) 1385 | body, err := doc.WriteToBytes() 1386 | if err != nil { 1387 | return nil, nil, err 1388 | } 1389 | 1390 | var builder bytes.Buffer 1391 | for i, v := range labels { 1392 | _, err = builder.Write([]byte(fmt.Sprintf(`"%d","%s"`+"\n", data[i], v))) 1393 | if err != nil { 1394 | return nil, nil, err 1395 | } 1396 | } 1397 | 1398 | return body, builder.Bytes(), nil 1399 | } 1400 | 1401 | // DefaultFilter returns true for all string/uint64 pairs 1402 | func DefaultFilter(_ string, _ uint64) bool { 1403 | return true 1404 | } 1405 | 1406 | type kvPair struct { 1407 | key string 1408 | value uint64 1409 | } 1410 | 1411 | func asSortedPairsAndMaxValue(data map[string]uint64, byValue, asVersion, asNumber bool, filterFunc func(string, uint64) bool) ([]kvPair, uint64) { 1412 | maxVal := uint64(0) 1413 | 1414 | var sp []kvPair 1415 | for k, v := range data { 1416 | if filterFunc(k, v) { 1417 | sp = append(sp, kvPair{ 1418 | key: k, 1419 | value: v, 1420 | }) 1421 | if v > maxVal { 1422 | maxVal = v 1423 | } 1424 | } 1425 | } 1426 | 1427 | if byValue { 1428 | sort.Slice(sp, func(i, j int) bool { 1429 | if sp[i].value < sp[j].value { 1430 | return true 1431 | } 1432 | if sp[i].value > sp[j].value { 1433 | return false 1434 | } 1435 | return sp[i].key < sp[j].key 1436 | }) 1437 | } else if asVersion { 1438 | sort.Slice(sp, func(i, j int) bool { 1439 | svI, err := semver.NewVersion(sp[i].key) 1440 | if err != nil { 1441 | return sp[i].key < sp[j].key 1442 | } 1443 | svJ, err := semver.NewVersion(sp[j].key) 1444 | if err != nil { 1445 | return sp[i].key < sp[j].key 1446 | } 1447 | 1448 | return svI.LessThan(svJ) 1449 | }) 1450 | } else if asNumber { 1451 | sort.Slice(sp, func(i, j int) bool { 1452 | nI, err := strconv.Atoi(sp[i].key) 1453 | if err != nil { 1454 | return sp[i].key < sp[j].key 1455 | } 1456 | nJ, err := strconv.Atoi(sp[j].key) 1457 | if err != nil { 1458 | return sp[i].key < sp[j].key 1459 | } 1460 | 1461 | return nI < nJ 1462 | }) 1463 | } else { 1464 | sort.Slice(sp, func(i, j int) bool { 1465 | return sp[i].key < sp[j].key 1466 | }) 1467 | } 1468 | 1469 | return sp, maxVal 1470 | } 1471 | 1472 | func pluginInstallsByMonthForName(db sq.BaseRunner, currentYear, currentMonth int, idToPlugin map[uint64]Plugin) (map[string]map[string]uint64, error) { 1473 | monthCount := make(map[string]map[string]uint64) 1474 | 1475 | rows, err := PSQL(db).Select("pr.id", "i.year", "i.month", "count(*)"). 1476 | From("instance_reports i, unnest(i.plugins) pr(id)"). 1477 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1478 | OrderBy("pr.id", "i.year", "i.month"). 1479 | GroupBy("pr.id", "i.year", "i.month"). 1480 | Query() 1481 | if err != nil { 1482 | return nil, err 1483 | } 1484 | defer func() { 1485 | _ = rows.Close() 1486 | }() 1487 | 1488 | for rows.Next() { 1489 | var i uint64 1490 | var y, m int 1491 | var c uint64 1492 | 1493 | err = rows.Scan(&i, &y, &m, &c) 1494 | if err != nil { 1495 | return nil, err 1496 | } 1497 | 1498 | if !(y == currentYear && m == currentMonth) { 1499 | p, ok := idToPlugin[i] 1500 | if !ok { 1501 | return nil, fmt.Errorf("no plugin found for id %d", i) 1502 | } 1503 | monthTS := startDateForYearMonth(y, m) 1504 | if _, ok := monthCount[p.Name]; !ok { 1505 | monthCount[p.Name] = make(map[string]uint64) 1506 | } 1507 | mStr := fmt.Sprintf("%d", monthTS.UnixMilli()) 1508 | if _, ok := monthCount[p.Name][mStr]; !ok { 1509 | monthCount[p.Name][mStr] = 0 1510 | } 1511 | monthCount[p.Name][mStr] += c 1512 | } 1513 | } 1514 | 1515 | return monthCount, nil 1516 | } 1517 | 1518 | func pluginInstallsByVersionForName(db sq.BaseRunner, year, month int, idToPlugin map[uint64]Plugin) (map[string]map[string]uint64, error) { 1519 | monthCount := make(map[string]map[string]uint64) 1520 | 1521 | rows, err := PSQL(db).Select("pr.id", "count(*)"). 1522 | From("instance_reports i, unnest(i.plugins) pr(id)"). 1523 | Where(sq.Eq{"i.year": year}). 1524 | Where(sq.Eq{"i.month": month}). 1525 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1526 | OrderBy("pr.id"). 1527 | GroupBy("pr.id"). 1528 | Query() 1529 | if err != nil { 1530 | return nil, err 1531 | } 1532 | defer func() { 1533 | _ = rows.Close() 1534 | }() 1535 | 1536 | for rows.Next() { 1537 | var i uint64 1538 | var c uint64 1539 | 1540 | err = rows.Scan(&i, &c) 1541 | if err != nil { 1542 | return nil, err 1543 | } 1544 | 1545 | p, ok := idToPlugin[i] 1546 | if !ok { 1547 | return nil, fmt.Errorf("no plugin found for id %d", i) 1548 | } 1549 | 1550 | if p.Version == questionVersion { 1551 | continue 1552 | } 1553 | 1554 | if _, ok := monthCount[p.Name]; !ok { 1555 | monthCount[p.Name] = make(map[string]uint64) 1556 | } 1557 | monthCount[p.Name][p.Version] = c 1558 | } 1559 | 1560 | return monthCount, nil 1561 | } 1562 | 1563 | // jvmIDsForJSON gets all jvm_versions IDs that we actually care about for reporting, filtering out eccentric versions. 1564 | func jvmIDsForJSON(db sq.BaseRunner) ([]uint64, error) { 1565 | var jvmIDs []uint64 1566 | rows, err := PSQL(db).Select("id"). 1567 | From(JVMVersionsTable). 1568 | Where(`name ~ '^(\d+|\d\.\d)$'`). 1569 | Query() 1570 | defer func() { 1571 | _ = rows.Close() 1572 | }() 1573 | if err != nil { 1574 | return nil, err 1575 | } 1576 | for rows.Next() { 1577 | var id uint64 1578 | err = rows.Scan(&id) 1579 | if err != nil { 1580 | return nil, err 1581 | } 1582 | jvmIDs = append(jvmIDs, id) 1583 | } 1584 | 1585 | return jvmIDs, nil 1586 | } 1587 | 1588 | func jenkinsVersions2x(db sq.BaseRunner) ([]uint64, error) { 1589 | var jvIDs []uint64 1590 | rows, err := PSQL(db).Select("id"). 1591 | From(JenkinsVersionsTable). 1592 | Where(`version ~ '^2\.'`). 1593 | Query() 1594 | defer func() { 1595 | _ = rows.Close() 1596 | }() 1597 | if err != nil { 1598 | return nil, err 1599 | } 1600 | for rows.Next() { 1601 | var id uint64 1602 | err = rows.Scan(&id) 1603 | if err != nil { 1604 | return nil, err 1605 | } 1606 | jvIDs = append(jvIDs, id) 1607 | } 1608 | 1609 | return jvIDs, nil 1610 | } 1611 | 1612 | func allOrderedMonths(db sq.BaseRunner, currentYear, currentMonth int) ([]yearMonth, error) { 1613 | var yearMonths []yearMonth 1614 | rows, err := PSQL(db).Select("year", "month"). 1615 | From(InstanceReportsTable). 1616 | Where(fmt.Sprintf("NOT (year = %d and month = %d)", currentYear, currentMonth)). 1617 | OrderBy("year", "month"). 1618 | GroupBy("year", "month"). 1619 | Query() 1620 | defer func() { 1621 | _ = rows.Close() 1622 | }() 1623 | if err != nil { 1624 | return nil, err 1625 | } 1626 | 1627 | for rows.Next() { 1628 | ym := yearMonth{} 1629 | err = rows.Scan(&ym.year, &ym.month) 1630 | if err != nil { 1631 | return nil, err 1632 | } 1633 | yearMonths = append(yearMonths, ym) 1634 | } 1635 | 1636 | return yearMonths, nil 1637 | } 1638 | 1639 | func installCountsByMonth(db sq.BaseRunner, currentYear, currentMonth int) (map[string]uint64, error) { 1640 | installs := make(map[string]uint64) 1641 | 1642 | rows, err := PSQL(db).Select("year", "month", "count(*)"). 1643 | From(InstanceReportsTable). 1644 | Where(sq.GtOrEq{"count_for_month": 2}). 1645 | GroupBy("year", "month"). 1646 | OrderBy("year", "month"). 1647 | Query() 1648 | if err != nil { 1649 | return nil, err 1650 | } 1651 | defer func() { 1652 | _ = rows.Close() 1653 | }() 1654 | 1655 | for rows.Next() { 1656 | var y, m int 1657 | var c uint64 1658 | 1659 | err = rows.Scan(&y, &m, &c) 1660 | if err != nil { 1661 | return nil, err 1662 | } 1663 | 1664 | if !(y == currentYear && m == currentMonth) { 1665 | startTS := fmt.Sprintf("%d", startDateForYearMonth(y, m).UnixMilli()) 1666 | installs[startTS] = c 1667 | } 1668 | } 1669 | 1670 | return installs, nil 1671 | } 1672 | 1673 | func maxInstanceVersionForMonth(db sq.BaseRunner, year, month int) (map[string]string, error) { 1674 | maxVersions := make(map[string]string) 1675 | 1676 | rows, err := PSQL(db).Select("i.instance_id", "max(jv.version)"). 1677 | From("instance_reports i"). 1678 | Join("jenkins_versions jv on jv.id = i.version"). 1679 | Where(sq.Eq{"i.year": year}). 1680 | Where(sq.Eq{"i.month": month}). 1681 | Where(sq.GtOrEq{"i.count_for_month": 2}). 1682 | Where(`jv.version ~ '^\d'`). 1683 | Where("jv.version not like '%private%'"). 1684 | GroupBy("i.instance_id"). 1685 | Query() 1686 | if err != nil { 1687 | return nil, err 1688 | } 1689 | defer func() { 1690 | _ = rows.Close() 1691 | }() 1692 | 1693 | for rows.Next() { 1694 | var id, version string 1695 | err = rows.Scan(&id, &version) 1696 | if err != nil { 1697 | return nil, err 1698 | } 1699 | maxVersions[id] = version 1700 | } 1701 | 1702 | return maxVersions, nil 1703 | } 1704 | 1705 | func allPluginNames(db sq.BaseRunner) ([]string, error) { 1706 | var names []string 1707 | 1708 | rows, err := PSQL(db).Select("name"). 1709 | From(PluginsTable). 1710 | GroupBy("name"). 1711 | Query() 1712 | if err != nil { 1713 | return nil, err 1714 | } 1715 | 1716 | for rows.Next() { 1717 | var n string 1718 | 1719 | err = rows.Scan(&n) 1720 | if err != nil { 1721 | return nil, err 1722 | } 1723 | 1724 | names = append(names, n) 1725 | } 1726 | 1727 | return names, nil 1728 | } 1729 | 1730 | func pluginIDsToPlugin(db sq.BaseRunner) (map[uint64]Plugin, error) { 1731 | plugins := make(map[uint64]Plugin) 1732 | 1733 | rows, err := PSQL(db).Select("id", "name", "version"). 1734 | From(PluginsTable). 1735 | Query() 1736 | if err != nil { 1737 | return nil, err 1738 | } 1739 | 1740 | for rows.Next() { 1741 | var i uint64 1742 | var n, v string 1743 | 1744 | err = rows.Scan(&i, &n, &v) 1745 | if err != nil { 1746 | return nil, err 1747 | } 1748 | 1749 | plugins[i] = Plugin{ 1750 | ID: i, 1751 | Name: n, 1752 | Version: v, 1753 | } 1754 | } 1755 | 1756 | return plugins, nil 1757 | } 1758 | 1759 | func writeFile(filename string, data []byte) error { 1760 | return ioutil.WriteFile(filename, data, 0644) //nolint:gosec 1761 | } 1762 | --------------------------------------------------------------------------------