57 |65 | `, err.Error())) 66 | return nil 67 | } 68 | 69 | res.Set("innerHTML", fmt.Sprintf(` 70 |58 | 59 | Parsing error 60 |61 |62 | %s 63 |64 |
71 |79 |72 | 73 | Success 74 |75 |76 | Input has been parsed successfully. 77 |78 |
%sWrite the following content as web.yml file and start Prometheus with --web.config.file=web.yml
%s99 |107 | `, err.Error())) 108 | } 109 | -------------------------------------------------------------------------------- /util/collectors/textfile_test.go: -------------------------------------------------------------------------------- 1 | // Copyright The o11y toolkit Authors 2 | // spdx-license-identifier: apache-2.0 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at: 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package collectors 17 | 18 | import ( 19 | "net/http" 20 | "net/http/httptest" 21 | "os" 22 | "testing" 23 | 24 | "github.com/alecthomas/kingpin/v2" 25 | "github.com/go-kit/log" 26 | "github.com/prometheus/client_golang/prometheus" 27 | "github.com/prometheus/client_golang/prometheus/promhttp" 28 | "github.com/prometheus/common/promlog" 29 | "github.com/prometheus/common/promlog/flag" 30 | ) 31 | 32 | type collectorAdapter struct { 33 | prometheus.Collector 34 | } 35 | 36 | // Describe implements the prometheus.Collector interface. 37 | func (a collectorAdapter) Describe(ch chan<- *prometheus.Desc) { 38 | // We have to send *some* metric in Describe, but we don't know which ones 39 | // we're going to get, so just send a dummy metric. 40 | ch <- prometheus.NewDesc("dummy_metric", "Dummy metric.", nil, nil) 41 | } 42 | 43 | func TestTextfileCollector(t *testing.T) { 44 | tests := []struct { 45 | path string 46 | out string 47 | }{ 48 | { 49 | path: "fixtures/textfile/nonexistent_path", 50 | out: "fixtures/textfile/nonexistent_path.out", 51 | }, 52 | { 53 | path: "fixtures/textfile/client_side_timestamp", 54 | out: "fixtures/textfile/client_side_timestamp.out", 55 | }, 56 | { 57 | path: "fixtures/textfile/different_metric_types", 58 | out: "fixtures/textfile/different_metric_types.out", 59 | }, 60 | { 61 | path: "fixtures/textfile/inconsistent_metrics", 62 | out: "fixtures/textfile/inconsistent_metrics.out", 63 | }, 64 | { 65 | path: "fixtures/textfile/histogram", 66 | out: "fixtures/textfile/histogram.out", 67 | }, 68 | { 69 | path: "fixtures/textfile/histogram_extra_dimension", 70 | out: "fixtures/textfile/histogram_extra_dimension.out", 71 | }, 72 | { 73 | path: "fixtures/textfile/summary", 74 | out: "fixtures/textfile/summary.out", 75 | }, 76 | { 77 | path: "fixtures/textfile/summary_extra_dimension", 78 | out: "fixtures/textfile/summary_extra_dimension.out", 79 | }, 80 | } 81 | 82 | for i, test := range tests { 83 | mtime := 1.0 84 | c := &textFileCollector{ 85 | path: test.path + "/metrics.prom", 86 | mtime: &mtime, 87 | logger: log.NewNopLogger(), 88 | } 89 | 90 | // Suppress a log message about `nonexistent_path` not existing, this is 91 | // expected and clutters the test output. 92 | promlogConfig := &promlog.Config{} 93 | flag.AddFlags(kingpin.CommandLine, promlogConfig) 94 | if _, err := kingpin.CommandLine.Parse([]string{"--log.level", "debug"}); err != nil { 95 | t.Fatal(err) 96 | } 97 | 98 | registry := prometheus.NewRegistry() 99 | registry.MustRegister(collectorAdapter{c}) 100 | 101 | rw := httptest.NewRecorder() 102 | promhttp.HandlerFor(registry, promhttp.HandlerOpts{}).ServeHTTP(rw, &http.Request{}) 103 | got := string(rw.Body.String()) 104 | 105 | want, err := os.ReadFile(test.out) 106 | if err != nil { 107 | t.Fatalf("%d. error reading fixture file %s: %s", i, test.out, err) 108 | } 109 | 110 | if string(want) != got { 111 | t.Fatalf("%d.%q want:\n\n%s\n\ngot:\n\n%s", i, test.path, string(want), got) 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /util/collectors/fixtures/textfile/glob_extra_dimension.out: -------------------------------------------------------------------------------- 1 | # HELP node_textfile_mtime_seconds Unixtime mtime of textfiles successfully read. 2 | # TYPE node_textfile_mtime_seconds gauge 3 | node_textfile_mtime_seconds{file="fixtures/textfile/histogram_extra_dimension/metrics.prom"} 1 4 | node_textfile_mtime_seconds{file="fixtures/textfile/summary_extra_dimension/metrics.prom"} 1 5 | # HELP node_textfile_scrape_error 1 if there was an error opening or reading a file, 0 otherwise 6 | # TYPE node_textfile_scrape_error gauge 7 | node_textfile_scrape_error 0 8 | # HELP prometheus_rule_evaluation_duration_seconds The duration for a rule to execute. 9 | # TYPE prometheus_rule_evaluation_duration_seconds summary 10 | prometheus_rule_evaluation_duration_seconds{handler="",rule_type="alerting",quantile="0.9"} 0.001765451 11 | prometheus_rule_evaluation_duration_seconds{handler="",rule_type="alerting",quantile="0.99"} 0.018672076 12 | prometheus_rule_evaluation_duration_seconds_sum{handler="",rule_type="alerting"} 214.85081044700146 13 | prometheus_rule_evaluation_duration_seconds_count{handler="",rule_type="alerting"} 185209 14 | prometheus_rule_evaluation_duration_seconds{handler="",rule_type="recording",quantile="0.5"} 4.3132e-05 15 | prometheus_rule_evaluation_duration_seconds{handler="",rule_type="recording",quantile="0.9"} 8.9295e-05 16 | prometheus_rule_evaluation_duration_seconds{handler="",rule_type="recording",quantile="0.99"} 0.000193657 17 | prometheus_rule_evaluation_duration_seconds_sum{handler="",rule_type="recording"} 185091.01317759082 18 | prometheus_rule_evaluation_duration_seconds_count{handler="",rule_type="recording"} 1.0020195e+08 19 | prometheus_rule_evaluation_duration_seconds{handler="foo",rule_type="alerting",quantile="0.5"} 0.000571464 20 | prometheus_rule_evaluation_duration_seconds_sum{handler="foo",rule_type="alerting"} 0 21 | prometheus_rule_evaluation_duration_seconds_count{handler="foo",rule_type="alerting"} 0 22 | # HELP prometheus_tsdb_compaction_chunk_range Final time range of chunks on their first compaction 23 | # TYPE prometheus_tsdb_compaction_chunk_range histogram 24 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="100"} 0 25 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="400"} 0 26 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="1600"} 0 27 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="6400"} 0 28 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="25600"} 7 29 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="102400"} 7 30 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="409600"} 1.412839e+06 31 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="1.6384e+06"} 1.69185e+06 32 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="6.5536e+06"} 1.691853e+06 33 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="2.62144e+07"} 1.691853e+06 34 | prometheus_tsdb_compaction_chunk_range_bucket{foo="bar",le="+Inf"} 1.691853e+06 35 | prometheus_tsdb_compaction_chunk_range_sum{foo="bar"} 6.71393432189e+11 36 | prometheus_tsdb_compaction_chunk_range_count{foo="bar"} 1.691853e+06 37 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="100"} 0 38 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="400"} 0 39 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="1600"} 0 40 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="6400"} 0 41 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="25600"} 7 42 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="102400"} 7 43 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="409600"} 1.412839e+06 44 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="1.6384e+06"} 1.69185e+06 45 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="6.5536e+06"} 1.691853e+06 46 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="2.62144e+07"} 1.691853e+06 47 | prometheus_tsdb_compaction_chunk_range_bucket{foo="baz",le="+Inf"} 1.691853e+06 48 | prometheus_tsdb_compaction_chunk_range_sum{foo="baz"} 6.71393432189e+11 49 | prometheus_tsdb_compaction_chunk_range_count{foo="baz"} 1.691853e+06 50 | -------------------------------------------------------------------------------- /docs/content/httpclient.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Prometheus client configuration 3 | --- 4 | 5 | Command line tools that connect to a Prometheus server can take a file path in 6 | a `--client.config` parameter. This is useful to connect to a Prometheus server 7 | with authentication or self-signed certificates. 8 | 9 | The file is written in [YAML format](https://en.wikipedia.org/wiki/YAML), 10 | defined by the scheme described below. 11 | Brackets indicate that a parameter is optional. For non-list parameters the 12 | value is set to the specified default. 13 | 14 | Generic placeholders are defined as follows: 15 | 16 | * `100 | 101 | Error 102 |103 |104 | %s 105 |106 |
| Username | 107 |Password | 108 |109 | |
|---|
Cost:
Increasing cost will increase Prometheus server CPU usage and time
118 | to authenticate users.
68 |76 | `, err.Error())) 77 | return nil 78 | } 79 | 80 | if strings.TrimSpace(args[0].String()) == "" { 81 | res.Set("innerHTML", ` 82 |69 | 70 | Parsing error 71 |72 |73 | %s 74 |75 |
83 |91 | `) 92 | return nil 93 | } 94 | 95 | if len(problems) == 0 { 96 | res.Set("innerHTML", ` 97 |84 | 85 | No input 86 |87 |88 | The input provided is empty. Please paste metrics into the text area. 89 |90 |
98 |106 | `) 107 | return nil 108 | } 109 | 110 | var pbs string 111 | 112 | for _, p := range problems { 113 | pbs += fmt.Sprintf("99 | 100 | Success 101 |102 |103 | Input has been parsed successfully. 104 |105 |
%s: %s118 |127 | `, pbs)) 128 | 129 | return nil 130 | } 131 | -------------------------------------------------------------------------------- /cmd/oy-csv-to-targets/main.go: -------------------------------------------------------------------------------- 1 | // Copyright The o11y toolkit Authors 2 | // spdx-license-identifier: apache-2.0 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at: 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "bytes" 20 | "encoding/csv" 21 | "encoding/json" 22 | "fmt" 23 | "io" 24 | "os" 25 | "path/filepath" 26 | 27 | "github.com/alecthomas/kingpin/v2" 28 | 29 | promlabels "github.com/prometheus/prometheus/model/labels" 30 | 31 | "github.com/o11ydev/oy-toolkit/util/cmd" 32 | ) 33 | 34 | var ( 35 | input = kingpin.Flag("input.file", "Path to a CSV file to use as an input.").PlaceHolder("input.csv").Required().String() 36 | output = kingpin.Flag("output.file", "Path to a json file to use as an output.").PlaceHolder("targets.json").String() 37 | ) 38 | 39 | type target struct { 40 | Labels promlabels.Labels `json:"labels"` 41 | Targets []string `json:"targets"` 42 | } 43 | 44 | func main() { 45 | logger := cmd.InitCmd("oy-csv-to-targets") 46 | 47 | err := csvToJSON(*input, *output) 48 | if err != nil { 49 | logger.Log("error", err.Error()) 50 | } 51 | } 52 | 53 | // Convert a CSV file to a JSON with Prometheus targets. 54 | func csvToJSON(csvFile, jsonFile string) error { 55 | f, err := os.Open(csvFile) 56 | if err != nil { 57 | return fmt.Errorf("failed to open CSV file: %w", err) 58 | } 59 | defer f.Close() 60 | 61 | r := csv.NewReader(f) 62 | 63 | // Read the first line of the CSV to get the headers (i.e. label names). 64 | headers, err := r.Read() 65 | if err != nil { 66 | return fmt.Errorf("failed to read headers from CSV: %w", err) 67 | } 68 | 69 | targets := []target{} 70 | 71 | // Read the rest of the CSV to get the target values 72 | for { 73 | record, err := r.Read() 74 | if err == io.EOF { 75 | break 76 | } 77 | if err != nil { 78 | return fmt.Errorf("failed to read record from CSV: %w", err) 79 | } 80 | 81 | // Create a map to hold the label values 82 | labels := map[string]string{} 83 | 84 | // Loop through the headers and record values to create the label 85 | // values. 86 | // Ignore the first one. 87 | for i, header := range headers { 88 | if i > 0 { 89 | labels[header] = record[i] 90 | } 91 | } 92 | 93 | // Create a new Prometheus target using the label values and the first 94 | // record value as the target address. 95 | t := target{ 96 | Labels: promlabels.FromMap(labels), 97 | Targets: []string{record[0]}, 98 | } 99 | 100 | targets = append(targets, t) 101 | } 102 | 103 | b, err := json.Marshal(targets) 104 | if err != nil { 105 | return fmt.Errorf("failed to marshal targets into JSON: %w", err) 106 | } 107 | 108 | // If there is no file, pretty print to stdout. 109 | if jsonFile == "" { 110 | var out bytes.Buffer 111 | err = json.Indent(&out, b, "", " ") 112 | if err != nil { 113 | return fmt.Errorf("failed to pretty print JSON: %w", err) 114 | } 115 | 116 | fmt.Println(out.String()) 117 | return nil 118 | } 119 | 120 | // Otherwise, output to the file using a temp file. 121 | // Create a temp file in the same directory as the destination file. 122 | tmpFile, err := os.CreateTemp(filepath.Dir(jsonFile), "tmp") 123 | if err != nil { 124 | return fmt.Errorf("failed to create temp file: %w", err) 125 | } 126 | 127 | _, err = tmpFile.Write(b) 128 | if err != nil { 129 | tmpFile.Close() 130 | os.Remove(tmpFile.Name()) 131 | return fmt.Errorf("failed to write to temp file: %w", err) 132 | } 133 | 134 | err = tmpFile.Close() 135 | if err != nil { 136 | os.Remove(tmpFile.Name()) 137 | return fmt.Errorf("failed to close temp file: %w", err) 138 | } 139 | 140 | // Rename the temp file to the destination file. 141 | err = os.Rename(tmpFile.Name(), jsonFile) 142 | if err != nil { 143 | os.Remove(tmpFile.Name()) 144 | return fmt.Errorf("failed to rename temp file: %w", err) 145 | } 146 | 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /docs/static/custom.css: -------------------------------------------------------------------------------- 1 | /* Light mode theming */ 2 | :root, 3 | :root[color-mode="light"] { 4 | --header-background: #ff004b; 5 | --header-font-color: #ffffff; 6 | 7 | --body-background: #ffffff; 8 | --body-font-color: #343a40; 9 | 10 | --mark-color: #ffab00; 11 | 12 | --button-background: #62cb97; 13 | --button-border-color: #4ec58a; 14 | 15 | --link-color: #518169; 16 | --link-color-visited: #c54e8a; 17 | 18 | --code-background: #f5f6f8; 19 | --code-accent-color: #e3e7eb; 20 | --code-accent-color-lite: #eff1f3; 21 | 22 | --code-copy-font-color: #6b7784; 23 | --code-copy-border-color: #adb4bc; 24 | --code-copy-success-color: #00c853; 25 | 26 | --accent-color: #e9ecef; 27 | --accent-color-lite: #f8f9fa; 28 | 29 | --control-icons: #b2bac1; 30 | 31 | --footer-background: #2f333e; 32 | --footer-font-color: #ffffff; 33 | --footer-link-color: #ffcc5c; 34 | --footer-link-color-visited: #ffcc5c; 35 | } 36 | @media (prefers-color-scheme: light) { 37 | :root { 38 | --header-background: #ff004b; 39 | --header-font-color: #ffffff; 40 | 41 | --body-background: #ffffff; 42 | --body-font-color: #343a40; 43 | 44 | --mark-color: #ffab00; 45 | 46 | --button-background: #62cb97; 47 | --button-border-color: #4ec58a; 48 | 49 | --link-color: #518169; 50 | --link-color-visited: #c54e8a; 51 | 52 | --code-background: #f5f6f8; 53 | --code-accent-color: #e3e7eb; 54 | --code-accent-color-lite: #eff1f3; 55 | 56 | --code-copy-font-color: #6b7784; 57 | --code-copy-border-color: #adb4bc; 58 | --code-copy-success-color: #00c853; 59 | 60 | --accent-color: #e9ecef; 61 | --accent-color-lite: #f8f9fa; 62 | 63 | --control-icons: #b2bac1; 64 | 65 | --footer-background: #2f333e; 66 | --footer-font-color: #ffffff; 67 | --footer-link-color: #ffcc5c; 68 | --footer-link-color-visited: #ffcc5c; 69 | } 70 | } 71 | 72 | /* Dark mode theming */ 73 | :root[color-mode="dark"] { 74 | --header-background: #ff004b; 75 | --header-font-color: #ffffff; 76 | 77 | --body-background: #343a40; 78 | --body-font-color: #ced3d8; 79 | 80 | --mark-color: #ffab00; 81 | 82 | --button-background: #62cb97; 83 | --button-border-color: #4ec58a; 84 | 85 | --link-color: #7ac29e; 86 | --link-color-visited: #c27a9e; 87 | 88 | --code-background: #2f353a; 89 | --code-accent-color: #262b2f; 90 | --code-accent-color-lite: #2b3035; 91 | 92 | --code-copy-font-color: #adb4bc; 93 | --code-copy-border-color: #808c98; 94 | --code-copy-success-color: #00c853; 95 | 96 | --accent-color: #2b3035; 97 | --accent-color-lite: #2f353a; 98 | 99 | --control-icons: #b2bac1; 100 | 101 | --footer-background: #2f333e; 102 | --footer-font-color: #ffffff; 103 | --footer-link-color: #ffcc5c; 104 | --footer-link-color-visited: #ffcc5c; 105 | } 106 | @media (prefers-color-scheme: dark) { 107 | :root { 108 | --header-background: #ff004b; 109 | --header-font-color: #ffffff; 110 | 111 | --body-background: #343a40; 112 | --body-font-color: #ced3d8; 113 | 114 | --mark-color: #ffab00; 115 | 116 | --button-background: #62cb97; 117 | --button-border-color: #4ec58a; 118 | 119 | --link-color: #7ac29e; 120 | --link-color-visited: #c27a9e; 121 | 122 | --code-background: #2f353a; 123 | --code-accent-color: #262b2f; 124 | --code-accent-color-lite: #2b3035; 125 | 126 | --code-copy-font-color: #adb4bc; 127 | --code-copy-border-color: #808c98; 128 | --code-copy-success-color: #00c853; 129 | 130 | --accent-color: #2b3035; 131 | --accent-color-lite: #2f353a; 132 | 133 | --control-icons: #b2bac1; 134 | 135 | --footer-background: #2f333e; 136 | --footer-font-color: #ffffff; 137 | --footer-link-color: #ffcc5c; 138 | --footer-link-color-visited: #ffcc5c; 139 | } 140 | } 141 | 142 | .gdoc-brand img { 143 | width: 15rem; 144 | height: 3rem; 145 | } 146 | 147 | .gdoc-brand > span > span { 148 | display: none; 149 | } 150 | 151 | .gdoc-header > .container { 152 | padding-top: 0.5rem; 153 | padding-bottom: 0.5rem; 154 | } 155 | 156 | @media screen and (max-width: 600px) { 157 | table.flext thead { 158 | display: none; 159 | } 160 | table.flext td { 161 | display: flex; 162 | } 163 | 164 | table.flext td.flext::before { 165 | content: attr(label); 166 | font-weight: bold; 167 | width: 120px; 168 | min-width: 120px; 169 | } 170 | } 171 | 172 | pre code { 173 | overflow-x:auto; 174 | } 175 | -------------------------------------------------------------------------------- /cmd/oy-periodic-queries/collector.go: -------------------------------------------------------------------------------- 1 | // Copyright The o11y toolkit Authors 2 | // spdx-license-identifier: apache-2.0 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at: 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "os" 22 | "strconv" 23 | "strings" 24 | "time" 25 | 26 | "github.com/go-kit/log" 27 | "github.com/go-kit/log/level" 28 | 29 | "github.com/prometheus/client_golang/api" 30 | v1 "github.com/prometheus/client_golang/api/prometheus/v1" 31 | "github.com/prometheus/client_golang/prometheus" 32 | "github.com/prometheus/common/model" 33 | 34 | "github.com/o11ydev/oy-toolkit/util/period" 35 | ) 36 | 37 | // newGroupCollector returns a new Collector exposing metrics from rules 38 | // evaluated at periodic intervals. 39 | func newGroupsCollector(logger log.Logger, client api.Client, groups []Group) (prometheus.Collector, error) { 40 | c := &groupsCollector{ 41 | groups: groups, 42 | logger: logger, 43 | client: client, 44 | cache: NewCache(24 * time.Hour), 45 | } 46 | return c, nil 47 | } 48 | 49 | type groupsCollector struct { 50 | groups []Group 51 | logger log.Logger 52 | client api.Client 53 | cache *Cache 54 | } 55 | 56 | // Collect implements the Collector interface. 57 | func (c *groupsCollector) Collect(ch chan<- prometheus.Metric) { 58 | var lastErr error 59 | 60 | defer func() { 61 | var val float64 = 1 62 | if lastErr != nil { 63 | level.Error(c.logger).Log("err", lastErr) 64 | val = 0 65 | } 66 | ch <- prometheus.MustNewConstMetric( 67 | prometheus.NewDesc("periodic_queries_success", "", nil, nil), 68 | prometheus.GaugeValue, 69 | val, 70 | ) 71 | }() 72 | 73 | metrics := []prometheus.Metric{} 74 | now := time.Now().Local() 75 | for _, g := range c.groups { 76 | for _, p := range period.CalculatePeriods(now, g.TimePeriod, time.Duration(g.Lookback), g.IncludeIncompleteRanges) { 77 | for _, r := range g.Rules { 78 | m, err := QueryRangeToMetric(c.logger, c.cache, p, r.Expr, c.client, r.Record, r.Labels) 79 | if err != nil { 80 | lastErr = err 81 | return 82 | } 83 | metrics = append(metrics, m...) 84 | } 85 | } 86 | } 87 | for _, metric := range metrics { 88 | ch <- metric 89 | } 90 | } 91 | 92 | func (c *groupsCollector) Describe(chan<- *prometheus.Desc) { 93 | } 94 | 95 | func QueryRangeToMetric(logger log.Logger, c *Cache, p period.Period, query string, client api.Client, metricName string, additionalLabels map[string]string) ([]prometheus.Metric, error) { 96 | queryAPI := v1.NewAPI(client) 97 | 98 | pq := replaceForPeriod(p, query) 99 | 100 | var ( 101 | result model.Value 102 | err error 103 | ok bool 104 | ) 105 | 106 | var found bool 107 | key := fmt.Sprintf("%d/%s", p.End.Unix(), pq) 108 | if p.Complete { 109 | if result, ok = c.Get(key); ok { 110 | found = true 111 | } 112 | } 113 | if !found { 114 | if pq != query { 115 | level.Debug(logger).Log("msg", "replaced query", "period_start", p.Start, "period_end", p.End, "period_complete", p.Complete, "query", query, "new_query", pq) 116 | } 117 | result, _, err = queryAPI.Query(context.Background(), pq, p.End) 118 | if err != nil { 119 | return nil, err 120 | } 121 | if p.Complete { 122 | c.Set(key, result) 123 | } 124 | } 125 | 126 | var metrics []prometheus.Metric 127 | metricLabels := make(prometheus.Labels, len(additionalLabels)) 128 | for k, v := range additionalLabels { 129 | metricLabels[k] = replaceForPeriod(p, v) 130 | } 131 | 132 | for _, vec := range result.(model.Vector) { 133 | labels := make(prometheus.Labels, len(vec.Metric)+len(metricLabels)) 134 | for k, v := range vec.Metric { 135 | if k == model.MetricNameLabel { 136 | continue 137 | } 138 | labels[string(k)] = string(v) 139 | } 140 | for k, v := range metricLabels { 141 | if k == model.MetricNameLabel { 142 | continue 143 | } 144 | labels[k] = v 145 | } 146 | 147 | sample := prometheus.MustNewConstMetric( 148 | prometheus.NewDesc(metricName, "", nil, labels), 149 | prometheus.UntypedValue, 150 | float64(vec.Value), 151 | ) 152 | 153 | metrics = append(metrics, sample) 154 | } 155 | 156 | return metrics, nil 157 | } 158 | 159 | func replaceForPeriod(p period.Period, v string) string { 160 | d := p.End.Sub(p.Start) 161 | replacements := map[string]string{ 162 | "RANGE": fmt.Sprintf("%.0fs", d.Seconds()), 163 | } 164 | for i := 0; i <= 100; i++ { 165 | replacements[strconv.Itoa(i)+"_PC_TIMESTAMP"] = fmt.Sprintf("%d", p.Start.Add(time.Duration(i/100)*d).Unix()) 166 | replacements[strconv.Itoa(i)+"_PC_RANGE"] = fmt.Sprintf("%.0fs", float64(i)*d.Seconds()/100) 167 | } 168 | 169 | return os.Expand(v, func(s string) string { 170 | if s == "$" { 171 | return "$" 172 | } 173 | if v, ok := replacements[s]; ok { 174 | return v 175 | } 176 | if strings.HasPrefix(s, "_") { 177 | return p.Start.Format(strings.TrimPrefix(s, "-")) 178 | } 179 | return p.End.Format(s) 180 | }) 181 | } 182 | -------------------------------------------------------------------------------- /cmd/oy-scrape-jitter/main.go: -------------------------------------------------------------------------------- 1 | // Copyright The o11y toolkit Authors 2 | // spdx-license-identifier: apache-2.0 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at: 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "image/color" 22 | "os" 23 | "time" 24 | 25 | "github.com/alecthomas/kingpin/v2" 26 | "github.com/go-kit/log" 27 | "github.com/go-kit/log/level" 28 | "github.com/prometheus/client_golang/api" 29 | apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" 30 | "github.com/prometheus/common/model" 31 | "gonum.org/v1/plot" 32 | "gonum.org/v1/plot/plotter" 33 | "gonum.org/v1/plot/vg" 34 | 35 | "github.com/o11ydev/oy-toolkit/util/client" 36 | "github.com/o11ydev/oy-toolkit/util/cmd" 37 | ) 38 | 39 | var ( 40 | metric = kingpin.Flag("metric", "Metric to use to determine jitter.").Default("up").String() 41 | qtime = kingpin.Flag("query.timestamp", "Timestamp of the query.").Int64() 42 | png = kingpin.Flag("plot.file", "Path to a file to write an image of the results.").PlaceHolder("file.png").String() 43 | plotLog = kingpin.Flag("plot.log-y", "Use logarithmic Y axis.").Bool() 44 | lookback = kingpin.Flag("lookback", "How much time to look in the past for scrapes.").Default("1h").Duration() 45 | divisor = kingpin.Flag("divisor", "Divisor to use to determine if a scrape is aligned.").Default("1s").Duration() 46 | unalignedOnly = kingpin.Flag("log.unaligned-only", "Only take unaligned targets in logging.").Bool() 47 | plotUnalignedOnly = kingpin.Flag("plot.unaligned-only", "Only take unaligned targets in plot.").Default("true").Bool() 48 | quiet = kingpin.Flag("log.results-only", "Only log final result.").Bool() 49 | ) 50 | 51 | func main() { 52 | c := client.InitCliFlags() 53 | logger := cmd.InitCmd("oy-scrape-jitter") 54 | 55 | promClient, err := client.NewClient(c) 56 | if err != nil { 57 | level.Error(logger).Log("msg", "Can't create Prometheus client", "err", err) 58 | os.Exit(1) 59 | } 60 | 61 | analyzeScrapeAlignment(logger, promClient) 62 | } 63 | 64 | func analyzeScrapeAlignment(logger log.Logger, promClient api.Client) { 65 | var plotValues plotter.Values 66 | 67 | tm := time.Now() 68 | if *qtime != 0 { 69 | tm = time.Unix(*qtime, 0) 70 | } 71 | 72 | api := apiv1.NewAPI(promClient) 73 | v, warnings, err := api.Query(context.Background(), fmt.Sprintf("%s[%dms]", *metric, lookback.Milliseconds()), tm) 74 | if err != nil { 75 | level.Error(logger).Log("msg", "Can't query up metrics", "err", err) 76 | os.Exit(1) 77 | } 78 | for w := range warnings { 79 | if err != nil { 80 | level.Warn(logger).Log("msg", w) 81 | } 82 | } 83 | 84 | if v.Type() != model.ValMatrix { 85 | if err != nil { 86 | level.Error(logger).Log("msg", "Wrong return type", "expected", model.ValMatrix, "got", v.Type()) 87 | os.Exit(1) 88 | } 89 | } 90 | 91 | result, _ := v.(model.Matrix) 92 | var goodTargets, badTargets int 93 | var maxTarget int64 94 | for _, r := range result { 95 | var good, bad float64 96 | var max int64 97 | var lastTs time.Time 98 | for _, s := range r.Values { 99 | if lastTs.IsZero() { 100 | lastTs = s.Timestamp.Time() 101 | continue 102 | } 103 | diff := s.Timestamp.Time().Sub(lastTs).Milliseconds() 104 | ok := diff%divisor.Milliseconds() == 0 105 | level.Debug(logger).Log("metric", r.Metric.String(), "prev", lastTs, "current", s.Timestamp.Time(), "difference", diff, "aligned", ok) 106 | if ok { 107 | good++ 108 | } else { 109 | bad++ 110 | if diff%divisor.Milliseconds() > divisor.Milliseconds()/2 { 111 | diff = divisor.Milliseconds() - diff%divisor.Milliseconds() 112 | } 113 | if diff > max { 114 | max = diff % divisor.Milliseconds() 115 | if max > maxTarget { 116 | maxTarget = max 117 | } 118 | } 119 | } 120 | if *png != "" { 121 | if !ok || !*plotUnalignedOnly { 122 | plotValues = append(plotValues, float64(diff%divisor.Milliseconds())) 123 | } 124 | } 125 | lastTs = s.Timestamp.Time() 126 | } 127 | 128 | if (bad != 0 || !*unalignedOnly) && !*quiet { 129 | level.Info(logger).Log("metric", r.Metric.String(), "aligned", good, "unaligned", bad, "max_ms", max, "pc", fmt.Sprintf("%.2f%%", 100*good/(bad+good))) 130 | } 131 | 132 | if bad == 0 { 133 | goodTargets++ 134 | } else { 135 | badTargets++ 136 | } 137 | } 138 | level.Info(logger).Log("aligned_targets", goodTargets, "unaligned_targets", badTargets, "max_ms", maxTarget) 139 | if *png != "" { 140 | err := makePlot(plotValues, *png) 141 | if err != nil { 142 | level.Error(logger).Log("msg", "Unable to plot data.", "err", err) 143 | } 144 | } 145 | } 146 | 147 | func makePlot(values plotter.Values, out string) error { 148 | p := plot.New() 149 | p.Title.Text = "Scrape timestamps jitter" 150 | p.X.Label.Text = "Timestamp difference (ms)" 151 | p.Y.Label.Text = "Scrapes" 152 | l := plot.NewLegend() 153 | l.Add("https://o11y.tools") 154 | l.Top = true 155 | l.TextStyle.Color = color.RGBA{0, 0, 0, 100} 156 | 157 | hist, err := plotter.NewHist(values, 25) 158 | if err != nil { 159 | return err 160 | } 161 | hist.FillColor = color.RGBA{255, 0, 74, 255} 162 | hist.LogY = *plotLog 163 | p.Add(hist) 164 | p.Legend = l 165 | 166 | if err := p.Save(4*vg.Inch, 3*vg.Inch, out); err != nil { 167 | return err 168 | } 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /cmd/oy-periodic-queries/README.md: -------------------------------------------------------------------------------- 1 | # oy-periodic-queries 2 | 3 | *oy-periodic-queries* is a tool that allows you to evaluate Prometheus 4 | recording rules and export the results as metrics, with defined boundaries such 5 | as monthly or weekly (*Only monthly is implemented at the moment*). 6 | 7 | It uses calendar months. 8 | 9 | With this exporter, you can easily calculate monthly resource usage, or any 10 | other metric that requires periodic evaluation. Simply specify the recording 11 | rule and time boundaries, and the exporter will run the rule and provide you 12 | with accurate, timely metrics. 13 | 14 | In the recording rule and labels, you have access to the following variables, that will be 15 | replaced: 16 | 17 | | Variable | Meaning | 18 | |----------|---------| 19 | | $RANGE | The range of the query, in duration | 20 | | $0_PC_TIMESTAMP | The start time of the query, in unix timestamp | 21 | | $x_PC_TIMESTAMP | The time located a x % of the range, in unix timestamp, x 22 | between 0 and 100 | 23 | | $100_PC_TIMESTAMP | The end time of the query, in unix timestamp | 24 | | $x_PC_RANGE | A duration that represents x % of the current range, x between 0 25 | and 100 | 26 | 27 | Other values in `${}` are formatted with [Go's 28 | Time.Format](https://pkg.go.dev/time#Time.Format), e.g. `"${2006-01}"` is turned 29 | into the month and day of the end of the range. Prepend `_` to have it relative 30 | to the start of the range: `"${_2006-01}"`. 31 | 32 | ## Configuration 33 | 34 | The syntax of a rule file is: 35 | 36 | ``` 37 | groups: 38 | [ -119 | 120 | Issues found 121 |122 |123 | The input can be parsed but there are linting issues: 124 |126 |%s
125 |