├── .bingo
    ├── .gitignore
    ├── README.md
    ├── Variables.mk
    ├── bingo.mod
    ├── bingo.sum
    ├── copyright.mod
    ├── copyright.sum
    ├── faillint.mod
    ├── faillint.sum
    ├── go.mod
    ├── goimports.mod
    ├── goimports.sum
    ├── golangci-lint.mod
    ├── golangci-lint.sum
    ├── mdox.mod
    ├── mdox.sum
    ├── misspell.mod
    ├── misspell.sum
    └── variables.env
├── .errcheck_excludes.txt
├── .github
    └── workflows
    │   └── go.yaml
├── .golangci.yml
├── .mdox.validate.yaml
├── CHANGELOG.md
├── COPYRIGHT
├── LICENSE
├── Makefile
├── README.md
├── cmd
    └── obslytics
    │   ├── export.go
    │   ├── export_bench_test.go
    │   ├── export_e2e_test.go
    │   └── main.go
├── go.mod
├── go.sum
├── pkg
    ├── dataframe
    │   ├── dataframe.go
    │   └── series.go
    ├── exporter
    │   ├── export.go
    │   ├── factory
    │   │   └── factory.go
    │   └── parquet
    │   │   └── parquet.go
    ├── series
    │   ├── factory
    │   │   └── factory.go
    │   ├── promread
    │   │   ├── promread.go
    │   │   └── promread_test.go
    │   ├── series.go
    │   └── storeapi
    │   │   ├── iter.go
    │   │   └── storeapi.go
    └── version
    │   └── version.go
└── scripts
    ├── build-check-comments.sh
    └── cleanup-white-noise.sh


/.bingo/.gitignore:
--------------------------------------------------------------------------------
 1 | 
 2 | # Ignore everything
 3 | *
 4 | 
 5 | # But not these files:
 6 | !.gitignore
 7 | !*.mod
 8 | !*.sum
 9 | !README.md
10 | !Variables.mk
11 | !variables.env
12 | 
13 | *tmp.mod
14 | 


--------------------------------------------------------------------------------
/.bingo/README.md:
--------------------------------------------------------------------------------
 1 | # Project Development Dependencies.
 2 | 
 3 | This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo.
 4 | 
 5 | * Run `bingo get` to install all tools having each own module file in this directory.
 6 | * Run `bingo get <tool>` to install <tool> that have own module file in this directory.
 7 | * For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $(<upper case tool name>) variable where <tool> is the .bingo/<tool>.mod.
 8 | * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool.
 9 | * For go: Import `.bingo/variables.go` to for variable names.
10 | * See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies.
11 | 
12 | ## Requirements
13 | 
14 | * Go 1.14+
15 | 


--------------------------------------------------------------------------------
/.bingo/Variables.mk:
--------------------------------------------------------------------------------
 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.7. DO NOT EDIT.
 2 | # All tools are designed to be build inside $GOBIN.
 3 | BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
 4 | GOPATH ?= $(shell go env GOPATH)
 5 | GOBIN  ?= $(firstword $(subst :, ,${GOPATH}))/bin
 6 | GO     ?= $(shell which go)
 7 | 
 8 | # Below generated variables ensure that every time a tool under each variable is invoked, the correct version
 9 | # will be used; reinstalling only if needed.
10 | # For example for bingo variable:
11 | #
12 | # In your main Makefile (for non array binaries):
13 | #
14 | #include .bingo/Variables.mk # Assuming -dir was set to .bingo .
15 | #
16 | #command: $(BINGO)
17 | #	@echo "Running bingo"
18 | #	@$(BINGO) <flags/args..>
19 | #
20 | BINGO := $(GOBIN)/bingo-v0.5.2
21 | $(BINGO): $(BINGO_DIR)/bingo.mod
22 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
23 | 	@echo "(re)installing $(GOBIN)/bingo-v0.5.2"
24 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=bingo.mod -o=$(GOBIN)/bingo-v0.5.2 "github.com/bwplotka/bingo"
25 | 
26 | COPYRIGHT := $(GOBIN)/copyright-v0.0.0-20220225185207-fe763185946b
27 | $(COPYRIGHT): $(BINGO_DIR)/copyright.mod
28 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
29 | 	@echo "(re)installing $(GOBIN)/copyright-v0.0.0-20220225185207-fe763185946b"
30 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=copyright.mod -o=$(GOBIN)/copyright-v0.0.0-20220225185207-fe763185946b "github.com/efficientgo/tools/copyright"
31 | 
32 | FAILLINT := $(GOBIN)/faillint-v1.8.0
33 | $(FAILLINT): $(BINGO_DIR)/faillint.mod
34 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
35 | 	@echo "(re)installing $(GOBIN)/faillint-v1.8.0"
36 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=faillint.mod -o=$(GOBIN)/faillint-v1.8.0 "github.com/fatih/faillint"
37 | 
38 | GOIMPORTS := $(GOBIN)/goimports-v0.0.0-20200526224456-8b020aee10d2
39 | $(GOIMPORTS): $(BINGO_DIR)/goimports.mod
40 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
41 | 	@echo "(re)installing $(GOBIN)/goimports-v0.0.0-20200526224456-8b020aee10d2"
42 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=goimports.mod -o=$(GOBIN)/goimports-v0.0.0-20200526224456-8b020aee10d2 "golang.org/x/tools/cmd/goimports"
43 | 
44 | GOLANGCI_LINT := $(GOBIN)/golangci-lint-v1.46.2
45 | $(GOLANGCI_LINT): $(BINGO_DIR)/golangci-lint.mod
46 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
47 | 	@echo "(re)installing $(GOBIN)/golangci-lint-v1.46.2"
48 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=golangci-lint.mod -o=$(GOBIN)/golangci-lint-v1.46.2 "github.com/golangci/golangci-lint/cmd/golangci-lint"
49 | 
50 | MDOX := $(GOBIN)/mdox-v0.9.0
51 | $(MDOX): $(BINGO_DIR)/mdox.mod
52 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
53 | 	@echo "(re)installing $(GOBIN)/mdox-v0.9.0"
54 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox"
55 | 
56 | MISSPELL := $(GOBIN)/misspell-v0.3.4
57 | $(MISSPELL): $(BINGO_DIR)/misspell.mod
58 | 	@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
59 | 	@echo "(re)installing $(GOBIN)/misspell-v0.3.4"
60 | 	@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=misspell.mod -o=$(GOBIN)/misspell-v0.3.4 "github.com/client9/misspell/cmd/misspell"
61 | 
62 | 


--------------------------------------------------------------------------------
/.bingo/bingo.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.14
4 | 
5 | require github.com/bwplotka/bingo v0.5.2
6 | 


--------------------------------------------------------------------------------
/.bingo/bingo.sum:
--------------------------------------------------------------------------------
 1 | github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
 2 | github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 3 | github.com/bwplotka/bingo v0.5.2 h1:iNCW7magHQK/ozLxoBVEhzhuFiftNQeFsfR9TuIEFxE=
 4 | github.com/bwplotka/bingo v0.5.2/go.mod h1:CNMrHaFo3AhgU86psqpMQ8BOac2SZMhYfd0On/Ubt64=
 5 | github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 8 | github.com/efficientgo/tools/core v0.0.0-20210201220623-8118984754c2 h1:GD19G/vhEa8amDJDBYcTaFXZjxKed67Ev0ZFPHdd/LQ=
 9 | github.com/efficientgo/tools/core v0.0.0-20210201220623-8118984754c2/go.mod h1:cFZoHUhKg31xkPnPjhPKFtevnx0Xcg67ptBRxbpaxtk=
10 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
11 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
12 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
13 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
14 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
15 | github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
16 | github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
17 | github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
18 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
19 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
21 | github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
22 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
23 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
24 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
25 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
26 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
27 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
28 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
29 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
30 | golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
31 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
32 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
33 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
34 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
35 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
36 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
37 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
38 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
39 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
40 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
41 | golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 h1:HlFl4V6pEMziuLXyRkm5BIYq1y1GAbb02pRlWvI54OM=
42 | golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
43 | golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407 h1:5zh5atpUEdIc478E/ebrIaHLKcfVvG6dL/fGv7BcMoM=
44 | golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
45 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
46 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
47 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
48 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
49 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
50 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
51 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
52 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
53 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
54 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
55 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
56 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
57 | mvdan.cc/editorconfig v0.1.1-0.20200121172147-e40951bde157/go.mod h1:Ge4atmRUYqueGppvJ7JNrtqpqokoJEFxYbP0Z+WeKS8=
58 | mvdan.cc/sh/v3 v3.2.4 h1:+fZaWcXWRjYAvqzEKoDhDM3DkxdDUykU2iw0VMKFe9s=
59 | mvdan.cc/sh/v3 v3.2.4/go.mod h1:fPQmabBpREM/XQ9YXSU5ZFZ/Sm+PmKP9/vkFHgYKJEI=
60 | 


--------------------------------------------------------------------------------
/.bingo/copyright.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.18
4 | 
5 | require github.com/efficientgo/tools/copyright v0.0.0-20220225185207-fe763185946b
6 | 


--------------------------------------------------------------------------------
/.bingo/copyright.sum:
--------------------------------------------------------------------------------
  1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
  4 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
  5 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 h1:EBTWhcAX7rNQ80RLwLCpHZBBrJuzallFHnF+yMXo928=
  6 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
  7 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
  8 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
  9 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 10 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 11 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 14 | github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79 h1:Pi2rMrMQeQ45UAZr8Ple+eSk9WCyglYpkFh0r22E00c=
 15 | github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79/go.mod h1:jQUsxCcf91LHRhOnGqrx/yrleJbosynzf29/UlCbzlk=
 16 | github.com/efficientgo/tools/copyright v0.0.0-20220225185207-fe763185946b h1:kq6nPaihA+UbbHZ/McVzH9Uti7eKPFCQYXCfImcwo7g=
 17 | github.com/efficientgo/tools/copyright v0.0.0-20220225185207-fe763185946b/go.mod h1:5J0wuuxLMX06WeEgnpf+SvTCptlR9+RHRNO/WEMAwSw=
 18 | github.com/efficientgo/tools/core v0.0.0-20210106193344-1108f4e7d16b h1:yi5z8x/FKDHrqtEFiAsxF5b7Sz2+CJrRwBC2kbyhVcA=
 19 | github.com/efficientgo/tools/core v0.0.0-20210106193344-1108f4e7d16b/go.mod h1:RJm2+KCRfMUwgEgRte3obd5uLdVY5YbDZjgSMPY0HSA=
 20 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 21 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 22 | github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE=
 23 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 24 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 25 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 26 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 27 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 28 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 29 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 30 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 31 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 32 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 33 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 34 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 35 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 36 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 37 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 38 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 39 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 40 | github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 41 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 42 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 43 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 44 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 45 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 46 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 47 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 48 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 49 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 50 | github.com/protoconfig/protoconfig/go v0.0.0-20210106192113-733758adefac h1:PWrv6uwNBua14NbS74ukVgXgdRDQPx/2B+Rf6KXXoQk=
 51 | github.com/protoconfig/protoconfig/go v0.0.0-20210106192113-733758adefac/go.mod h1:ig8lL2CeTS14ijDIIRoi6pTap0BHc0Xrnke+SKmn9QM=
 52 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 53 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 54 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 55 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 56 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 57 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 58 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 59 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 60 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 61 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 62 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 63 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 64 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 65 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 66 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 67 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 68 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 69 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 70 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 71 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 72 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 73 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 74 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 75 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 76 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 77 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 78 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 79 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 80 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 81 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 82 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 83 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 84 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 85 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 86 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 87 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 88 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 89 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 90 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 91 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 92 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 93 | golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 94 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 95 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 96 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 97 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 98 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 99 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
100 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
101 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
102 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
103 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
104 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
105 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
106 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
107 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
108 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
109 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
110 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
111 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
112 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
113 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
114 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
115 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
116 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
117 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
118 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
119 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
120 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
121 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
122 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
123 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
124 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
125 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
126 | 


--------------------------------------------------------------------------------
/.bingo/faillint.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.14
4 | 
5 | require github.com/fatih/faillint v1.8.0
6 | 


--------------------------------------------------------------------------------
/.bingo/faillint.sum:
--------------------------------------------------------------------------------
 1 | dmitri.shuralyov.com/go/generated v0.0.0-20170818220700-b1254a446363 h1:o4lAkfETerCnr1kF9/qwkwjICnU+YLHNDCM8h2xj7as=
 2 | dmitri.shuralyov.com/go/generated v0.0.0-20170818220700-b1254a446363/go.mod h1:WG7q7swWsS2f9PYpt5DoEP/EBYWx8We5UoRltn9vJl8=
 3 | github.com/fatih/faillint v1.8.0 h1:wV/mhyU+FcDtXx4RByy+H2FjrwHU9hEiFMyWPYmKqPE=
 4 | github.com/fatih/faillint v1.8.0/go.mod h1:Yu1OT32SIjnX4Kn/h4/YPQOuNfuITtL3Gps70ac4lQk=
 5 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 6 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 7 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 8 | golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
 9 | golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
10 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
11 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
12 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
13 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
14 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
15 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
16 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
17 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
18 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
20 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
21 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
22 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
23 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
24 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
25 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
26 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
27 | golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
28 | golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
29 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
30 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
31 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
32 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
33 | 


--------------------------------------------------------------------------------
/.bingo/go.mod:
--------------------------------------------------------------------------------
1 | module _ // Fake go.mod auto-created by 'bingo' for go -moddir compatibility with non-Go projects. Commit this file, together with other .mod files.


--------------------------------------------------------------------------------
/.bingo/goimports.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.14
4 | 
5 | require golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2 // cmd/goimports
6 | 


--------------------------------------------------------------------------------
/.bingo/goimports.sum:
--------------------------------------------------------------------------------
 1 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 2 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 3 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 4 | golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 5 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 6 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 7 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 8 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 9 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
10 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
11 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
12 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
13 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
14 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
15 | golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2 h1:21BqcH/onxtGHn1A2GDOJjZnbt4Nlez629S3eaR+eYs=
16 | golang.org/x/tools v0.0.0-20200526224456-8b020aee10d2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
17 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
18 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
19 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
20 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
21 | 


--------------------------------------------------------------------------------
/.bingo/golangci-lint.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.14
4 | 
5 | require github.com/golangci/golangci-lint v1.46.2 // cmd/golangci-lint
6 | 


--------------------------------------------------------------------------------
/.bingo/mdox.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.18
4 | 
5 | require github.com/bwplotka/mdox v0.9.0
6 | 


--------------------------------------------------------------------------------
/.bingo/misspell.mod:
--------------------------------------------------------------------------------
1 | module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2 | 
3 | go 1.19
4 | 
5 | require github.com/client9/misspell v0.3.4 // cmd/misspell
6 | 


--------------------------------------------------------------------------------
/.bingo/misspell.sum:
--------------------------------------------------------------------------------
1 | github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
2 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
3 | 


--------------------------------------------------------------------------------
/.bingo/variables.env:
--------------------------------------------------------------------------------
 1 | # Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.7. DO NOT EDIT.
 2 | # All tools are designed to be build inside $GOBIN.
 3 | # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk.
 4 | GOBIN=${GOBIN:=$(go env GOBIN)}
 5 | 
 6 | if [ -z "$GOBIN" ]; then
 7 | 	GOBIN="$(go env GOPATH)/bin"
 8 | fi
 9 | 
10 | 
11 | BINGO="${GOBIN}/bingo-v0.5.2"
12 | 
13 | COPYRIGHT="${GOBIN}/copyright-v0.0.0-20220225185207-fe763185946b"
14 | 
15 | FAILLINT="${GOBIN}/faillint-v1.8.0"
16 | 
17 | GOIMPORTS="${GOBIN}/goimports-v0.0.0-20200526224456-8b020aee10d2"
18 | 
19 | GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.46.2"
20 | 
21 | MDOX="${GOBIN}/mdox-v0.9.0"
22 | 
23 | MISSPELL="${GOBIN}/misspell-v0.3.4"
24 | 
25 | 


--------------------------------------------------------------------------------
/.errcheck_excludes.txt:
--------------------------------------------------------------------------------
1 | (github.com/go-kit/log.Logger).Log
2 | fmt.Fprintln
3 | fmt.Fprint
4 | 


--------------------------------------------------------------------------------
/.github/workflows/go.yaml:
--------------------------------------------------------------------------------
 1 | name: go
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 |     tags:
 8 |   pull_request:
 9 | 
10 | jobs:
11 |   lint:
12 |     runs-on: ubuntu-latest
13 |     name: Linters (Static Analysis) for Go
14 |     steps:
15 |       - name: Checkout code into the Go module directory.
16 |         uses: actions/checkout@v3
17 | 
18 |       - name: Install Go
19 |         uses: actions/setup-go@v3
20 |         with:
21 |           go-version: 1.19.x
22 | 
23 |       - uses: actions/cache@v3
24 |         with:
25 |           path: ~/go/pkg/mod
26 |           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
27 | 
28 |       - name: Linting & vetting.
29 |         env:
30 |           GOBIN: /tmp/.bin
31 |         run: make lint
32 |   tests:
33 |     runs-on: ${{ matrix.platform }}
34 |     strategy:
35 |       fail-fast: false
36 |       matrix:
37 |         go: [ '1.19.x']
38 |         ## TODO(bwplotka): Add macos-latest once we can figure out how to add docker reliably there.
39 |         platform: [ubuntu-latest]
40 | 
41 |     name: Unit tests on Go ${{ matrix.go }} ${{ matrix.platform }}
42 |     steps:
43 |       - name: Checkout code into the Go module directory.
44 |         uses: actions/checkout@v3
45 | 
46 |       - name: Install Docker
47 |         uses: docker-practice/actions-setup-docker@0.0.1
48 | 
49 |       - name: Install Go
50 |         uses: actions/setup-go@v3
51 |         with:
52 |           go-version: ${{ matrix.go }}
53 | 
54 |       - uses: actions/cache@v3
55 |         with:
56 |           path: ~/go/pkg/mod
57 |           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
58 | 
59 |       - name: Run unit tests.
60 |         env:
61 |           GOBIN: /tmp/.bin
62 |         run: make test


--------------------------------------------------------------------------------
/.golangci.yml:
--------------------------------------------------------------------------------
 1 | # This file contains all available configuration options
 2 | # with their default values.
 3 | 
 4 | # options for analysis running
 5 | run:
 6 |   # timeout for analysis, e.g. 30s, 5m, default is 1m
 7 |   deadline: 5m
 8 | 
 9 |   # exit code when at least one issue was found, default is 1
10 |   issues-exit-code: 1
11 | 
12 |   # which dirs to skip: they won't be analyzed;
13 |   # can use regexp here: generated.*, regexp is applied on full path;
14 |   # default value is empty list, but next dirs are always skipped independently
15 |   # from this option's value:
16 |   #     vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
17 |   skip-dirs:
18 |     - vendor
19 | 
20 | # output configuration options
21 | output:
22 |   # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
23 |   format: colored-line-number
24 | 
25 |   # print lines of code with issue, default is true
26 |   print-issued-lines: true
27 | 
28 |   # print linter name in the end of issue text, default is true
29 |   print-linter-name: true
30 | 
31 | linters:
32 |   enable:
33 |     # Sorted alphabetically.
34 |     - deadcode
35 |     - errcheck
36 |     - goconst
37 |     - godot
38 |     - gofmt
39 |     - goimports
40 |     - gosimple
41 |     - govet
42 |     - ineffassign
43 |     - misspell
44 |     - staticcheck
45 |     - structcheck
46 |     - typecheck
47 |     - unparam
48 |     - unused
49 |     - varcheck
50 |     - exportloopref
51 | 
52 | linters-settings:
53 |   errcheck:
54 |     exclude: ./.errcheck_excludes.txt
55 |   misspell:
56 |     locale: US
57 |   goconst:
58 |     min-occurrences: 5
59 |   goimports:
60 |     local-prefixes: github.com/thanos-community/obslytics
61 | 
62 | 


--------------------------------------------------------------------------------
/.mdox.validate.yaml:
--------------------------------------------------------------------------------
 1 | version: 1
 2 | 
 3 | validators:
 4 |   # Validators to skip checking PR/issue links of Thanos, Prometheus and Cortex.
 5 |   - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)thanos-io\/thanos(\/pull\/|\/issues\/)'
 6 |     type: 'githubPullsIssues'
 7 |   - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)prometheus\/prometheus(\/pull\/|\/issues\/)'
 8 |     type: 'githubPullsIssues'
 9 |   - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)cortexproject\/cortex(\/pull\/|\/issues\/)'
10 |     type: 'githubPullsIssues'
11 |   # Ignore Thanos release links.
12 |   - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)thanos-io\/thanos(\/releases\/)'
13 |     type: 'ignore'
14 |   # Causes http stream errors with statuscode 0 sometimes. But is safe to skip.
15 |   - regex: 'slack\.cncf\.io'
16 |     type: 'ignore'
17 |   # 301 errors even when curl-ed.
18 |   - regex: 'envoyproxy\.io'
19 |     type: 'ignore'
20 |   # couldn't reach even when curl-ed.
21 |   - regex: 'cloud\.baidu\.com'
22 |     type: 'ignore'
23 |   # 403 when curl-ed from GitHub actions, though not from a developer machine. Likely due to secondary rate limits.
24 |   - regex: 'docs\.github\.com'
25 |     type: 'ignore'


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | All notable changes to this project will be documented in this file.
 4 | 
 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 6 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 7 | 
 8 | NOTE: As semantic versioning states all 0.y.z releases can contain breaking changes in API (flags, grpc API, any backward compatibility)
 9 | 
10 | We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.)
11 | 
12 | ## Unreleased
13 | 


--------------------------------------------------------------------------------
/COPYRIGHT:
--------------------------------------------------------------------------------
1 | Copyright (c) The Thanos Community Authors.
2 | Licensed under the Apache License 2.0.


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
  1 | include .bingo/Variables.mk
  2 | FILES_TO_FMT      	 ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
  3 | MDOX_VALIDATE_CONFIG ?= .mdox.validate.yaml
  4 | 
  5 | GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin
  6 | 
  7 | # Tools.
  8 | GIT ?= $(shell which git)
  9 | 
 10 | # Support gsed on OSX (installed via brew), falling back to sed. On Linux
 11 | # systems gsed won't be installed, so will use sed as expected.
 12 | SED ?= $(shell which gsed 2>/dev/null || which sed)
 13 | 
 14 | define require_clean_work_tree
 15 | 	@git update-index -q --ignore-submodules --refresh
 16 | 
 17 | 	@if ! git diff-files --quiet --ignore-submodules --; then \
 18 | 		echo >&2 "cannot $1: you have unstaged changes."; \
 19 | 		git diff-files --name-status -r --ignore-submodules -- >&2; \
 20 | 		echo >&2 "Please commit or stash them."; \
 21 | 		exit 1; \
 22 | 	fi
 23 | 
 24 | 	@if ! git diff-index --cached --quiet HEAD --ignore-submodules --; then \
 25 | 		echo >&2 "cannot $1: your index contains uncommitted changes."; \
 26 | 		git diff-index --cached --name-status -r --ignore-submodules HEAD -- >&2; \
 27 | 		echo >&2 "Please commit or stash them."; \
 28 | 		exit 1; \
 29 | 	fi
 30 | endef
 31 | 
 32 | help: ## Displays help.
 33 | 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 34 | 
 35 | .PHONY: all
 36 | all: format build
 37 | 
 38 | .PHONY: build
 39 | build: ## Build obsyltics.
 40 | 	@echo ">> Building obslytics"
 41 | 	@go install github.com/thanos-community/obslytics/cmd/obslytics
 42 | 
 43 | .PHONY: check-comments
 44 | check-comments: ## Checks Go code comments if they have trailing period (excludes protobuffers and vendor files). Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
 45 | 	@echo ">> checking Go comments trailing periods\n\n\n"
 46 | 	@./scripts/build-check-comments.sh
 47 | 
 48 | .PHONY: deps
 49 | deps: ## Ensures fresh go.mod and go.sum.
 50 | 	@go mod tidy
 51 | 	@go mod verify
 52 | 
 53 | .PHONY: docs
 54 | docs: ## Generates docs for all thanos commands, localise links, ensure GitHub format.
 55 | docs: $(MDOX)
 56 | 	@echo ">> generating docs"
 57 | 	PATH="${PATH}:$(GOBIN)" $(MDOX) fmt README.md
 58 | 	$(MAKE) white-noise-cleanup
 59 | 
 60 | .PHONY: check-docs
 61 | check-docs: ## Checks docs against discrepancy with flags, links, white noise.
 62 | check-docs: $(MDOX)
 63 | 	@echo ">> checking docs"
 64 | 	PATH="${PATH}:$(GOBIN)" $(MDOX) fmt -l --links.validate.config-file=$(MDOX_VALIDATE_CONFIG) README.md
 65 | 	$(MAKE) white-noise-cleanup
 66 | 	$(call require_clean_work_tree,'run make docs and commit changes')
 67 | 
 68 | .PHONY: format
 69 | format: ## Formats Go code.
 70 | format: $(GOIMPORTS)
 71 | 	@echo ">> formatting code"
 72 | 	@$(GOIMPORTS) -w -local github.com/thanos-community/obslytics $(FILES_TO_FMT)
 73 | 
 74 | .PHONY: test
 75 | test: ## Runs all Go unit tests.
 76 | export GOCACHE=/tmp/cache
 77 | test:
 78 | 	@echo ">> running unit tests (without cache)"
 79 | 	@rm -rf $(GOCACHE)
 80 | 	@go test -v -timeout=30m $(shell go list ./... | grep -v /vendor/);
 81 | 
 82 | 
 83 | .PHONY: check-git
 84 | check-git:
 85 | ifneq ($(GIT),)
 86 | 	@test -x $(GIT) || (echo >&2 "No git executable binary found at $(GIT)."; exit 1)
 87 | else
 88 | 	@echo >&2 "No git binary found."; exit 1
 89 | endif
 90 | 
 91 | # PROTIP:
 92 | # Add
 93 | #      --cpu-profile-path string   Path to CPU profile output file
 94 | #      --mem-profile-path string   Path to memory profile output file
 95 | # to debug big allocations during linting.
 96 | .PHONY: lint
 97 | lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) $(COPYRIGHT) build format docs check-git deps
 98 | 	$(call require_clean_work_tree,'detected not clean work tree before running lint, previous job changed something?')
 99 | 	@echo ">> verifying modules being imported"
100 | 	@# TODO(bwplotka): Add, Printf, DefaultRegisterer, NewGaugeFunc and MustRegister once exception are accepted.
101 | 	@$(FAILLINT) -paths "errors=github.com/efficientgo/core/errors,\
102 | fmt.{Errorf}=github.com/efficientgo/core/errors.{Wrap,Wrapf},\
103 | github.com/prometheus/prometheus/pkg/testutils=github.com/efficientgo/core/testutil,\
104 | github.com/stretchr/testify=github.com/efficientgo/core/testutil" ./...
105 | 	@$(FAILLINT) -paths "fmt.{Print,Println,Sprint,Errorf}" -ignore-tests ./...
106 | 	@echo ">> linting all of the Go files GOGC=${GOGC}"
107 | 	@$(GOLANGCI_LINT) run
108 | 	@echo ">> ensuring Copyright headers"
109 | 	@$(COPYRIGHT) $(shell go list -f "{{.Dir}}" ./... | xargs -i find "{}" -name "*.go")
110 | 	@echo ">> detecting misspells"
111 | 	@find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error
112 | 	$(call require_clean_work_tree, run make lint and commit changes')
113 | 
114 | .PHONY: white-noise-cleanup
115 | white-noise-cleanup: ## Cleans up white noise in docs.
116 | white-noise-cleanup:
117 | 	@echo ">> cleaning up white noise"
118 | 	@find . -type f \( -name "*.md" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Obslytics (Observability Analytics)
 2 | 
 3 | [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/thanos-community/obslytics) [![Latest Release](https://img.shields.io/github/release/thanos-community/obslytics.svg?style=flat-square)](https://github.com/thanos-community/obslytics/releases/latest) [![CI](https://github.com/thanos-community/obslytics/workflows/go/badge.svg)](https://github.com/thanos-community/obslytics/actions?query=workflow%3Ago) [![Go Report Card](https://goreportcard.com/badge/github.com/thanos-community/obslytics)](https://goreportcard.com/report/github.com/thanos-community/obslytics) [![Slack](https://img.shields.io/badge/join%20slack-%23analytics-brightgreen.svg)](https://slack.cncf.io/)
 4 | 
 5 | ## Usage
 6 | 
 7 | ```bash mdox-exec="obslytics --help"
 8 | usage: obslytics [<flags>] <command> [<args> ...]
 9 | 
10 | Integrate Observability data into your Analytics pipelines
11 | 
12 | Flags:
13 |   -h, --help               Show context-sensitive help (also try --help-long and
14 |                            --help-man).
15 |       --version            Show application version.
16 |       --log.level=info     Log filtering level.
17 |       --log.format=logfmt  Log format to use.
18 | 
19 | Commands:
20 |   help [<command>...]
21 |     Show help.
22 | 
23 |   export --match=MATCH --min-time=MIN-TIME --max-time=MAX-TIME --resolution=RESOLUTION [<flags>]
24 |     Export observability series data into popular analytics formats.
25 | 
26 | 
27 | ```
28 | 
29 | ## Contributing
30 | 
31 | Any contributions are welcome! Just use GitHub Issues and Pull Requests as usual. We follow [Thanos Go coding style](https://thanos.io/contributing/coding-style-guide.md/) guide.
32 | 
33 | ## Initial Author
34 | 
35 | [@bwplotka](https://bwplotka.dev)
36 | 


--------------------------------------------------------------------------------
/cmd/obslytics/export.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package main
  5 | 
  6 | import (
  7 | 	"context"
  8 | 	"fmt"
  9 | 	"os"
 10 | 	"time"
 11 | 
 12 | 	"github.com/efficientgo/core/errors"
 13 | 	"github.com/go-kit/log"
 14 | 	"github.com/oklog/run"
 15 | 	"github.com/prometheus/prometheus/model/timestamp"
 16 | 	"github.com/prometheus/prometheus/promql/parser"
 17 | 	"github.com/thanos-io/objstore/client"
 18 | 	"github.com/thanos-io/thanos/pkg/model"
 19 | 	"gopkg.in/yaml.v2"
 20 | 
 21 | 	"github.com/thanos-community/obslytics/pkg/dataframe"
 22 | 	"github.com/thanos-community/obslytics/pkg/exporter"
 23 | 	"github.com/thanos-community/obslytics/pkg/series"
 24 | 
 25 | 	extflag "github.com/efficientgo/tools/extkingpin"
 26 | 	"gopkg.in/alecthomas/kingpin.v2"
 27 | 
 28 | 	exportertfactory "github.com/thanos-community/obslytics/pkg/exporter/factory"
 29 | 	infactory "github.com/thanos-community/obslytics/pkg/series/factory"
 30 | )
 31 | 
 32 | func registerExport(m map[string]setupFunc, app *kingpin.Application) {
 33 | 	cmd := app.Command("export", "Export observability series data into popular analytics formats.")
 34 | 	inputFlag := extflag.RegisterPathOrContent(cmd, "input-config", "YAML for input, series configuration.", extflag.WithRequired())
 35 | 	outputFlag := extflag.RegisterPathOrContent(cmd, "output-config", "YAML for dataframe export configuration.")
 36 | 
 37 | 	// TODO(bwplotka): Describe more how the format looks like.
 38 | 	matchersStr := cmd.Flag("match", "Metric matcher for metrics to export (e.g up{a=\"1\"}").Required().String()
 39 | 	timeFmt := time.RFC3339
 40 | 
 41 | 	var mint, maxt model.TimeOrDurationValue
 42 | 	cmd.Flag("min-time", fmt.Sprintf("The lower boundary of the time series in %s or duration format", timeFmt)).
 43 | 		Required().SetValue(&mint)
 44 | 
 45 | 	cmd.Flag("max-time", fmt.Sprintf("The upper boundary of the time series in %s or duration format", timeFmt)).
 46 | 		Required().SetValue(&maxt)
 47 | 
 48 | 	resolution := cmd.Flag("resolution", "Sample resolution (e.g. 30m)").Required().Duration()
 49 | 	dbgOut := cmd.Flag("debug", "Show additional debug info (such as produced table)").Bool()
 50 | 
 51 | 	m["export"] = func(g *run.Group, logger log.Logger) error {
 52 | 		ctx, cancel := context.WithCancel(context.Background())
 53 | 		g.Add(func() error {
 54 | 			inputCfg, err := inputFlag.Content()
 55 | 			if err != nil {
 56 | 				return err
 57 | 			}
 58 | 
 59 | 			inputConfig := series.Config{}
 60 | 			if err := yaml.UnmarshalStrict(inputCfg, &inputConfig); err != nil {
 61 | 				return err
 62 | 			}
 63 | 
 64 | 			outputCfg, err := outputFlag.Content()
 65 | 			if err != nil {
 66 | 				return err
 67 | 			}
 68 | 
 69 | 			outputConfig := exporter.Config{
 70 | 				// Default Storage Type is Filesystem.
 71 | 				Storage: client.BucketConfig{Type: client.FILESYSTEM},
 72 | 			}
 73 | 			if err := yaml.UnmarshalStrict(outputCfg, &outputConfig); err != nil {
 74 | 				return err
 75 | 			}
 76 | 
 77 | 			return export(ctx, logger, *matchersStr, inputConfig, outputConfig, mint, maxt, *resolution, *dbgOut)
 78 | 		}, func(error) { cancel() })
 79 | 		return nil
 80 | 	}
 81 | }
 82 | 
 83 | func export(
 84 | 	ctx context.Context,
 85 | 	logger log.Logger,
 86 | 	matchersStr string,
 87 | 	inputConfig series.Config,
 88 | 	outputCfg exporter.Config,
 89 | 	mint, maxt model.TimeOrDurationValue,
 90 | 	resolution time.Duration,
 91 | 	printDebug bool,
 92 | ) error {
 93 | 	matchers, err := parser.ParseMetricSelector(matchersStr)
 94 | 	if err != nil {
 95 | 		return errors.Wrap(err, "parsing provided matchers")
 96 | 	}
 97 | 
 98 | 	in, err := infactory.NewSeriesReader(logger, inputConfig)
 99 | 	if err != nil {
100 | 		return err
101 | 	}
102 | 
103 | 	exp, err := exportertfactory.NewExporter(logger, outputCfg)
104 | 	if err != nil {
105 | 		return err
106 | 	}
107 | 
108 | 	ser, err := in.Read(ctx, series.Params{
109 | 		Matchers: matchers,
110 | 		MinTime:  timestamp.Time(mint.PrometheusTimestamp()),
111 | 		MaxTime:  timestamp.Time(maxt.PrometheusTimestamp()),
112 | 	})
113 | 	if err != nil {
114 | 		return err
115 | 	}
116 | 
117 | 	df, err := dataframe.FromSeries(ser, resolution, func(o *dataframe.AggrsOptions) {
118 | 		// TODO(inecas): Expose the enabled aggregations via flag.
119 | 		o.Count.Enabled = true
120 | 		o.Sum.Enabled = true
121 | 		o.Min.Enabled = true
122 | 		o.Max.Enabled = true
123 | 	})
124 | 	if err != nil {
125 | 		return errors.Wrap(err, "dataframe creation")
126 | 	}
127 | 
128 | 	if printDebug {
129 | 		dataframe.Print(os.Stdout, df)
130 | 	}
131 | 
132 | 	if err := exp.Export(ctx, df); err != nil {
133 | 		return errors.Wrapf(err, "export dataframe")
134 | 	}
135 | 	return nil
136 | }
137 | 


--------------------------------------------------------------------------------
/cmd/obslytics/export_bench_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package main
  5 | 
  6 | import (
  7 | 	"context"
  8 | 	"fmt"
  9 | 	"net"
 10 | 	"os"
 11 | 	"path/filepath"
 12 | 	"sync"
 13 | 	"testing"
 14 | 	"time"
 15 | 
 16 | 	"github.com/efficientgo/core/testutil"
 17 | 	"github.com/go-kit/log"
 18 | 	"github.com/prometheus/prometheus/model/labels"
 19 | 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 20 | 	"github.com/thanos-io/objstore/client"
 21 | 	"github.com/thanos-io/objstore/providers/filesystem"
 22 | 	"github.com/thanos-io/thanos/pkg/model"
 23 | 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 24 | 	"github.com/thanos-io/thanos/pkg/store/storepb"
 25 | 	"google.golang.org/grpc"
 26 | 
 27 | 	"github.com/thanos-community/obslytics/pkg/exporter"
 28 | 	"github.com/thanos-community/obslytics/pkg/series"
 29 | )
 30 | 
 31 | func BenchmarkExport(b *testing.B) {
 32 | 	// Generate large enough input case.
 33 | 	b.Run("10000 series with 100 labels with one sample", func(b *testing.B) {
 34 | 		b.Skip("FOR NOW")
 35 | 		resps := make([]*storepb.SeriesResponse, 0, 10000)
 36 | 		for ser := 0; ser < cap(resps); ser++ {
 37 | 			lset := make(labels.Labels, 0, 100)
 38 | 			for lab := 0; lab < cap(lset); lab++ {
 39 | 				lset = append(lset, labels.Label{Name: fmt.Sprintf("label-%v", lab), Value: "something"})
 40 | 			}
 41 | 			resps = append(resps, storeSeriesResponse(b, append(lset, labels.Label{Name: "cluster", Value: fmt.Sprintf("value-%v", ser)}), []sample{{t: 0, v: 0}}))
 42 | 		}
 43 | 		benchExport(b, resps)
 44 | 	})
 45 | 	b.Run("1000 series with 100 labels with 1000 samples", func(b *testing.B) {
 46 | 		resps := make([]*storepb.SeriesResponse, 0, 1000)
 47 | 		for ser := 0; ser < cap(resps); ser++ {
 48 | 			lset := make(labels.Labels, 0, 100)
 49 | 			for lab := 0; lab < cap(lset); lab++ {
 50 | 				lset = append(lset, labels.Label{Name: fmt.Sprintf("label-%v", lab), Value: "something"})
 51 | 			}
 52 | 
 53 | 			samples := make([]sample, 0, 1000)
 54 | 			for s := 0; s < cap(samples); s++ {
 55 | 				samples = append(samples, sample{t: int64(s), v: float64(s)})
 56 | 			}
 57 | 			resps = append(resps, storeSeriesResponse(b, append(lset, labels.Label{Name: "cluster", Value: fmt.Sprintf("value-%v", ser)}), samples))
 58 | 		}
 59 | 		benchExport(b, resps)
 60 | 	})
 61 | 	b.Run("1 series with 100 labels with millions sample", func(b *testing.B) {
 62 | 		b.Skip("FOR NOW")
 63 | 		resps := make([]*storepb.SeriesResponse, 0, 1)
 64 | 		lset := make(labels.Labels, 0, 100)
 65 | 		for lab := 0; lab < cap(lset); lab++ {
 66 | 			lset = append(lset, labels.Label{Name: fmt.Sprintf("label-%v", lab), Value: "something"})
 67 | 		}
 68 | 
 69 | 		samples := make([]sample, 0, 10e6)
 70 | 		for s := 0; s < cap(samples); s++ {
 71 | 			samples = append(samples, sample{t: int64(s), v: float64(s)})
 72 | 		}
 73 | 		resps = append(resps, storeSeriesResponse(b, append(lset, labels.Label{Name: "cluster", Value: "value-0"}), samples))
 74 | 		benchExport(b, resps)
 75 | 	})
 76 | 
 77 | }
 78 | 
 79 | func benchExport(b *testing.B, resp []*storepb.SeriesResponse) {
 80 | 	tmpDir, err := os.MkdirTemp("", "export-bench")
 81 | 	testutil.Ok(b, err)
 82 | 	defer testutil.Ok(b, os.RemoveAll(tmpDir))
 83 | 
 84 | 	// Create local grpc service.
 85 | 	srv := grpc.NewServer()
 86 | 	testServer := &testThanosSeriesServer{resps: resp}
 87 | 	storepb.RegisterStoreServer(srv, testServer)
 88 | 
 89 | 	list, err := net.Listen("tcp", "localhost:0")
 90 | 	testutil.Ok(b, err)
 91 | 	wg := sync.WaitGroup{}
 92 | 	wg.Add(1)
 93 | 	go func() {
 94 | 		_ = srv.Serve(list)
 95 | 		wg.Done()
 96 | 	}()
 97 | 
 98 | 	var (
 99 | 		ctx      = context.Background()
100 | 		logger   = log.NewNopLogger()
101 | 		matchers = "{something=\"doesnotmatter\"}"
102 | 	)
103 | 
104 | 	b.ReportAllocs()
105 | 	b.ResetTimer()
106 | 	for i := 0; i < b.N; i++ {
107 | 		testutil.Ok(b, export(
108 | 			ctx,
109 | 			logger,
110 | 			matchers,
111 | 			series.Config{
112 | 				Type:     series.STOREAPI,
113 | 				Endpoint: list.Addr().String(),
114 | 			}, exporter.Config{
115 | 				Type: exporter.PARQUET,
116 | 				Storage: client.BucketConfig{
117 | 					Type: client.FILESYSTEM,
118 | 					Config: filesystem.Config{
119 | 						Directory: filepath.Join(tmpDir, fmt.Sprintf("%v", i)),
120 | 					},
121 | 				},
122 | 			},
123 | 			model.TimeOrDurationValue{},
124 | 			model.TimeOrDurationValue{},
125 | 			5*time.Minute,
126 | 			false,
127 | 		))
128 | 	}
129 | 
130 | 	srv.GracefulStop()
131 | 	srv.Stop()
132 | 	wg.Wait()
133 | }
134 | 
135 | type sample struct {
136 | 	t int64
137 | 	v float64
138 | }
139 | 
140 | type testThanosSeriesServer struct {
141 | 	// This field just exist to pseudo-implement the unused methods of the interface.
142 | 	storepb.StoreServer
143 | 
144 | 	resps []*storepb.SeriesResponse
145 | }
146 | 
147 | func (s *testThanosSeriesServer) Series(_ *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error {
148 | 	for _, resp := range s.resps {
149 | 		if err := srv.Send(resp); err != nil {
150 | 			return err
151 | 		}
152 | 	}
153 | 	return nil
154 | }
155 | 
156 | // storeSeriesResponse creates test storepb.SeriesResponse that includes series with single chunk that stores all the given samples.
157 | func storeSeriesResponse(t testing.TB, lset labels.Labels, smplChunks ...[]sample) *storepb.SeriesResponse {
158 | 	var s storepb.Series
159 | 
160 | 	for _, l := range lset {
161 | 		s.Labels = append(s.Labels, labelpb.ZLabel{Name: l.Name, Value: l.Value})
162 | 	}
163 | 
164 | 	for _, smpls := range smplChunks {
165 | 		c := chunkenc.NewXORChunk()
166 | 		a, err := c.Appender()
167 | 		testutil.Ok(t, err)
168 | 
169 | 		for _, smpl := range smpls {
170 | 			a.Append(smpl.t, smpl.v)
171 | 		}
172 | 
173 | 		ch := storepb.AggrChunk{
174 | 			MinTime: smpls[0].t,
175 | 			MaxTime: smpls[len(smpls)-1].t,
176 | 			Raw:     &storepb.Chunk{Type: storepb.Chunk_XOR, Data: c.Bytes()},
177 | 		}
178 | 
179 | 		s.Chunks = append(s.Chunks, ch)
180 | 	}
181 | 	return storepb.NewSeriesResponse(&s)
182 | }
183 | 


--------------------------------------------------------------------------------
/cmd/obslytics/export_e2e_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | // Copyright (c) The Thanos Authors.
  5 | // Licensed under the Apache License 2.0.
  6 | 
  7 | package main
  8 | 
  9 | import (
 10 | 	"context"
 11 | 	"fmt"
 12 | 	"io"
 13 | 	"os"
 14 | 	"testing"
 15 | 	"time"
 16 | 
 17 | 	"github.com/efficientgo/e2e"
 18 | 	e2emon "github.com/efficientgo/e2e/monitoring"
 19 | 
 20 | 	"github.com/go-kit/log"
 21 | 	"github.com/prometheus/prometheus/model/labels"
 22 | 	"github.com/thanos-io/objstore"
 23 | 	"github.com/thanos-io/thanos/pkg/exthttp"
 24 | 
 25 | 	"github.com/thanos-community/obslytics/pkg/dataframe"
 26 | 	"github.com/thanos-community/obslytics/pkg/exporter"
 27 | 	"github.com/thanos-community/obslytics/pkg/exporter/parquet"
 28 | 	"github.com/thanos-community/obslytics/pkg/series"
 29 | 	"github.com/thanos-community/obslytics/pkg/series/promread"
 30 | 	"github.com/thanos-community/obslytics/pkg/series/storeapi"
 31 | 
 32 | 	"github.com/efficientgo/core/testutil"
 33 | 	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
 34 | )
 35 | 
 36 | // defaultPromConfig returns Prometheus config that sets Prometheus to:
 37 | // * expose 2 external labels, source and replica.
 38 | // * scrape fake target. This will produce up == 0 metric which we can assert on.
 39 | // * optionally remote write endpoint to write into.
 40 | func defaultPromConfig(name string, replica int, remoteWriteEndpoint, ruleFile string) string {
 41 | 	config := fmt.Sprintf(`
 42 | global:
 43 |   external_labels:
 44 |     prometheus: %v
 45 |     replica: %v
 46 | scrape_configs:
 47 | - job_name: 'myself'
 48 |   # Quick scrapes for test purposes.
 49 |   scrape_interval: 1s
 50 |   scrape_timeout: 1s
 51 |   static_configs:
 52 |   - targets: ['localhost:9090']
 53 | `, name, replica)
 54 | 
 55 | 	if remoteWriteEndpoint != "" {
 56 | 		config = fmt.Sprintf(`
 57 | %s
 58 | remote_write:
 59 | - url: "%s"
 60 |   # Don't spam receiver on mistake.
 61 |   queue_config:
 62 |     min_backoff: 2s
 63 |     max_backoff: 10s
 64 | `, config, remoteWriteEndpoint)
 65 | 	}
 66 | 
 67 | 	if ruleFile != "" {
 68 | 		config = fmt.Sprintf(`
 69 | %s
 70 | rule_files:
 71 | -  "%s"
 72 | `, config, ruleFile)
 73 | 	}
 74 | 
 75 | 	return config
 76 | }
 77 | 
 78 | func exportToParquet(t *testing.T, ctx context.Context, r series.Reader, bkt objstore.Bucket, mint, maxt time.Time, fileName string) {
 79 | 	s, err := r.Read(ctx, series.Params{
 80 | 		Matchers: []*labels.Matcher{
 81 | 			labels.MustNewMatcher(labels.MatchEqual, "__name__", "prometheus_tsdb_head_series"),
 82 | 		},
 83 | 		MinTime: mint,
 84 | 		MaxTime: maxt,
 85 | 	})
 86 | 	testutil.Ok(t, err)
 87 | 
 88 | 	df, err := dataframe.FromSeries(s, 3*time.Second, func(o *dataframe.AggrsOptions) {
 89 | 		// TODO(inecas): Expose the enabled aggregations via flag.
 90 | 		o.Count.Enabled = true
 91 | 		o.Sum.Enabled = true
 92 | 		o.Min.Enabled = true
 93 | 		o.Max.Enabled = true
 94 | 	})
 95 | 	testutil.Ok(t, err)
 96 | 
 97 | 	t.Log("Dataframe:", dataframe.ToString(df))
 98 | 	testutil.Ok(t, exporter.New(parquet.NewEncoder(), fileName, bkt).Export(ctx, df))
 99 | }
100 | 
101 | func TestRemoteReadAndThanos_Parquet_e2e(t *testing.T) {
102 | 	t.Parallel()
103 | 
104 | 	e, err := e2e.NewDockerEnvironment("export")
105 | 	testutil.Ok(t, err)
106 | 	t.Cleanup(e.Close)
107 | 
108 | 	mint := time.Now()
109 | 	testutil.Ok(t, os.Setenv("THANOS_IMAGE", "quay.io/thanos/thanos:v0.29.0"))
110 | 
111 | 	prom, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "prom", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
112 | 	testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar))
113 | 
114 | 	testutil.Ok(t, prom.WaitSumMetricsWithOptions(e2emon.Greater(512), []string{"prometheus_tsdb_head_samples_appended_total"}, e2emon.WaitMissingMetrics()))
115 | 	maxt := time.Now()
116 | 
117 | 	logger := log.NewLogfmtLogger(os.Stderr)
118 | 	bkt := objstore.NewInMemBucket()
119 | 
120 | 	t.Run("export metric from RemoteRead to parquet file", func(t *testing.T) {
121 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
122 | 		defer cancel()
123 | 
124 | 		api, err := promread.NewSeries(logger, series.Config{
125 | 			Endpoint:  "http://" + prom.Endpoint("http") + "/api/v1/read",
126 | 			TLSConfig: exthttp.TLSConfig{InsecureSkipVerify: true},
127 | 		})
128 | 		testutil.Ok(t, err)
129 | 
130 | 		exportToParquet(t, ctx, api, bkt, mint, maxt, "something/yolo.parquet")
131 | 	})
132 | 
133 | 	t.Run("export metric from StoreAPI to parquet file", func(t *testing.T) {
134 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
135 | 		defer cancel()
136 | 
137 | 		api, err := storeapi.NewSeries(logger, series.Config{
138 | 			Endpoint:  sidecar.Endpoint("grpc"),
139 | 			TLSConfig: exthttp.TLSConfig{InsecureSkipVerify: true},
140 | 		})
141 | 		testutil.Ok(t, err)
142 | 
143 | 		exportToParquet(t, ctx, api, bkt, mint, maxt, "something/yolo2.parquet")
144 | 	})
145 | 
146 | 	result, err := bkt.Get(context.Background(), "something/yolo.parquet")
147 | 	testutil.Ok(t, err)
148 | 
149 | 	resultBytes1, err := io.ReadAll(result)
150 | 	testutil.Ok(t, err)
151 | 	testutil.Ok(t, result.Close())
152 | 
153 | 	// TODO(bwplotka): Assert properly the actual result, vs what metric actually gives.
154 | 	testutil.Assert(t, 1610 <= len(resultBytes1)) // Output varies from 2197 to 1610, debug shows me sometimes two rows, is this expected?
155 | 
156 | 	result, err = bkt.Get(context.Background(), "something/yolo2.parquet")
157 | 	testutil.Ok(t, err)
158 | 
159 | 	resultBytes2, err := io.ReadAll(result)
160 | 	testutil.Ok(t, err)
161 | 	testutil.Ok(t, result.Close())
162 | 
163 | 	// TODO(bwplotka): Assert properly the actual result, vs what metric actually gives.
164 | 	testutil.Assert(t, 1610 <= len(resultBytes2)) // Output varies from 2197 to 1610, debug shows me sometimes two rows, is this expected?
165 | 
166 | 	// Data from both StoreAPI and Remote Read should be the same.
167 | 	testutil.Equals(t, resultBytes1, resultBytes2)
168 | }
169 | 


--------------------------------------------------------------------------------
/cmd/obslytics/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package main
  5 | 
  6 | import (
  7 | 	"fmt"
  8 | 	"os"
  9 | 	"os/signal"
 10 | 	"path/filepath"
 11 | 	"runtime"
 12 | 	"syscall"
 13 | 
 14 | 	"github.com/efficientgo/core/errors"
 15 | 	"github.com/go-kit/log"
 16 | 	"github.com/go-kit/log/level"
 17 | 	"github.com/oklog/run"
 18 | 	"github.com/prometheus/common/version"
 19 | 	"go.uber.org/automaxprocs/maxprocs"
 20 | 	"gopkg.in/alecthomas/kingpin.v2"
 21 | )
 22 | 
 23 | const (
 24 | 	logFormatLogfmt = "logfmt"
 25 | 	logFormatJson   = "json"
 26 | )
 27 | 
 28 | type setupFunc func(*run.Group, log.Logger) error
 29 | 
 30 | func main() {
 31 | 	if os.Getenv("DEBUG") != "" {
 32 | 		runtime.SetMutexProfileFraction(10)
 33 | 		runtime.SetBlockProfileRate(10)
 34 | 	}
 35 | 
 36 | 	app := kingpin.New(filepath.Base(os.Args[0]), "Integrate Observability data into your Analytics pipelines")
 37 | 	app.Version(version.Version)
 38 | 	app.HelpFlag.Short('h')
 39 | 
 40 | 	logLevel := app.Flag("log.level", "Log filtering level.").
 41 | 		Default("info").Enum("error", "warn", "info", "debug")
 42 | 	logFormat := app.Flag("log.format", "Log format to use.").
 43 | 		Default(logFormatLogfmt).Enum(logFormatLogfmt, logFormatJson)
 44 | 
 45 | 	cmds := map[string]setupFunc{}
 46 | 	registerExport(cmds, app)
 47 | 
 48 | 	cmd, err := app.Parse(os.Args[1:])
 49 | 	if err != nil {
 50 | 		fmt.Fprintln(os.Stderr, errors.Wrapf(err, "error parsing commandline arguments"))
 51 | 		app.Usage(os.Args[1:])
 52 | 		os.Exit(2)
 53 | 	}
 54 | 
 55 | 	var logger log.Logger
 56 | 	{
 57 | 		var lvl level.Option
 58 | 		switch *logLevel {
 59 | 		case "error":
 60 | 			lvl = level.AllowError()
 61 | 		case "warn":
 62 | 			lvl = level.AllowWarn()
 63 | 		case "info":
 64 | 			lvl = level.AllowInfo()
 65 | 		case "debug":
 66 | 			lvl = level.AllowDebug()
 67 | 		default:
 68 | 			panic("unexpected log level")
 69 | 		}
 70 | 		logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
 71 | 		if *logFormat == logFormatJson {
 72 | 			logger = log.NewJSONLogger(log.NewSyncWriter(os.Stderr))
 73 | 		}
 74 | 		logger = level.NewFilter(logger, lvl)
 75 | 		logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller)
 76 | 	}
 77 | 
 78 | 	loggerAdapter := func(template string, args ...interface{}) {
 79 | 		level.Debug(logger).Log("msg", fmt.Sprintf(template, args))
 80 | 	}
 81 | 
 82 | 	// Running in container with limits but with empty/wrong value of GOMAXPROCS env var could lead to throttling by cpu
 83 | 	// maxprocs will automate adjustment by using cgroups info about cpu limit if it set as value for runtime.GOMAXPROCS.
 84 | 	undo, err := maxprocs.Set(maxprocs.Logger(loggerAdapter))
 85 | 	defer undo()
 86 | 	if err != nil {
 87 | 		level.Warn(logger).Log("msg", "failed to set GOMAXPROCS", "err", err)
 88 | 	}
 89 | 
 90 | 	var g run.Group
 91 | 	if err := cmds[cmd](&g, logger); err != nil {
 92 | 		level.Error(logger).Log("err", fmt.Sprintf("%v", errors.Wrapf(err, "%s command failed", cmd)))
 93 | 		os.Exit(1)
 94 | 	}
 95 | 
 96 | 	// Listen for termination signals.
 97 | 	{
 98 | 		cancel := make(chan struct{})
 99 | 		g.Add(func() error {
100 | 			return interrupt(logger, cancel)
101 | 		}, func(error) {
102 | 			close(cancel)
103 | 		})
104 | 	}
105 | 
106 | 	if err := g.Run(); err != nil {
107 | 		level.Error(logger).Log("msg", "running command failed", "err", err)
108 | 		os.Exit(1)
109 | 	}
110 | 	if cmd != "block plan" {
111 | 		level.Info(logger).Log("msg", "exiting", "cmd", cmd)
112 | 	}
113 | }
114 | 
115 | func interrupt(logger log.Logger, cancel <-chan struct{}) error {
116 | 	c := make(chan os.Signal, 1)
117 | 	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
118 | 	select {
119 | 	case s := <-c:
120 | 		level.Info(logger).Log("msg", "caught signal. Exiting.", "signal", s)
121 | 		return nil
122 | 	case <-cancel:
123 | 		return errors.New("canceled")
124 | 	}
125 | }
126 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
  1 | module github.com/thanos-community/obslytics
  2 | 
  3 | go 1.19
  4 | 
  5 | require (
  6 | 	github.com/efficientgo/core v1.0.0-rc.0
  7 | 	github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a
  8 | 	github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd
  9 | 	github.com/go-kit/log v0.2.1
 10 | 	github.com/oklog/run v1.1.0
 11 | 	github.com/prometheus/common v0.37.0
 12 | 	github.com/prometheus/prometheus v0.39.1
 13 | 	github.com/thanos-io/objstore v0.0.0-20221006135717-79dcec7fe604
 14 | 	github.com/thanos-io/thanos v0.29.0
 15 | 	github.com/xitongsys/parquet-go v1.6.2
 16 | 	github.com/xitongsys/parquet-go-source v0.0.0-20221025031416-9877e685ef65
 17 | 	go.uber.org/automaxprocs v1.5.1
 18 | 	google.golang.org/grpc v1.49.0
 19 | 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 20 | 	gopkg.in/yaml.v2 v2.4.0
 21 | )
 22 | 
 23 | require (
 24 | 	cloud.google.com/go v0.104.0 // indirect
 25 | 	cloud.google.com/go/compute v1.7.0 // indirect
 26 | 	cloud.google.com/go/iam v0.3.0 // indirect
 27 | 	cloud.google.com/go/storage v1.27.0 // indirect
 28 | 	cloud.google.com/go/trace v1.2.0 // indirect
 29 | 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1 // indirect
 30 | 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect
 31 | 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
 32 | 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 // indirect
 33 | 	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
 34 | 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3 // indirect
 35 | 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 // indirect
 36 | 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 37 | 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 38 | 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 39 | 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 40 | 	github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible // indirect
 41 | 	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
 42 | 	github.com/apache/thrift v0.14.2 // indirect
 43 | 	github.com/armon/go-metrics v0.4.0 // indirect
 44 | 	github.com/armon/go-radix v1.0.0 // indirect
 45 | 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 46 | 	github.com/aws/aws-sdk-go v1.44.109 // indirect
 47 | 	github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
 48 | 	github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
 49 | 	github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
 50 | 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
 51 | 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
 52 | 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
 53 | 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
 54 | 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
 55 | 	github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
 56 | 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
 57 | 	github.com/aws/smithy-go v1.11.2 // indirect
 58 | 	github.com/baidubce/bce-sdk-go v0.9.111 // indirect
 59 | 	github.com/beorn7/perks v1.0.1 // indirect
 60 | 	github.com/blang/semver/v4 v4.0.0 // indirect
 61 | 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
 62 | 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 63 | 	github.com/cespare/xxhash v1.1.0 // indirect
 64 | 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 65 | 	github.com/clbanning/mxj v1.8.4 // indirect
 66 | 	github.com/coreos/go-semver v0.3.0 // indirect
 67 | 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 68 | 	github.com/davecgh/go-spew v1.1.1 // indirect
 69 | 	github.com/dennwc/varint v1.0.0 // indirect
 70 | 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 71 | 	github.com/dustin/go-humanize v1.0.0 // indirect
 72 | 	github.com/edsrzf/mmap-go v1.1.0 // indirect
 73 | 	github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd // indirect
 74 | 	github.com/elastic/go-sysinfo v1.8.1 // indirect
 75 | 	github.com/elastic/go-windows v1.0.1 // indirect
 76 | 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 77 | 	github.com/fatih/color v1.13.0 // indirect
 78 | 	github.com/felixge/fgprof v0.9.2 // indirect
 79 | 	github.com/felixge/httpsnoop v1.0.3 // indirect
 80 | 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 81 | 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 82 | 	github.com/go-logr/logr v1.2.3 // indirect
 83 | 	github.com/go-logr/stdr v1.2.2 // indirect
 84 | 	github.com/go-ole/go-ole v1.2.6 // indirect
 85 | 	github.com/go-openapi/analysis v0.21.2 // indirect
 86 | 	github.com/go-openapi/errors v0.20.2 // indirect
 87 | 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 88 | 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 89 | 	github.com/go-openapi/loads v0.21.1 // indirect
 90 | 	github.com/go-openapi/spec v0.20.4 // indirect
 91 | 	github.com/go-openapi/strfmt v0.21.3 // indirect
 92 | 	github.com/go-openapi/swag v0.21.1 // indirect
 93 | 	github.com/go-openapi/validate v0.21.0 // indirect
 94 | 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 95 | 	github.com/gofrs/flock v0.8.1 // indirect
 96 | 	github.com/gogo/googleapis v1.4.0 // indirect
 97 | 	github.com/gogo/protobuf v1.3.2 // indirect
 98 | 	github.com/gogo/status v1.1.1 // indirect
 99 | 	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
100 | 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
101 | 	github.com/golang/protobuf v1.5.2 // indirect
102 | 	github.com/golang/snappy v0.0.4 // indirect
103 | 	github.com/google/btree v1.0.1 // indirect
104 | 	github.com/google/go-cmp v0.5.9 // indirect
105 | 	github.com/google/go-querystring v1.1.0 // indirect
106 | 	github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40 // indirect
107 | 	github.com/google/uuid v1.3.0 // indirect
108 | 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
109 | 	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
110 | 	github.com/gorilla/mux v1.8.0 // indirect
111 | 	github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 // indirect
112 | 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
113 | 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect
114 | 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
115 | 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1 // indirect
116 | 	github.com/hashicorp/consul/api v1.15.2 // indirect
117 | 	github.com/hashicorp/errwrap v1.1.0 // indirect
118 | 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
119 | 	github.com/hashicorp/go-hclog v0.16.2 // indirect
120 | 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
121 | 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
122 | 	github.com/hashicorp/go-multierror v1.1.1 // indirect
123 | 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
124 | 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
125 | 	github.com/hashicorp/golang-lru v0.5.4 // indirect
126 | 	github.com/hashicorp/memberlist v0.3.1 // indirect
127 | 	github.com/hashicorp/serf v0.9.7 // indirect
128 | 	github.com/jmespath/go-jmespath v0.4.0 // indirect
129 | 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
130 | 	github.com/josharian/intern v1.0.0 // indirect
131 | 	github.com/jpillora/backoff v1.0.0 // indirect
132 | 	github.com/json-iterator/go v1.1.12 // indirect
133 | 	github.com/julienschmidt/httprouter v1.3.0 // indirect
134 | 	github.com/klauspost/compress v1.15.9 // indirect
135 | 	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
136 | 	github.com/kylelemons/godebug v1.1.0 // indirect
137 | 	github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect
138 | 	github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect
139 | 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
140 | 	github.com/mailru/easyjson v0.7.7 // indirect
141 | 	github.com/mattn/go-colorable v0.1.12 // indirect
142 | 	github.com/mattn/go-isatty v0.0.14 // indirect
143 | 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
144 | 	github.com/miekg/dns v1.1.50 // indirect
145 | 	github.com/minio/md5-simd v1.1.2 // indirect
146 | 	github.com/minio/minio-go/v7 v7.0.37 // indirect
147 | 	github.com/minio/sha256-simd v1.0.0 // indirect
148 | 	github.com/mitchellh/go-homedir v1.1.0 // indirect
149 | 	github.com/mitchellh/mapstructure v1.5.0 // indirect
150 | 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
151 | 	github.com/modern-go/reflect2 v1.0.2 // indirect
152 | 	github.com/mozillazg/go-httpheader v0.2.1 // indirect
153 | 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
154 | 	github.com/ncw/swift v1.0.53 // indirect
155 | 	github.com/oklog/ulid v1.3.1 // indirect
156 | 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
157 | 	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
158 | 	github.com/opentracing/opentracing-go v1.2.0 // indirect
159 | 	github.com/oracle/oci-go-sdk/v65 v65.13.0 // indirect
160 | 	github.com/pierrec/lz4/v4 v4.1.8 // indirect
161 | 	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
162 | 	github.com/pkg/errors v0.9.1 // indirect
163 | 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
164 | 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
165 | 	github.com/prometheus/alertmanager v0.24.0 // indirect
166 | 	github.com/prometheus/client_golang v1.13.0 // indirect
167 | 	github.com/prometheus/client_model v0.2.0 // indirect
168 | 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
169 | 	github.com/prometheus/exporter-toolkit v0.7.1 // indirect
170 | 	github.com/prometheus/procfs v0.8.0 // indirect
171 | 	github.com/rogpeppe/go-internal v1.9.0 // indirect
172 | 	github.com/rs/xid v1.4.0 // indirect
173 | 	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
174 | 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
175 | 	github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
176 | 	github.com/shirou/gopsutil/v3 v3.22.9 // indirect
177 | 	github.com/sirupsen/logrus v1.9.0 // indirect
178 | 	github.com/sony/gobreaker v0.5.0 // indirect
179 | 	github.com/stretchr/objx v0.4.0 // indirect
180 | 	github.com/stretchr/testify v1.8.0 // indirect
181 | 	github.com/tencentyun/cos-go-sdk-v5 v0.7.34 // indirect
182 | 	github.com/tklauser/go-sysconf v0.3.10 // indirect
183 | 	github.com/tklauser/numcpus v0.4.0 // indirect
184 | 	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
185 | 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
186 | 	github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a // indirect
187 | 	github.com/weaveworks/common v0.0.0-20220706100410-67d27ed40fae // indirect
188 | 	github.com/weaveworks/promrus v1.2.0 // indirect
189 | 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
190 | 	go.elastic.co/apm v1.11.0 // indirect
191 | 	go.elastic.co/apm/module/apmhttp v1.11.0 // indirect
192 | 	go.elastic.co/apm/module/apmot v1.11.0 // indirect
193 | 	go.elastic.co/fastjson v1.1.0 // indirect
194 | 	go.etcd.io/etcd/api/v3 v3.5.4 // indirect
195 | 	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
196 | 	go.etcd.io/etcd/client/v3 v3.5.4 // indirect
197 | 	go.mongodb.org/mongo-driver v1.10.2 // indirect
198 | 	go.opencensus.io v0.23.0 // indirect
199 | 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.0 // indirect
200 | 	go.opentelemetry.io/contrib/propagators/autoprop v0.34.0 // indirect
201 | 	go.opentelemetry.io/contrib/propagators/aws v1.9.0 // indirect
202 | 	go.opentelemetry.io/contrib/propagators/b3 v1.9.0 // indirect
203 | 	go.opentelemetry.io/contrib/propagators/jaeger v1.9.0 // indirect
204 | 	go.opentelemetry.io/contrib/propagators/ot v1.9.0 // indirect
205 | 	go.opentelemetry.io/contrib/samplers/jaegerremote v0.3.0 // indirect
206 | 	go.opentelemetry.io/otel v1.10.0 // indirect
207 | 	go.opentelemetry.io/otel/bridge/opentracing v1.10.0 // indirect
208 | 	go.opentelemetry.io/otel/exporters/jaeger v1.8.0 // indirect
209 | 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
210 | 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
211 | 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
212 | 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.10.0 // indirect
213 | 	go.opentelemetry.io/otel/metric v0.32.0 // indirect
214 | 	go.opentelemetry.io/otel/sdk v1.10.0 // indirect
215 | 	go.opentelemetry.io/otel/trace v1.10.0 // indirect
216 | 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
217 | 	go.uber.org/atomic v1.10.0 // indirect
218 | 	go.uber.org/goleak v1.2.0 // indirect
219 | 	go.uber.org/multierr v1.8.0 // indirect
220 | 	go.uber.org/zap v1.21.0 // indirect
221 | 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
222 | 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
223 | 	golang.org/x/net v0.0.0-20220920203100-d0c6ba3f52d9 // indirect
224 | 	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
225 | 	golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
226 | 	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
227 | 	golang.org/x/text v0.3.7 // indirect
228 | 	golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 // indirect
229 | 	golang.org/x/tools v0.1.12 // indirect
230 | 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
231 | 	google.golang.org/api v0.97.0 // indirect
232 | 	google.golang.org/appengine v1.6.7 // indirect
233 | 	google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
234 | 	google.golang.org/protobuf v1.28.1 // indirect
235 | 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
236 | 	gopkg.in/ini.v1 v1.66.6 // indirect
237 | 	gopkg.in/yaml.v3 v3.0.1 // indirect
238 | 	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
239 | )
240 | 
241 | // Compatibility constraints
242 | 
243 | replace (
244 | 	// Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86.
245 | 	// Required by Cortex https://github.com/cortexproject/cortex/pull/3051.
246 | 	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
247 | 
248 | 	github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e
249 | 
250 | 	// Override due to https://github.com/weaveworks/common/issues/239
251 | 	google.golang.org/grpc => google.golang.org/grpc v1.45.0
252 | 
253 | 	// Overriding to use latest commit.
254 | 	gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497
255 | 
256 | 	// From Prometheus.
257 | 	k8s.io/klog => github.com/simonpasquier/klog-gokit v0.3.0
258 | 	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.0.0
259 | )
260 | 


--------------------------------------------------------------------------------
/pkg/dataframe/dataframe.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package dataframe
 5 | 
 6 | import (
 7 | 	"bytes"
 8 | 	"fmt"
 9 | 	"io"
10 | 	"text/tabwriter"
11 | 	"time"
12 | )
13 | 
14 | type Type string
15 | 
16 | const (
17 | 	TypeString Type = "string"
18 | 	TypeFloat  Type = "float"
19 | 	TypeUint   Type = "uint"
20 | 	TypeTime   Type = "time"
21 | )
22 | 
23 | type Column struct {
24 | 	Name string
25 | 	Type Type
26 | }
27 | 
28 | // Schema defines columns to be exposed by the dataframe.
29 | type Schema []Column
30 | 
31 | // RowsIterator exposes the rows of the dataframe.
32 | type RowsIterator interface {
33 | 	Next() bool
34 | 	At() Row
35 | }
36 | 
37 | // Row stores a single line of a table - the order of columns is defined by the Schema.
38 | type Row []interface{}
39 | 
40 | // Dataframe exposes ingested data to be used to turn into tabular format.
41 | type Dataframe interface {
42 | 	Schema() Schema
43 | 	RowsIterator() RowsIterator
44 | }
45 | 
46 | // Print formats the dataframe into format usable for debugging and testing purposes (e.g. in
47 | // examples). Uses tabwriter to produce the table in readable format and shortens
48 | // fields when possible (such as using only time part of a timestamp) so it fits
49 | // nicer into the output.
50 | func Print(w io.Writer, df Dataframe) {
51 | 	tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
52 | 
53 | 	printHeader(tw, df)
54 | 	i := df.RowsIterator()
55 | 	for i.Next() {
56 | 		printRow(tw, df.Schema(), i.At())
57 | 	}
58 | 	_ = tw.Flush()
59 | }
60 | 
61 | func ToString(df Dataframe) string {
62 | 	b := &bytes.Buffer{}
63 | 	Print(b, df)
64 | 	return b.String()
65 | }
66 | 
67 | func printHeader(w io.Writer, df Dataframe) {
68 | 	// Adding | <-   -> | around the lines to avoid dealing with training spaces
69 | 	// in example output checking.
70 | 	fmt.Fprint(w, "| ")
71 | 	for _, c := range df.Schema() {
72 | 		fmt.Fprintf(w, "%s\t", c.Name)
73 | 	}
74 | 	fmt.Fprint(w, "|\n")
75 | }
76 | 
77 | func printRow(w io.Writer, s Schema, r Row) {
78 | 	fmt.Fprint(w, "| ")
79 | 	for i, cell := range r {
80 | 		c := s[i]
81 | 		switch c.Type {
82 | 		case TypeString:
83 | 			fmt.Fprintf(w, "%s\t", cell)
84 | 		case TypeFloat:
85 | 			v := cell.(float64)
86 | 			fmt.Fprintf(w, "%.0f\t", v)
87 | 		case TypeUint:
88 | 			v := cell.(uint64)
89 | 			fmt.Fprintf(w, "%d\t", v)
90 | 		case TypeTime:
91 | 			v := cell.(time.Time)
92 | 			fmt.Fprintf(w, "%s\t", v.Format("15:04:05"))
93 | 		default:
94 | 			fmt.Fprintf(w, "%s\t", cell)
95 | 		}
96 | 	}
97 | 	fmt.Fprint(w, "|\n")
98 | }
99 | 


--------------------------------------------------------------------------------
/pkg/dataframe/series.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package dataframe
  5 | 
  6 | import (
  7 | 	"sort"
  8 | 	"time"
  9 | 
 10 | 	"github.com/efficientgo/core/errors"
 11 | 	"github.com/prometheus/prometheus/model/labels"
 12 | 	"github.com/prometheus/prometheus/model/timestamp"
 13 | 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 14 | 
 15 | 	"github.com/thanos-community/obslytics/pkg/series"
 16 | )
 17 | 
 18 | // AggrOption defines options for a single aggregation.
 19 | type AggrOption struct {
 20 | 	// Should the aggregation be used?
 21 | 	Enabled bool
 22 | 	// Column to store the aggregation at.
 23 | 	Column string
 24 | }
 25 | 
 26 | // AggrsOptions ia a collections of aggregations-related options. Determines
 27 | // what aggregations are enabled etc..
 28 | type AggrsOptions struct {
 29 | 	// Function to suggest the time of the first sample based on the resolution
 30 | 	// and the initial time of a series. By default, it uses time.Truncate(resolution)
 31 | 	// to normalize against the beginning of epoch.
 32 | 	initSampleTimeFunc func(time.Duration, time.Time) time.Time
 33 | 
 34 | 	Sum   AggrOption
 35 | 	Count AggrOption
 36 | 	Min   AggrOption
 37 | 	Max   AggrOption
 38 | }
 39 | 
 40 | // By default, all aggregations are disabled and target columns set with `_` prefix.
 41 | func defaultSeriesAggrsOptions() AggrsOptions {
 42 | 	return AggrsOptions{
 43 | 		initSampleTimeFunc: func(res time.Duration, t time.Time) time.Time {
 44 | 			return t.Truncate(res)
 45 | 		},
 46 | 
 47 | 		Sum:   AggrOption{Column: "_sum"},
 48 | 		Count: AggrOption{Column: "_count"},
 49 | 		Min:   AggrOption{Column: "_min"},
 50 | 		Max:   AggrOption{Column: "_max"},
 51 | 	}
 52 | }
 53 | 
 54 | type AggrOptionFunc func(*AggrsOptions)
 55 | 
 56 | func evalOptions(optFuncs []AggrOptionFunc) *AggrsOptions {
 57 | 	opt := defaultSeriesAggrsOptions()
 58 | 	for _, o := range optFuncs {
 59 | 		o(&opt)
 60 | 	}
 61 | 	return &opt
 62 | }
 63 | 
 64 | type aggregatedSeries struct {
 65 | 	labels      labels.Labels
 66 | 	hash        uint64
 67 | 	sampleStart time.Time
 68 | 	sampleEnd   time.Time
 69 | 	minTime     time.Time
 70 | 	maxTime     time.Time
 71 | 	count       uint64
 72 | 	min         float64
 73 | 	max         float64
 74 | 	sum         float64
 75 | }
 76 | 
 77 | type seriesAggregator struct {
 78 | 	df         *seriesDataframe
 79 | 	resolution time.Duration
 80 | 	options    AggrsOptions
 81 | }
 82 | 
 83 | // IteratorFromSeries returns iterator that produce dataframe for every series.
 84 | // TODO(bwplotka): Dataframe allows us to do bit more streaming approach. Consider this.
 85 | func FromSeries(r series.Set, resolution time.Duration, opts ...AggrOptionFunc) (Dataframe, error) {
 86 | 	defer r.Close()
 87 | 
 88 | 	// TODO(bwplotka): What if resolution is 0?
 89 | 	a := &seriesAggregator{
 90 | 		resolution: resolution,
 91 | 		options:    *evalOptions(opts),
 92 | 		df:         &seriesDataframe{seriesRecordSets: make(map[uint64]*seriesRecordSet)},
 93 | 	}
 94 | 
 95 | 	var activeSeries *aggregatedSeries
 96 | 	var currentHash uint64
 97 | 	for r.Next() {
 98 | 		s := r.At()
 99 | 		ls := s.Labels()
100 | 		seriesHash := ls.Hash()
101 | 
102 | 		i := s.Iterator()
103 | 		if !i.Next() {
104 | 			// Series without samples.
105 | 			if err := i.Err(); err != nil {
106 | 				return nil, err
107 | 			}
108 | 			continue
109 | 		}
110 | 
111 | 		if currentHash != seriesHash {
112 | 			if activeSeries != nil {
113 | 				_ = a.finalizeSample(activeSeries, activeSeries.sampleEnd)
114 | 			}
115 | 
116 | 			mint, _ := i.At()
117 | 			sampleStart := a.options.initSampleTimeFunc(resolution, timestamp.Time(mint))
118 | 			sampleEnd := sampleStart.Add(resolution)
119 | 
120 | 			activeSeries = &aggregatedSeries{labels: ls, hash: seriesHash, sampleStart: sampleStart, sampleEnd: sampleEnd}
121 | 			currentHash = seriesHash
122 | 		}
123 | 
124 | 		if !i.Seek(timestamp.FromTime(activeSeries.sampleStart)) {
125 | 			// No chunks after the sampleStart to process.
126 | 			if err := i.Err(); err != nil {
127 | 				return nil, err
128 | 			}
129 | 			continue
130 | 		}
131 | 
132 | 		if err := a.ingestSamples(activeSeries, i); err != nil {
133 | 			return nil, errors.Wrap(err, "aggregating samples")
134 | 		}
135 | 	}
136 | 
137 | 	if activeSeries != nil {
138 | 		_ = a.finalizeSample(activeSeries, activeSeries.sampleEnd)
139 | 	}
140 | 
141 | 	// We postpone the schema calculation to the time just before sending the df out
142 | 	// so that we can use the ingested data to determine the labels to be exported.
143 | 	a.df.schema = a.getSchema()
144 | 	return a.df, r.Err()
145 | }
146 | 
147 | // ingestSamples ingests samples provided via an iterator for single series. We
148 | // assume the iterator returns values ordered by the timestamp.
149 | // The iterator is expected to already be at the point of the first sample after as.sampleStart.
150 | func (a *seriesAggregator) ingestSamples(as *aggregatedSeries, i chunkenc.Iterator) error {
151 | 	var (
152 | 		ts int64
153 | 		v  float64
154 | 		t  time.Time
155 | 	)
156 | 	for {
157 | 		ts, v = i.At()
158 | 		t = timestamp.Time(ts)
159 | 		if t.Before(as.sampleStart) {
160 | 			return errors.Newf("Chunk timestamp %s is less than the sampleStart %s", t, as.sampleStart)
161 | 		}
162 | 		if t.After(as.sampleEnd) {
163 | 			as = a.finalizeSample(as, t)
164 | 		}
165 | 
166 | 		if as.count == 0 {
167 | 			as.minTime = t
168 | 			as.maxTime = t
169 | 			as.min = v
170 | 			as.max = v
171 | 		}
172 | 		if as.maxTime.After(t) {
173 | 			return errors.Newf("Incoming chunks are not sorted by timestamp: expected %s after %s", t, as.maxTime)
174 | 		}
175 | 		as.maxTime = t
176 | 		as.count += 1
177 | 		as.sum += v
178 | 		if as.max < v {
179 | 			as.max = v
180 | 		}
181 | 		if as.min > v {
182 | 			as.min = v
183 | 		}
184 | 		if !i.Next() {
185 | 			return i.Err()
186 | 		}
187 | 	}
188 | }
189 | 
190 | // finalizeSample adds the active aggregated series into the final dataframe when we've reached the
191 | // sample end time. Returns pointer to a new instance of the aggregatedSeries.
192 | func (a *seriesAggregator) finalizeSample(as *aggregatedSeries, nextT time.Time) *aggregatedSeries {
193 | 	if as.count > 0 {
194 | 		a.df.addSeries(as, a.options)
195 | 	}
196 | 
197 | 	// calculate the next sample cycle to contain the nextT time. First calculate how many
198 | 	// whole resolution cycles are between current sampleStart and nextT and then add
199 | 	// those cycles to the current sampleStart.
200 | 	nextSampleCycle := (nextT.Unix() - as.sampleStart.Unix()) / (int64)(a.resolution/time.Second)
201 | 	nextSampleStart := as.sampleStart.Add((time.Duration(nextSampleCycle)) * a.resolution)
202 | 
203 | 	return &aggregatedSeries{
204 | 		labels:      as.labels,
205 | 		hash:        as.hash,
206 | 		sampleStart: nextSampleStart,
207 | 		sampleEnd:   nextSampleStart.Add(a.resolution),
208 | 	}
209 | }
210 | 
211 | // getLabelNames assumes all series having the same labels and just takes the first
212 | // series to get the label names.
213 | // The returned strings are always sorted alphabetically.
214 | func (a *seriesAggregator) getLabelNames() []string {
215 | 	// TODO(inecas): The labels can be changing over time: to workaround
216 | 	// this problem, it could have to add an option to explicitly provide the list
217 | 	// of labels we want to export and fill in NULLs in case the label would be missing.
218 | 	var (
219 | 		ls  labels.Labels
220 | 		ret []string
221 | 	)
222 | 	// Create a union set (map) of all the label names.
223 | 	lsMap := make(map[string]string)
224 | 	for _, s := range a.df.seriesRecordSets {
225 | 		for labelName, labelValue := range s.Labels.Map() {
226 | 			lsMap[labelName] = labelValue
227 | 		}
228 | 	}
229 | 	ls = labels.FromMap(lsMap)
230 | 
231 | 	for _, l := range ls {
232 | 		if l.Name == "__name__" {
233 | 			continue
234 | 		}
235 | 		ret = append(ret, l.Name)
236 | 	}
237 | 	sort.Strings(ret)
238 | 	return ret
239 | }
240 | 
241 | func (a *seriesAggregator) getSchema() Schema {
242 | 	ao := a.options
243 | 	schema := Schema{}
244 | 
245 | 	for _, l := range a.getLabelNames() {
246 | 		schema = append(schema, Column{Name: l, Type: TypeString})
247 | 	}
248 | 
249 | 	timeColumns := []Column{
250 | 		{Name: "_sample_start", Type: TypeTime},
251 | 		{Name: "_sample_end", Type: TypeTime},
252 | 		{Name: "_min_time", Type: TypeTime},
253 | 		{Name: "_max_time", Type: TypeTime},
254 | 	}
255 | 	schema = append(schema, timeColumns...)
256 | 
257 | 	if ao.Count.Enabled {
258 | 		schema = append(schema, Column{Name: ao.Count.Column, Type: TypeUint})
259 | 	}
260 | 	if ao.Sum.Enabled {
261 | 		schema = append(schema, Column{Name: ao.Sum.Column, Type: TypeFloat})
262 | 	}
263 | 	if ao.Min.Enabled {
264 | 		schema = append(schema, Column{Name: ao.Min.Column, Type: TypeFloat})
265 | 	}
266 | 	if ao.Max.Enabled {
267 | 		schema = append(schema, Column{Name: ao.Max.Column, Type: TypeFloat})
268 | 	}
269 | 
270 | 	return schema
271 | }
272 | 
273 | // seriesDataframe implements dataframe.Dataframe.
274 | type seriesDataframe struct {
275 | 	schema           Schema
276 | 	seriesRecordSets map[uint64]*seriesRecordSet
277 | 	seriesOrder      []uint64
278 | }
279 | 
280 | func (df *seriesDataframe) addSeries(as *aggregatedSeries, opts AggrsOptions) {
281 | 	rs, ok := df.seriesRecordSets[as.hash]
282 | 	if !ok {
283 | 		rs = df.addRecordSet(as.labels)
284 | 	}
285 | 
286 | 	vals := map[string]interface{}{
287 | 		"_sample_start": as.sampleStart,
288 | 		"_sample_end":   as.sampleEnd,
289 | 		"_min_time":     as.minTime,
290 | 		"_max_time":     as.maxTime,
291 | 	}
292 | 
293 | 	for _, l := range as.labels {
294 | 		if l.Name == "__name__" {
295 | 			continue
296 | 		}
297 | 		vals[l.Name] = l.Value
298 | 	}
299 | 
300 | 	if opts.Count.Enabled {
301 | 		vals[opts.Count.Column] = as.count
302 | 	}
303 | 	if opts.Sum.Enabled {
304 | 		vals[opts.Sum.Column] = as.sum
305 | 	}
306 | 	if opts.Min.Enabled {
307 | 		vals[opts.Min.Column] = as.min
308 | 	}
309 | 	if opts.Max.Enabled {
310 | 		vals[opts.Max.Column] = as.max
311 | 	}
312 | 	rs.Records = append(rs.Records, Record{Values: vals})
313 | }
314 | 
315 | // Initiate new recordset for specific label.
316 | func (df *seriesDataframe) addRecordSet(ls labels.Labels) *seriesRecordSet {
317 | 	rs := &seriesRecordSet{Labels: ls, Records: make([]Record, 0)}
318 | 	hash := ls.Hash()
319 | 	df.seriesRecordSets[hash] = rs
320 | 	df.seriesOrder = append(df.seriesOrder, hash)
321 | 	return rs
322 | }
323 | 
324 | func (df seriesDataframe) Schema() Schema {
325 | 	return df.schema
326 | }
327 | 
328 | func (df seriesDataframe) RowsIterator() RowsIterator {
329 | 	rs := make([]seriesRecordSet, 0, len(df.seriesRecordSets))
330 | 	for _, v := range df.seriesOrder {
331 | 		rs = append(rs, *df.seriesRecordSets[v])
332 | 	}
333 | 	return &seriesDataframeRowIterator{seriesRecordSets: rs, schema: df.schema, seriesPos: 0, recordPos: -1}
334 | }
335 | 
336 | // seriesDataframeRowIterator implements dataframe.RowIterator.
337 | type seriesDataframeRowIterator struct {
338 | 	seriesRecordSets []seriesRecordSet
339 | 	schema           Schema
340 | 	seriesPos        int
341 | 	recordPos        int
342 | }
343 | 
344 | func (i *seriesDataframeRowIterator) Next() bool {
345 | 	if len(i.seriesRecordSets) == 0 {
346 | 		return false
347 | 	}
348 | 	s := i.seriesRecordSets[i.seriesPos]
349 | 
350 | 	if i.recordPos < len(s.Records)-1 {
351 | 		i.recordPos += 1
352 | 		return true
353 | 	}
354 | 
355 | 	if i.seriesPos < len(i.seriesRecordSets)-1 {
356 | 		i.seriesPos += 1
357 | 		i.recordPos = 0
358 | 		return true
359 | 	}
360 | 
361 | 	return false
362 | }
363 | 
364 | func (i *seriesDataframeRowIterator) At() Row {
365 | 	s := i.seriesRecordSets[i.seriesPos]
366 | 	ret := make([]interface{}, 0, len(i.schema))
367 | 	vals := s.Records[i.recordPos].Values
368 | 	for _, c := range i.schema {
369 | 		ret = append(ret, vals[c.Name])
370 | 	}
371 | 	return ret
372 | }
373 | 
374 | // seriesRecordSet is a set of records for specific labels values.
375 | type seriesRecordSet struct {
376 | 	Labels  labels.Labels
377 | 	Records []Record
378 | }
379 | 
380 | // Record is a single instance of values for specific sample.
381 | type Record struct {
382 | 	Values map[string]interface{}
383 | }
384 | 


--------------------------------------------------------------------------------
/pkg/exporter/export.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package exporter
 5 | 
 6 | import (
 7 | 	"context"
 8 | 	"io"
 9 | 
10 | 	"github.com/efficientgo/core/errors"
11 | 	"github.com/thanos-io/objstore"
12 | 	"github.com/thanos-io/objstore/client"
13 | 
14 | 	"github.com/thanos-community/obslytics/pkg/dataframe"
15 | )
16 | 
17 | type Type string
18 | 
19 | const (
20 | 	PARQUET Type = "PARQUET"
21 | )
22 | 
23 | // Config contains the options determining the object storage where files will be uploaded to.
24 | type Config struct {
25 | 	Type    Type                `yaml:"type"`
26 | 	Path    string              `yaml:"path"`
27 | 	Storage client.BucketConfig `yaml:"storage"`
28 | }
29 | 
30 | // An Encoder writes serialized type to an output stream.
31 | type Encoder interface {
32 | 	// TODO(bwplotka): Consider more generic option with interface{} if we have more types than Dataframe.
33 | 	Encode(io.Writer, dataframe.Dataframe) (err error)
34 | }
35 | 
36 | type Exporter struct {
37 | 	enc Encoder
38 | 
39 | 	path string
40 | 	bkt  objstore.Bucket
41 | }
42 | 
43 | func New(c Encoder, path string, bkt objstore.Bucket) *Exporter {
44 | 	return &Exporter{
45 | 		enc:  c,
46 | 		path: path,
47 | 		bkt:  bkt,
48 | 	}
49 | }
50 | 
51 | // Export encodes and streams the dataframe to given bucket. On error partial result might occur.
52 | // It's caller responsibility to clean after error.
53 | func (e *Exporter) Export(ctx context.Context, df dataframe.Dataframe) (err error) {
54 | 	r, w := io.Pipe()
55 | 
56 | 	errch := make(chan error, 1)
57 | 	go func() {
58 | 		// TODO(bwplotka): Log error from close (e.g using runutil.Close... package).
59 | 		defer w.Close()
60 | 		if err := e.enc.Encode(w, df); err != nil {
61 | 			errch <- errors.Wrap(err, "encode")
62 | 			return
63 | 		}
64 | 		errch <- nil
65 | 	}()
66 | 	defer func() {
67 | 		// TODO(bwplotka): Log error from close (e.g using runutil.Close... package).
68 | 		_ = r.Close()
69 | 		if cerr := <-errch; cerr != nil && err == nil {
70 | 			err = cerr
71 | 		}
72 | 	}()
73 | 
74 | 	if err := e.bkt.Upload(ctx, e.path, r); err != nil {
75 | 		return errors.Wrap(err, "upload")
76 | 	}
77 | 	return nil
78 | }
79 | 


--------------------------------------------------------------------------------
/pkg/exporter/factory/factory.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package factory
 5 | 
 6 | import (
 7 | 	"path"
 8 | 	"strings"
 9 | 
10 | 	"github.com/efficientgo/core/errors"
11 | 	"github.com/go-kit/log"
12 | 	"github.com/thanos-io/objstore/client"
13 | 	"gopkg.in/yaml.v2"
14 | 
15 | 	"github.com/thanos-community/obslytics/pkg/exporter"
16 | 	"github.com/thanos-community/obslytics/pkg/exporter/parquet"
17 | 	"github.com/thanos-community/obslytics/pkg/version"
18 | )
19 | 
20 | // NewExporter returns exporter based on configuration file.
21 | func NewExporter(logger log.Logger, cfg exporter.Config) (*exporter.Exporter, error) {
22 | 	storageConf, err := yaml.Marshal(cfg.Storage)
23 | 	if err != nil {
24 | 		return nil, errors.Wrap(err, "storage configuration")
25 | 	}
26 | 	bkt, err := client.NewBucket(logger, storageConf, nil, path.Join("obslytics", version.Version))
27 | 	if err != nil {
28 | 		return nil, errors.Wrap(err, "creating storage")
29 | 	}
30 | 
31 | 	var e exporter.Encoder
32 | 	switch exporter.Type(strings.ToUpper(string(cfg.Type))) {
33 | 	case exporter.PARQUET:
34 | 		e = parquet.NewEncoder()
35 | 	default:
36 | 		return nil, errors.Newf("unsupported export type %v", cfg.Type)
37 | 	}
38 | 
39 | 	return exporter.New(e, cfg.Path, bkt), nil
40 | }
41 | 


--------------------------------------------------------------------------------
/pkg/exporter/parquet/parquet.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package parquet
 5 | 
 6 | import (
 7 | 	"fmt"
 8 | 	"io"
 9 | 	"time"
10 | 
11 | 	"github.com/efficientgo/core/errors"
12 | 	parquetwriter "github.com/xitongsys/parquet-go-source/writerfile"
13 | 	"github.com/xitongsys/parquet-go/parquet"
14 | 	"github.com/xitongsys/parquet-go/source"
15 | 	"github.com/xitongsys/parquet-go/writer"
16 | 
17 | 	"github.com/thanos-community/obslytics/pkg/dataframe"
18 | 	"github.com/thanos-community/obslytics/pkg/exporter"
19 | )
20 | 
21 | // Compile-time check if parquet Encoder implements exporter.Encoder interface.
22 | var _ exporter.Encoder = &Encoder{}
23 | 
24 | type Encoder struct{}
25 | 
26 | func NewEncoder() *Encoder {
27 | 	return &Encoder{}
28 | }
29 | 
30 | func (e *Encoder) Encode(w io.Writer, df dataframe.Dataframe) (err error) {
31 | 	parqf := parquetwriter.NewWriterFile(w)
32 | 	parqw, err := initCSVWriter(parqf, df)
33 | 	if err != nil {
34 | 		return errors.Wrap(err, "initializing the schema")
35 | 	}
36 | 	defer func() {
37 | 		if serr := parqw.WriteStop(); serr != nil && err == nil {
38 | 			err = serr
39 | 		}
40 | 	}()
41 | 
42 | 	i := df.RowsIterator()
43 | 	s := df.Schema()
44 | 	for i.Next() {
45 | 		r := i.At()
46 | 		d := make([]interface{}, 0, len(r))
47 | 		for i, cell := range r {
48 | 			c := s[i]
49 | 			switch c.Type {
50 | 			case dataframe.TypeString:
51 | 				d = append(d, cell)
52 | 			case dataframe.TypeFloat:
53 | 				d = append(d, cell)
54 | 			case dataframe.TypeUint:
55 | 				v := cell.(uint64)
56 | 				// There has been some issue with uint and parquet-go, typecasting to int64 instead.
57 | 				d = append(d, int64(v))
58 | 			case dataframe.TypeTime:
59 | 				v := cell.(time.Time)
60 | 				d = append(d, v.Unix()*1000)
61 | 			default:
62 | 				d = append(d, cell)
63 | 			}
64 | 		}
65 | 		if err := parqw.Write(d); err != nil {
66 | 			return errors.Wrap(err, "writing a row")
67 | 		}
68 | 	}
69 | 	return nil
70 | }
71 | 
72 | func initCSVWriter(parqf source.ParquetFile, df dataframe.Dataframe) (*writer.CSVWriter, error) {
73 | 	schema := df.Schema()
74 | 	pqSchema := make([]string, 0, len(schema))
75 | 	for _, c := range schema {
76 | 		var pqType string
77 | 		switch c.Type {
78 | 		case dataframe.TypeString:
79 | 			pqType = "BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"
80 | 		case dataframe.TypeFloat:
81 | 			pqType = "DOUBLE"
82 | 		case dataframe.TypeUint:
83 | 			pqType = "INT64, convertedtype=UINT_64"
84 | 		case dataframe.TypeTime:
85 | 			pqType = "INT64, convertedtype=TIMESTAMP_MILLIS"
86 | 		}
87 | 		pqSchema = append(pqSchema, fmt.Sprintf("name=%s, type=%s", c.Name, pqType))
88 | 	}
89 | 
90 | 	parqw, err := writer.NewCSVWriter(pqSchema, parqf, 4)
91 | 	if err != nil {
92 | 		return nil, err
93 | 	}
94 | 	parqw.CompressionType = parquet.CompressionCodec_SNAPPY
95 | 
96 | 	return parqw, nil
97 | }
98 | 


--------------------------------------------------------------------------------
/pkg/series/factory/factory.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package factory
 5 | 
 6 | import (
 7 | 	"strings"
 8 | 
 9 | 	"github.com/efficientgo/core/errors"
10 | 	"github.com/go-kit/log"
11 | 
12 | 	"github.com/thanos-community/obslytics/pkg/series"
13 | 	"github.com/thanos-community/obslytics/pkg/series/promread"
14 | 	"github.com/thanos-community/obslytics/pkg/series/storeapi"
15 | )
16 | 
17 | // NewSeriesReader creates series.Reader based on configuration file.
18 | func NewSeriesReader(logger log.Logger, cfg series.Config) (series.Reader, error) {
19 | 	switch series.Type(strings.ToUpper(string(cfg.Type))) {
20 | 	case series.REMOTEREAD:
21 | 		return promread.NewSeries(logger, cfg)
22 | 	case series.STOREAPI:
23 | 		return storeapi.NewSeries(logger, cfg)
24 | 	default:
25 | 		return nil, errors.Newf("unsupported Reader type %s", cfg.Type)
26 | 	}
27 | }
28 | 


--------------------------------------------------------------------------------
/pkg/series/promread/promread.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package promread
  5 | 
  6 | import (
  7 | 	"context"
  8 | 	"net/url"
  9 | 	"path"
 10 | 
 11 | 	"github.com/efficientgo/core/errors"
 12 | 	"github.com/go-kit/log"
 13 | 	config_util "github.com/prometheus/common/config"
 14 | 	"github.com/prometheus/common/model"
 15 | 	"github.com/prometheus/prometheus/model/labels"
 16 | 	"github.com/prometheus/prometheus/model/timestamp"
 17 | 	"github.com/prometheus/prometheus/prompb"
 18 | 	"github.com/prometheus/prometheus/storage"
 19 | 	"github.com/prometheus/prometheus/storage/remote"
 20 | 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 21 | 
 22 | 	"github.com/thanos-community/obslytics/pkg/series"
 23 | 	"github.com/thanos-community/obslytics/pkg/version"
 24 | )
 25 | 
 26 | type Series struct {
 27 | 	logger log.Logger
 28 | 	conf   series.Config
 29 | }
 30 | 
 31 | func NewSeries(logger log.Logger, conf series.Config) (Series, error) {
 32 | 	return Series{logger: logger, conf: conf}, nil
 33 | }
 34 | 
 35 | // TranslatePromMatchers returns proto matchers (prompb) from Prometheus matchers.
 36 | // NOTE: It allocates memory.
 37 | func TranslatePromMatchers(ms ...*labels.Matcher) ([]*prompb.LabelMatcher, error) {
 38 | 	res := make([]*prompb.LabelMatcher, 0, len(ms))
 39 | 	for _, m := range ms {
 40 | 		var t prompb.LabelMatcher_Type
 41 | 
 42 | 		switch m.Type {
 43 | 		case labels.MatchEqual:
 44 | 			t = prompb.LabelMatcher_EQ
 45 | 		case labels.MatchNotEqual:
 46 | 			t = prompb.LabelMatcher_NEQ
 47 | 		case labels.MatchRegexp:
 48 | 			t = prompb.LabelMatcher_RE
 49 | 		case labels.MatchNotRegexp:
 50 | 			t = prompb.LabelMatcher_NRE
 51 | 		default:
 52 | 			return nil, errors.Newf("unrecognized matcher type %d", m.Type)
 53 | 		}
 54 | 		res = append(res, &prompb.LabelMatcher{Type: t, Name: m.Name, Value: m.Value})
 55 | 	}
 56 | 	return res, nil
 57 | }
 58 | 
 59 | func (i Series) Read(ctx context.Context, params series.Params) (series.Set, error) {
 60 | 	tlsConfig := config_util.TLSConfig{
 61 | 		CAFile:             i.conf.TLSConfig.CAFile,
 62 | 		CertFile:           i.conf.TLSConfig.CertFile,
 63 | 		KeyFile:            i.conf.TLSConfig.KeyFile,
 64 | 		ServerName:         i.conf.TLSConfig.ServerName,
 65 | 		InsecureSkipVerify: i.conf.TLSConfig.InsecureSkipVerify,
 66 | 	}
 67 | 
 68 | 	httpConfig := config_util.HTTPClientConfig{
 69 | 		TLSConfig: tlsConfig,
 70 | 	}
 71 | 
 72 | 	parsedUrl, err := url.Parse(i.conf.Endpoint)
 73 | 	if err != nil {
 74 | 		return nil, err
 75 | 	}
 76 | 	timeoutDuration, err := model.ParseDuration("10s")
 77 | 	if err != nil {
 78 | 		return nil, err
 79 | 	}
 80 | 
 81 | 	clientConfig := &remote.ClientConfig{
 82 | 		URL:              &config_util.URL{URL: parsedUrl},
 83 | 		Timeout:          timeoutDuration,
 84 | 		HTTPClientConfig: httpConfig,
 85 | 	}
 86 | 
 87 | 	client, err := remote.NewReadClient(path.Join("obslytics", version.Version), clientConfig)
 88 | 	if err != nil {
 89 | 		return nil, err
 90 | 	}
 91 | 
 92 | 	promLabelMatchers, err := TranslatePromMatchers(params.Matchers...)
 93 | 	if err != nil {
 94 | 		return nil, err
 95 | 	}
 96 | 
 97 | 	// Construct Query.
 98 | 	query := &prompb.Query{
 99 | 		StartTimestampMs: timestamp.FromTime(params.MinTime),
100 | 		EndTimestampMs:   timestamp.FromTime(params.MaxTime),
101 | 		Matchers:         promLabelMatchers,
102 | 	}
103 | 	// TODO: Move to streaming remote read version when available.
104 | 	readResponse, err := client.Read(ctx, query)
105 | 	if err != nil {
106 | 		return nil, err
107 | 	}
108 | 
109 | 	readSeriesList := make([]ReadSeries, 0, len(readResponse.Timeseries))
110 | 
111 | 	// Convert Timeseries List to a Read Series List.
112 | 	for index := range readResponse.Timeseries {
113 | 		readSeriesList = append(readSeriesList, ReadSeries{
114 | 			timeseries: *readResponse.Timeseries[index],
115 | 		})
116 | 	}
117 | 
118 | 	return &iterator{
119 | 		ctx:                ctx,
120 | 		client:             client,
121 | 		seriesList:         readSeriesList,
122 | 		currentSeriesIndex: -1,
123 | 	}, nil
124 | }
125 | 
126 | // iterator implements input.Set.
127 | type iterator struct {
128 | 	ctx                context.Context
129 | 	client             remote.ReadClient
130 | 	seriesList         []ReadSeries
131 | 	currentSeriesIndex int
132 | }
133 | 
134 | func (i *iterator) Next() bool {
135 | 	// Return false if the last index is already reached.
136 | 	if i.currentSeriesIndex+1 > len(i.seriesList)-1 {
137 | 		return false
138 | 	}
139 | 	i.currentSeriesIndex++
140 | 	return true
141 | 
142 | }
143 | 
144 | func (i *iterator) At() storage.Series {
145 | 	return i.seriesList[i.currentSeriesIndex]
146 | }
147 | 
148 | func (i *iterator) Warnings() storage.Warnings { return nil }
149 | func (i *iterator) Err() error                 { return nil }
150 | func (i *iterator) Close() error               { return nil }
151 | 
152 | // ReadSeries implements storage.Series.
153 | type ReadSeries struct {
154 | 	timeseries prompb.TimeSeries
155 | }
156 | 
157 | func (r ReadSeries) Labels() labels.Labels {
158 | 	var labelList []labels.Label
159 | 	for i := range r.timeseries.Labels {
160 | 		labelList = append(labelList, labels.Label{
161 | 			Name:  r.timeseries.Labels[i].Name,
162 | 			Value: r.timeseries.Labels[i].Value,
163 | 		})
164 | 
165 | 	}
166 | 	return labels.New(labelList...)
167 | }
168 | 
169 | func (r ReadSeries) Iterator() chunkenc.Iterator {
170 | 	return readChunk{series: r.timeseries}.Iterator()
171 | }
172 | 
173 | // readChunkIterator implements input.ChunkIterator.
174 | type readChunkIterator struct {
175 | 	Chunk              readChunk
176 | 	currentSampleIndex int
177 | }
178 | 
179 | func (c *readChunkIterator) Next() bool {
180 | 	// Return false if the last index is reached.
181 | 	if c.currentSampleIndex+1 > len(c.Chunk.series.Samples)-1 {
182 | 		return false
183 | 	}
184 | 	c.currentSampleIndex++
185 | 	return true
186 | }
187 | 
188 | // Seek advances the iterator forward to the first sample with the timestamp equal or greater than t.
189 | // If current sample found by previous `Next` or `Seek` operation already has this property, Seek has no effect.
190 | // Seek returns true, if such sample exists, false otherwise.
191 | // Set is exhausted when the Seek returns false.
192 | func (c *readChunkIterator) Seek(t int64) bool {
193 | 	if c.currentSampleIndex < 0 {
194 | 		c.currentSampleIndex = 0
195 | 	}
196 | 	for c.Chunk.series.Samples[c.currentSampleIndex].Timestamp < t {
197 | 		if !c.Next() {
198 | 			return false
199 | 		}
200 | 	}
201 | 	return true
202 | }
203 | 
204 | // At returns the current timestamp/value pair.
205 | // Before the iterator has advanced At behavior is unspecified.
206 | func (c *readChunkIterator) At() (int64, float64) {
207 | 	return c.Chunk.series.Samples[c.currentSampleIndex].Timestamp, c.Chunk.series.Samples[c.currentSampleIndex].Value
208 | }
209 | 
210 | // Err returns the current error. It should be used only after iterator is
211 | // exhausted, that is `Next` or `Seek` returns false.
212 | func (c *readChunkIterator) Err() error {
213 | 	return nil
214 | }
215 | 
216 | type readChunk struct {
217 | 	chunkenc.Chunk
218 | 	series prompb.TimeSeries
219 | }
220 | 
221 | func (c readChunk) Iterator() chunkenc.Iterator {
222 | 	return &readChunkIterator{Chunk: c, currentSampleIndex: -1}
223 | }
224 | 


--------------------------------------------------------------------------------
/pkg/series/promread/promread_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package promread
  5 | 
  6 | import (
  7 | 	"context"
  8 | 	"fmt"
  9 | 	"math"
 10 | 	"os"
 11 | 	"testing"
 12 | 	"time"
 13 | 
 14 | 	"github.com/efficientgo/core/testutil"
 15 | 	"github.com/efficientgo/e2e"
 16 | 	e2emon "github.com/efficientgo/e2e/monitoring"
 17 | 	"github.com/prometheus/prometheus/model/labels"
 18 | 	"github.com/prometheus/prometheus/model/timestamp"
 19 | 	http_util "github.com/thanos-io/thanos/pkg/exthttp"
 20 | 	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
 21 | 
 22 | 	"github.com/thanos-community/obslytics/pkg/series"
 23 | )
 24 | 
 25 | // defaultPromConfig returns Prometheus config that sets Prometheus to:
 26 | // * expose 2 external labels, source and replica.
 27 | // * scrape fake target. This will produce up == 0 metric which we can assert on.
 28 | // * optionally remote write endpoint to write into.
 29 | func defaultPromConfig(name string, replica int, remoteWriteEndpoint, ruleFile string) string {
 30 | 	config := fmt.Sprintf(`
 31 | global:
 32 |   external_labels:
 33 |     prometheus: %v
 34 |     replica: %v
 35 | scrape_configs:
 36 | - job_name: 'myself'
 37 |   # Quick scrapes for test purposes.
 38 |   scrape_interval: 1s
 39 |   scrape_timeout: 1s
 40 |   static_configs:
 41 |   - targets: ['localhost:9090']
 42 | `, name, replica)
 43 | 
 44 | 	if remoteWriteEndpoint != "" {
 45 | 		config = fmt.Sprintf(`
 46 | %s
 47 | remote_write:
 48 | - url: "%s"
 49 |   # Don't spam receiver on mistake.
 50 |   queue_config:
 51 |     min_backoff: 2s
 52 |     max_backoff: 10s
 53 | `, config, remoteWriteEndpoint)
 54 | 	}
 55 | 
 56 | 	if ruleFile != "" {
 57 | 		config = fmt.Sprintf(`
 58 | %s
 59 | rule_files:
 60 | -  "%s"
 61 | `, config, ruleFile)
 62 | 	}
 63 | 
 64 | 	return config
 65 | }
 66 | 
 67 | func TestRemoteReadInput_Open(t *testing.T) {
 68 | 	t.Parallel()
 69 | 
 70 | 	e, err := e2e.NewDockerEnvironment("remote-read")
 71 | 	testutil.Ok(t, err)
 72 | 	t.Cleanup(e.Close)
 73 | 
 74 | 	testutil.Ok(t, os.Setenv("THANOS_IMAGE", "quay.io/thanos/thanos:v0.29.0"))
 75 | 
 76 | 	prom, sidecar := e2ethanos.NewPrometheusWithSidecar(e, "prom", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage(), "")
 77 | 	testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar))
 78 | 
 79 | 	testutil.Ok(t, prom.WaitSumMetricsWithOptions(e2emon.Greater(512), []string{"prometheus_tsdb_head_samples_appended_total"}, e2emon.WaitMissingMetrics()))
 80 | 
 81 | 	t.Run("test remote read input", func(t *testing.T) {
 82 | 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 83 | 		defer cancel()
 84 | 
 85 | 		inConfig := series.Config{
 86 | 			Endpoint: "http://" + prom.Endpoint("http") + "/api/v1/read",
 87 | 			TLSConfig: http_util.TLSConfig{
 88 | 				InsecureSkipVerify: true,
 89 | 			},
 90 | 		}
 91 | 		remoteReadInput, err := NewSeries(nil, inConfig)
 92 | 		testutil.Ok(t, err)
 93 | 
 94 | 		minT := time.Now().Add(-6 * time.Hour)
 95 | 		maxT := timestamp.Time(math.MaxInt64)
 96 | 
 97 | 		inSeriesParams := series.Params{
 98 | 			Matchers: []*labels.Matcher{
 99 | 				labels.MustNewMatcher(labels.MatchEqual, "__name__", "prometheus_tsdb_head_series"),
100 | 			},
101 | 			MinTime: minT,
102 | 			MaxTime: maxT,
103 | 		}
104 | 
105 | 		inSeriesIter, err := remoteReadInput.Read(ctx, inSeriesParams)
106 | 		testutil.Ok(t, err)
107 | 
108 | 		// Go to the first series.
109 | 		testutil.Assert(t, inSeriesIter.Next() == true)
110 | 
111 | 		currentSeries := inSeriesIter.At()
112 | 		currentSeriesChunkIter := currentSeries.Iterator()
113 | 
114 | 		// Test for "__name__" label value.
115 | 		testutil.Assert(t, currentSeries.Labels().Get("__name__") == "prometheus_tsdb_head_series")
116 | 
117 | 		// Go to the first sample.
118 | 		testutil.Assert(t, currentSeriesChunkIter.Next() == true)
119 | 
120 | 		// Current Sample.
121 | 		metricTimestamp, metricValue := currentSeriesChunkIter.At()
122 | 		// Check if metric timestamp is valid.
123 | 		testutil.Assert(t, metricTimestamp > 0)
124 | 		// The first metric value is 0.
125 | 		testutil.Assert(t, metricValue == 0)
126 | 
127 | 		// Test if iteration inside chunk works, go to next sample.
128 | 		testutil.Assert(t, currentSeriesChunkIter.Next())
129 | 		metricTimestamp, metricValue = currentSeriesChunkIter.At()
130 | 		// Check if metric timestamp is valid.
131 | 		testutil.Assert(t, metricTimestamp > 0)
132 | 		// The second metric value is non 0.
133 | 		testutil.Assert(t, metricValue > 0)
134 | 
135 | 		// There are only two metric values inside the chunk, so this should return false.
136 | 		testutil.Assert(t, !currentSeriesChunkIter.Next())
137 | 
138 | 		// There is only one series, so this should return false.
139 | 		testutil.Assert(t, !inSeriesIter.Next())
140 | 
141 | 	})
142 | }
143 | 


--------------------------------------------------------------------------------
/pkg/series/series.go:
--------------------------------------------------------------------------------
 1 | // Copyright (c) The Thanos Community Authors.
 2 | // Licensed under the Apache License 2.0.
 3 | 
 4 | package series
 5 | 
 6 | import (
 7 | 	"context"
 8 | 	"time"
 9 | 
10 | 	"github.com/prometheus/prometheus/model/labels"
11 | 	"github.com/prometheus/prometheus/storage"
12 | 	http_util "github.com/thanos-io/thanos/pkg/exthttp"
13 | )
14 | 
15 | type Type string
16 | 
17 | const (
18 | 	REMOTEREAD Type = "REMOTEREAD"
19 | 	STOREAPI   Type = "STOREAPI"
20 | )
21 | 
22 | // Config contains the options determining the endpoint to talk to.
23 | type Config struct {
24 | 	Endpoint  string              `yaml:"endpoint"`
25 | 	TLSConfig http_util.TLSConfig `yaml:"tls_config"`
26 | 	Type      Type                `yaml:"type"`
27 | }
28 | 
29 | // Params determines what data should be loaded from the input.
30 | type Params struct {
31 | 	Matchers []*labels.Matcher
32 | 	MinTime  time.Time
33 | 	MaxTime  time.Time
34 | }
35 | 
36 | type Reader interface {
37 | 	Read(context.Context, Params) (Set, error)
38 | }
39 | 
40 | // Set allows iterating through all series in tn the input.
41 | // The set is expected to iterate series by series. The same series can be partitioned between multiple iterations.
42 | type Set interface {
43 | 	storage.SeriesSet
44 | 	Close() error
45 | }
46 | 


--------------------------------------------------------------------------------
/pkg/series/storeapi/iter.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package storeapi
  5 | 
  6 | import (
  7 | 	"github.com/efficientgo/core/errors"
  8 | 	"github.com/prometheus/prometheus/model/labels"
  9 | 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 10 | 
 11 | 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 12 | 	"github.com/thanos-io/thanos/pkg/store/storepb"
 13 | )
 14 | 
 15 | // chunkSeries implements storage.Series for a series on storepb types.
 16 | type chunkSeries struct {
 17 | 	lset       labels.Labels
 18 | 	chunks     []storepb.AggrChunk
 19 | 	mint, maxt int64
 20 | 	aggrs      []storepb.Aggr
 21 | }
 22 | 
 23 | // newChunkSeries allows to iterate over samples for each sorted and non-overlapped chunks.
 24 | func newChunkSeries(lset labels.Labels, chunks []storepb.AggrChunk, mint, maxt int64, aggrs []storepb.Aggr) *chunkSeries {
 25 | 	return &chunkSeries{
 26 | 		lset:   lset,
 27 | 		chunks: chunks,
 28 | 		mint:   mint,
 29 | 		maxt:   maxt,
 30 | 		aggrs:  aggrs,
 31 | 	}
 32 | }
 33 | 
 34 | func (s *chunkSeries) Labels() labels.Labels {
 35 | 	return s.lset
 36 | }
 37 | 
 38 | func (s *chunkSeries) Iterator() chunkenc.Iterator {
 39 | 	var sit chunkenc.Iterator
 40 | 	its := make([]chunkenc.Iterator, 0, len(s.chunks))
 41 | 
 42 | 	if len(s.aggrs) == 1 {
 43 | 		switch s.aggrs[0] {
 44 | 		case storepb.Aggr_COUNT:
 45 | 			for _, c := range s.chunks {
 46 | 				its = append(its, getFirstIterator(c.Count, c.Raw))
 47 | 			}
 48 | 			sit = newChunkSeriesIterator(its)
 49 | 		case storepb.Aggr_SUM:
 50 | 			for _, c := range s.chunks {
 51 | 				its = append(its, getFirstIterator(c.Sum, c.Raw))
 52 | 			}
 53 | 			sit = newChunkSeriesIterator(its)
 54 | 		case storepb.Aggr_MIN:
 55 | 			for _, c := range s.chunks {
 56 | 				its = append(its, getFirstIterator(c.Min, c.Raw))
 57 | 			}
 58 | 			sit = newChunkSeriesIterator(its)
 59 | 		case storepb.Aggr_MAX:
 60 | 			for _, c := range s.chunks {
 61 | 				its = append(its, getFirstIterator(c.Max, c.Raw))
 62 | 			}
 63 | 			sit = newChunkSeriesIterator(its)
 64 | 		case storepb.Aggr_COUNTER:
 65 | 			for _, c := range s.chunks {
 66 | 				its = append(its, getFirstIterator(c.Counter, c.Raw))
 67 | 			}
 68 | 			sit = downsample.NewApplyCounterResetsIterator(its...)
 69 | 		default:
 70 | 			return errSeriesIterator{err: errors.Newf("unexpected result aggregate type %v", s.aggrs)}
 71 | 		}
 72 | 		return newBoundedSeriesIterator(sit, s.mint, s.maxt)
 73 | 	}
 74 | 
 75 | 	if len(s.aggrs) != 2 {
 76 | 		return errSeriesIterator{err: errors.Newf("unexpected result aggregate type %v", s.aggrs)}
 77 | 	}
 78 | 
 79 | 	switch {
 80 | 	case s.aggrs[0] == storepb.Aggr_SUM && s.aggrs[1] == storepb.Aggr_COUNT,
 81 | 		s.aggrs[0] == storepb.Aggr_COUNT && s.aggrs[1] == storepb.Aggr_SUM:
 82 | 
 83 | 		for _, c := range s.chunks {
 84 | 			if c.Raw != nil {
 85 | 				its = append(its, getFirstIterator(c.Raw))
 86 | 			} else {
 87 | 				sum, cnt := getFirstIterator(c.Sum), getFirstIterator(c.Count)
 88 | 				its = append(its, downsample.NewAverageChunkIterator(cnt, sum))
 89 | 			}
 90 | 		}
 91 | 		sit = newChunkSeriesIterator(its)
 92 | 	default:
 93 | 		return errSeriesIterator{err: errors.Newf("unexpected result aggregate type %v", s.aggrs)}
 94 | 	}
 95 | 	return newBoundedSeriesIterator(sit, s.mint, s.maxt)
 96 | }
 97 | 
 98 | func getFirstIterator(cs ...*storepb.Chunk) chunkenc.Iterator {
 99 | 	for _, c := range cs {
100 | 		if c == nil {
101 | 			continue
102 | 		}
103 | 		chk, err := chunkenc.FromData(chunkEncoding(c.Type), c.Data)
104 | 		if err != nil {
105 | 			return errSeriesIterator{err}
106 | 		}
107 | 		return chk.Iterator(nil)
108 | 	}
109 | 	return errSeriesIterator{errors.New("no valid chunk found")}
110 | }
111 | 
112 | func chunkEncoding(e storepb.Chunk_Encoding) chunkenc.Encoding {
113 | 	switch e {
114 | 	case storepb.Chunk_XOR:
115 | 		return chunkenc.EncXOR
116 | 	}
117 | 	return 255 // Invalid.
118 | }
119 | 
120 | type errSeriesIterator struct {
121 | 	err error
122 | }
123 | 
124 | func (errSeriesIterator) Seek(int64) bool      { return false }
125 | func (errSeriesIterator) Next() bool           { return false }
126 | func (errSeriesIterator) At() (int64, float64) { return 0, 0 }
127 | func (it errSeriesIterator) Err() error        { return it.err }
128 | 
129 | // boundedSeriesIterator wraps a series iterator and ensures that it only emits
130 | // samples within a fixed time range.
131 | type boundedSeriesIterator struct {
132 | 	it         chunkenc.Iterator
133 | 	mint, maxt int64
134 | }
135 | 
136 | func newBoundedSeriesIterator(it chunkenc.Iterator, mint, maxt int64) *boundedSeriesIterator {
137 | 	return &boundedSeriesIterator{it: it, mint: mint, maxt: maxt}
138 | }
139 | 
140 | func (it *boundedSeriesIterator) Seek(t int64) (ok bool) {
141 | 	if t > it.maxt {
142 | 		return false
143 | 	}
144 | 	if t < it.mint {
145 | 		t = it.mint
146 | 	}
147 | 	return it.it.Seek(t)
148 | }
149 | 
150 | func (it *boundedSeriesIterator) At() (t int64, v float64) {
151 | 	return it.it.At()
152 | }
153 | 
154 | func (it *boundedSeriesIterator) Next() bool {
155 | 	if !it.it.Next() {
156 | 		return false
157 | 	}
158 | 	t, _ := it.it.At()
159 | 
160 | 	// Advance the iterator if we are before the valid interval.
161 | 	if t < it.mint {
162 | 		if !it.Seek(it.mint) {
163 | 			return false
164 | 		}
165 | 		t, _ = it.it.At()
166 | 	}
167 | 	// Once we passed the valid interval, there is no going back.
168 | 	return t <= it.maxt
169 | }
170 | 
171 | func (it *boundedSeriesIterator) Err() error {
172 | 	return it.it.Err()
173 | }
174 | 
175 | // chunkSeriesIterator implements a series iterator on top
176 | // of a list of time-sorted, non-overlapping chunks.
177 | type chunkSeriesIterator struct {
178 | 	chunks []chunkenc.Iterator
179 | 	i      int
180 | }
181 | 
182 | func newChunkSeriesIterator(cs []chunkenc.Iterator) chunkenc.Iterator {
183 | 	if len(cs) == 0 {
184 | 		// This should not happen. StoreAPI implementations should not send empty results.
185 | 		return errSeriesIterator{err: errors.Newf("store returned an empty result")}
186 | 	}
187 | 	return &chunkSeriesIterator{chunks: cs}
188 | }
189 | 
190 | func (it *chunkSeriesIterator) Seek(t int64) (ok bool) {
191 | 	// We generally expect the chunks already to be cut down
192 | 	// to the range we are interested in. There's not much to be gained from
193 | 	// hopping across chunks so we just call next until we reach t.
194 | 	for {
195 | 		ct, _ := it.At()
196 | 		if ct >= t {
197 | 			return true
198 | 		}
199 | 		if !it.Next() {
200 | 			return false
201 | 		}
202 | 	}
203 | }
204 | 
205 | func (it *chunkSeriesIterator) At() (t int64, v float64) {
206 | 	return it.chunks[it.i].At()
207 | }
208 | 
209 | func (it *chunkSeriesIterator) Next() bool {
210 | 	lastT, _ := it.At()
211 | 
212 | 	if it.chunks[it.i].Next() {
213 | 		return true
214 | 	}
215 | 	if it.Err() != nil {
216 | 		return false
217 | 	}
218 | 	if it.i >= len(it.chunks)-1 {
219 | 		return false
220 | 	}
221 | 	// Chunks are guaranteed to be ordered but not generally guaranteed to not overlap.
222 | 	// We must ensure to skip any overlapping range between adjacent chunks.
223 | 	it.i++
224 | 	return it.Seek(lastT + 1)
225 | }
226 | 
227 | func (it *chunkSeriesIterator) Err() error {
228 | 	return it.chunks[it.i].Err()
229 | }
230 | 


--------------------------------------------------------------------------------
/pkg/series/storeapi/storeapi.go:
--------------------------------------------------------------------------------
  1 | // Copyright (c) The Thanos Community Authors.
  2 | // Licensed under the Apache License 2.0.
  3 | 
  4 | package storeapi
  5 | 
  6 | import (
  7 | 	"context"
  8 | 	"io"
  9 | 
 10 | 	"github.com/efficientgo/core/errors"
 11 | 	"github.com/go-kit/log"
 12 | 	"github.com/prometheus/prometheus/model/timestamp"
 13 | 	"github.com/prometheus/prometheus/storage"
 14 | 	"github.com/thanos-io/thanos/pkg/extgrpc"
 15 | 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 16 | 	"github.com/thanos-io/thanos/pkg/store/storepb"
 17 | 	tracing "github.com/thanos-io/thanos/pkg/tracing/client"
 18 | 	"google.golang.org/grpc"
 19 | 
 20 | 	"github.com/thanos-community/obslytics/pkg/series"
 21 | )
 22 | 
 23 | // Series implements input.Reader.
 24 | type Series struct {
 25 | 	logger log.Logger
 26 | 	conf   series.Config
 27 | }
 28 | 
 29 | func NewSeries(logger log.Logger, conf series.Config) (Series, error) {
 30 | 	return Series{logger: logger, conf: conf}, nil
 31 | }
 32 | 
 33 | func (i Series) Read(ctx context.Context, params series.Params) (series.Set, error) {
 34 | 	// set as true for authenticated connection if cert, key and/or ca are defined.
 35 | 	secure := i.conf.TLSConfig.CertFile != "" ||
 36 | 		i.conf.TLSConfig.KeyFile != "" ||
 37 | 		i.conf.TLSConfig.CAFile != ""
 38 | 	dialOpts, err := extgrpc.StoreClientGRPCOpts(i.logger, nil, tracing.NoopTracer(),
 39 | 		secure,
 40 | 		i.conf.TLSConfig.InsecureSkipVerify,
 41 | 		i.conf.TLSConfig.CertFile,
 42 | 		i.conf.TLSConfig.KeyFile,
 43 | 		i.conf.TLSConfig.CAFile,
 44 | 		i.conf.Endpoint,
 45 | 	)
 46 | 
 47 | 	if err != nil {
 48 | 		return nil, errors.Wrap(err, "error initializing GRPC options")
 49 | 	}
 50 | 
 51 | 	conn, err := grpc.DialContext(ctx, i.conf.Endpoint, dialOpts...)
 52 | 	if err != nil {
 53 | 		return nil, errors.Wrap(err, "error initializing GRPC dial context")
 54 | 	}
 55 | 
 56 | 	matchers, err := storepb.PromMatchersToMatchers(params.Matchers...)
 57 | 	if err != nil {
 58 | 		return nil, err
 59 | 	}
 60 | 
 61 | 	client := storepb.NewStoreClient(conn)
 62 | 	seriesClient, err := client.Series(ctx, &storepb.SeriesRequest{
 63 | 		MinTime:                 timestamp.FromTime(params.MinTime),
 64 | 		MaxTime:                 timestamp.FromTime(params.MaxTime),
 65 | 		Matchers:                matchers,
 66 | 		PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT,
 67 | 	})
 68 | 	if err != nil {
 69 | 		return nil, errors.Wrapf(err, "storepb.Series against %v", i.conf.Endpoint)
 70 | 	}
 71 | 
 72 | 	return &iterator{
 73 | 		ctx:    ctx,
 74 | 		conn:   conn,
 75 | 		client: seriesClient,
 76 | 		mint:   timestamp.FromTime(params.MinTime),
 77 | 		maxt:   timestamp.FromTime(params.MaxTime),
 78 | 	}, nil
 79 | }
 80 | 
 81 | // iterator implements input.Set.
 82 | type iterator struct {
 83 | 	ctx           context.Context
 84 | 	conn          *grpc.ClientConn
 85 | 	client        storepb.Store_SeriesClient
 86 | 	currentSeries *storepb.Series
 87 | 
 88 | 	mint, maxt int64
 89 | 
 90 | 	err error
 91 | }
 92 | 
 93 | func (i *iterator) Next() bool {
 94 | 	seriesResp, err := i.client.Recv()
 95 | 	if err == io.EOF {
 96 | 		return false
 97 | 	}
 98 | 	if err != nil {
 99 | 		i.err = err
100 | 		return false
101 | 	}
102 | 
103 | 	i.currentSeries = seriesResp.GetSeries()
104 | 	return true
105 | }
106 | 
107 | func (i *iterator) At() storage.Series {
108 | 	// We support only raw data for now.
109 | 	return newChunkSeries(
110 | 		labelpb.ZLabelsToPromLabels(i.currentSeries.Labels),
111 | 		i.currentSeries.Chunks,
112 | 		i.mint, i.maxt,
113 | 		[]storepb.Aggr{storepb.Aggr_COUNT, storepb.Aggr_SUM},
114 | 	)
115 | }
116 | 
117 | func (i *iterator) Warnings() storage.Warnings { return nil }
118 | 
119 | func (i *iterator) Err() error {
120 | 	return i.err
121 | }
122 | 
123 | func (i *iterator) Close() error {
124 | 	if err := i.client.CloseSend(); err != nil {
125 | 		return err
126 | 	}
127 | 
128 | 	return i.conn.Close()
129 | }
130 | 


--------------------------------------------------------------------------------
/pkg/version/version.go:
--------------------------------------------------------------------------------
1 | // Copyright (c) The Thanos Community Authors.
2 | // Licensed under the Apache License 2.0.
3 | 
4 | package version
5 | 
6 | // Version returns 'obslytics' version.
7 | const Version = "v0.1.0"
8 | 


--------------------------------------------------------------------------------
/scripts/build-check-comments.sh:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | 
  3 | set -eu
  4 | 
  5 | # Reliable cross-platform comment checker for go files.
  6 | # Uses docker and stable linux distro for reliable grep regex and
  7 | # to make sure we work on Linux and OS X and oh even Windows! :)
  8 | # Main issue is minor incompatibies between grep on various platforms.
  9 | #
 10 | # Linux: set USE_DOCKER=yes or USE_DOCKER=false to force run in docker
 11 | # All other platforms: uses docker by default. Set USE_DOCKER=no or USE_DOCKER=false to override.
 12 | #
 13 | # Checks Go code comments if they have trailing period (excludes protobuffers and vendor files).
 14 | # Comments with more than 3 spaces at beginning are omitted from the check, example: '//    - foo'.
 15 | # This does not include top-level commments for funcs and types.
 16 | #
 17 | # Example:
 18 | #   func main() {
 19 | #	    // comment without period, will trigger check
 20 | #	    //comment without leading space, will trigger check.
 21 | #	    // comment without trailing space, will trigger check.
 22 | #	    // good comment, leading space, ends with period, no trailing space.
 23 | #     //     - more than 3 leading spaces, will pass
 24 | #	    app := kingpin.New(filepath.Base(os.Args[0]), "A block storage based long-term storage for Prometheus")
 25 | #   }
 26 | 
 27 | 
 28 | # Abs path to project dir and this script, should work on all OS's
 29 | declare ROOT_DIR="$(cd $(dirname "${BASH_SOURCE}")/.. && pwd)"
 30 | declare THIS_SCRIPT="$(cd $(dirname "${BASH_SOURCE}") && pwd)/$(basename "${BASH_SOURCE}")"
 31 | 
 32 | # Image to use if we do docker-based commands. NB: busybox is no good for this.
 33 | declare IMAGE="debian:9-slim"
 34 | 
 35 | # User can explicitly ask to run in docker
 36 | declare USE_DOCKER=${USE_DOCKER:=""}
 37 | 
 38 | # For OS X, always use Docker as we have nasty
 39 | # compat GNU/BSG issues with grep.
 40 | if test "Linux" != "$(uname || true)"
 41 | then
 42 |   # Allow overriding for non-linux platforms
 43 |   if test "no" != "${USE_DOCKER}" && test "false" != "${USE_DOCKER}"
 44 |   then
 45 |     USE_DOCKER="yes"
 46 |   fi
 47 | fi
 48 | 
 49 | if test "yes" == "${USE_DOCKER}" || test "true" == "${USE_DOCKER}"
 50 | then
 51 |     # Make sure we only attach TTY if we have it, CI builds won't have it.
 52 |     declare TTY_FLAG=""
 53 |     if [ -t 1 ]
 54 |     then
 55 |         TTY_FLAG="-t"
 56 |     fi
 57 | 
 58 |     # Annoying issue with ownership of files in mapped volumes.
 59 |     # Need to run with same UID and GID in container as we do
 60 |     # on the machine, otherwise all output will be owned by root.
 61 |     # Doesn't happen on OS X but does on Linux. So we will do
 62 |     # UID and GID for Linux only (this won't work on OS X anyway).
 63 |     declare USER_FLAG=""
 64 |     if test "Linux" == "$(uname || true)"
 65 |     then
 66 |         USER_FLAG="-u $(id -u):$(id -g)"
 67 |     fi
 68 | 
 69 |     printf "\n\n\n This will run in Docker. \n If you get an error, ensure Docker is installed. \n\n\n"
 70 |     (
 71 |       set -x
 72 |       docker run \
 73 |               -i \
 74 |               ${TTY_FLAG} \
 75 |               ${USER_FLAG} \
 76 |               --rm \
 77 |               -v "${ROOT_DIR}":"${ROOT_DIR}":cached \
 78 |               -w "${ROOT_DIR}" \
 79 |               "${IMAGE}" \
 80 |               "${THIS_SCRIPT}"
 81 |     )
 82 |     exit 0
 83 | fi
 84 | 
 85 | function check_comments {
 86 |     # no bombing out on errors with grep
 87 |     set +e
 88 | 
 89 |     # This is quite mad but don't fear the https://regex101.com/ helps a lot.
 90 |     grep -Przo --color --include \*.go --exclude \*.pb.go --exclude bindata.go --exclude-dir vendor \
 91 |      '\n.*\s+//(\s{0,3}[^\s^+][^\n]+[^.?!:]{2}|[^\s].*)\n[ \t]*[^/\s].*\n' ./
 92 |     res=$?
 93 |     set -e
 94 | 
 95 |     # man grep: Normally, the exit status is 0 if selected lines are found and 1 otherwise.
 96 |     # But the exit status is 2 if an error occurred, unless the -q or --quiet or --silent
 97 |     # option is used and a selected line is found.
 98 |     if test "0" == "${res}"  # found something
 99 |     then
100 |       printf "\n\n\n Error: Found comments without trailing period. Comments has to be full sentences.\n\n\n"
101 |       exit 1
102 |     elif test "1" == "${res}" # nothing found, all clear
103 |     then
104 |       printf "\n\n\n All comment formatting is good, Spartan.\n\n\n"
105 |       exit 0
106 |     else  # grep error
107 |       printf "\n\n\n Hmmm something didn't work, issues with grep?.\n\n\n"
108 |       exit 2
109 |     fi
110 | }
111 | 
112 | check_comments
113 | 


--------------------------------------------------------------------------------
/scripts/cleanup-white-noise.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | SED_BIN=${SED_BIN:-sed}
3 | 
4 | ${SED_BIN} -i 's/[ \t]*$//' "$@"
5 | 


--------------------------------------------------------------------------------