├── cmd ├── testdata │ ├── boltdb │ │ ├── keys-with-history.txt │ │ ├── page2item2-key.txt │ │ ├── page2item0-key.txt │ │ ├── page2item1-key.txt │ │ ├── db │ │ ├── db-with-history │ │ ├── page2item0.bin │ │ ├── page2item1.bin │ │ ├── page2item2.bin │ │ ├── keys.txt │ │ ├── page2item0-summary.txt │ │ ├── page2item2-summary.txt │ │ └── page2item1-summary.txt │ ├── meta │ │ ├── pod.txt │ │ └── job.txt │ ├── proto │ │ ├── job.bin │ │ └── pod.bin │ ├── storage │ │ ├── job.bin │ │ ├── pod.bin │ │ └── pod-with-key.bin │ ├── json │ │ ├── job.json │ │ ├── pod.json │ │ └── pod-with-key.txt │ └── yaml │ │ ├── job.yaml │ │ └── pod.yaml ├── root.go ├── checksum.go ├── encode_test.go ├── encode.go ├── extract_test.go ├── analyze.go ├── decode_test.go ├── decode.go └── extract.go ├── db-with-crd ├── pkg ├── data │ ├── testdata │ │ └── boltdb │ │ │ ├── db │ │ │ └── db-with-crd │ ├── data_test.go │ └── data.go └── encoding │ ├── encoding_test.go │ ├── scheme.go │ └── encoding.go ├── README.md ├── code-of-conduct.md ├── .gitignore ├── OWNERS ├── SECURITY.md ├── RELEASE.md ├── SECURITY_CONTACTS ├── .travis.yml ├── main.go ├── Dockerfile ├── go.mod ├── REVIEWING.md ├── CONTRIBUTING.md ├── Makefile ├── go.sum └── LICENSE /cmd/testdata/boltdb/keys-with-history.txt: -------------------------------------------------------------------------------- 1 | a 5 1 5 2 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item2-key.txt: -------------------------------------------------------------------------------- 1 | /registry/jobs/default/pi 2 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item0-key.txt: -------------------------------------------------------------------------------- 1 | /registry/namespaces/default 2 | -------------------------------------------------------------------------------- /db-with-crd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/db-with-crd -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item1-key.txt: -------------------------------------------------------------------------------- 1 | /registry/pods/default/pi-dqtsw 2 | -------------------------------------------------------------------------------- /cmd/testdata/meta/pod.txt: -------------------------------------------------------------------------------- 1 | TypeMeta.APIVersion: v1 2 | TypeMeta.Kind: Pod 3 | -------------------------------------------------------------------------------- /cmd/testdata/meta/job.txt: -------------------------------------------------------------------------------- 1 | TypeMeta.APIVersion: batch/v1 2 | TypeMeta.Kind: Job 3 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/boltdb/db -------------------------------------------------------------------------------- /cmd/testdata/proto/job.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/proto/job.bin -------------------------------------------------------------------------------- /cmd/testdata/proto/pod.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/proto/pod.bin -------------------------------------------------------------------------------- /pkg/data/testdata/boltdb/db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/pkg/data/testdata/boltdb/db -------------------------------------------------------------------------------- /cmd/testdata/storage/job.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/storage/job.bin -------------------------------------------------------------------------------- /cmd/testdata/storage/pod.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/storage/pod.bin -------------------------------------------------------------------------------- /cmd/testdata/boltdb/db-with-history: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/boltdb/db-with-history -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/boltdb/page2item0.bin -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/boltdb/page2item1.bin -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/boltdb/page2item2.bin -------------------------------------------------------------------------------- /cmd/testdata/storage/pod-with-key.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/cmd/testdata/storage/pod-with-key.bin -------------------------------------------------------------------------------- /pkg/data/testdata/boltdb/db-with-crd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpbetz/auger/HEAD/pkg/data/testdata/boltdb/db-with-crd -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Auger 2 | 3 | Auger has migrated to https://github.com/etcd-io/auger 4 | 5 | This repository has been archived. 6 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/keys.txt: -------------------------------------------------------------------------------- 1 | /registry/jobs/default/pi 2 | /registry/namespaces/default 3 | /registry/pods/default/pi-dqtsw 4 | compact_rev_key 5 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item0-summary.txt: -------------------------------------------------------------------------------- 1 | Key: /registry/namespaces/default 2 | Version: 1 3 | CreateRevision: 4 4 | ModRevision: 4 5 | Lease: 0 6 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item2-summary.txt: -------------------------------------------------------------------------------- 1 | Key: /registry/jobs/default/pi 2 | Version: 3 3 | CreateRevision: 82647 4 | ModRevision: 82703 5 | Lease: 0 6 | -------------------------------------------------------------------------------- /cmd/testdata/boltdb/page2item1-summary.txt: -------------------------------------------------------------------------------- 1 | Key: /registry/pods/default/pi-dqtsw 2 | Version: 5 3 | CreateRevision: 82648 4 | ModRevision: 82702 5 | Lease: 0 6 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | auger 2 | build 3 | vendor 4 | 5 | # IDEs 6 | *.iml 7 | *~ 8 | .idea/ 9 | 10 | # OS X 11 | .DS_Store 12 | 13 | # Go 14 | *.out 15 | *.exe 16 | *.dll 17 | *.so 18 | *.dylib 19 | coverage.html 20 | .glide/ 21 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | 3 | approvers: 4 | - siyuanfoundation 5 | - jmhbnz 6 | - wenjiaswe 7 | 8 | emeritus_approvers: 9 | - jpbetz 10 | - mml 11 | - cheftako 12 | - Quentin-M 13 | - logicalhan 14 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Announcements 4 | 5 | Join the [kubernetes-security-announce] group for security and vulnerability announcements. 6 | 7 | ## Reporting a Vulnerability 8 | 9 | Instructions for reporting a vulnerability can be found on the 10 | [Kubernetes Security and Disclosure Information] page. 11 | 12 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | The Auger Project is released on an as-needed basis. The process is as follows: 4 | 5 | 1. An issue is proposing a new release with a changelog since the last release 6 | 1. A quorum of [OWNERS](OWNERS) must LGTM this release 7 | 1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` 8 | 1. The release issue is closed 9 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Security Response Committee to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | 14 | siyuanfoundation 15 | jmhbnz 16 | wenjiaswe 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | language: go 16 | go: 17 | - 1.19.1 18 | script: 19 | - env GO111MODULE=on make test 20 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/jpbetz/auger/cmd" 23 | ) 24 | 25 | func main() { 26 | if err := cmd.RootCmd.Execute(); err != nil { 27 | fmt.Println(err) 28 | os.Exit(1) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.16-alpine 16 | 17 | RUN apk add --no-cache curl git && rm -rf /var/cache/apk/* 18 | 19 | WORKDIR /go/src/github.com/jpbetz/auger 20 | ADD . /go/src/github.com/jpbetz/auger 21 | RUN go get github.com/jpbetz/auger && chmod +x /go/bin/auger 22 | 23 | ENTRYPOINT ["/go/bin/auger"] 24 | -------------------------------------------------------------------------------- /cmd/testdata/json/job.json: -------------------------------------------------------------------------------- 1 | {"kind":"Job","apiVersion":"batch/v1","metadata":{"name":"pi","namespace":"default","selfLink":"/apis/batch/v1/namespaces/default/jobs/pi","uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","creationTimestamp":"2017-06-27T16:35:34Z","labels":{"controller-uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","job-name":"pi"}},"spec":{"parallelism":1,"completions":1,"selector":{"matchLabels":{"controller-uid":"a4acc46c-5b56-11e7-8d4b-42010a800002"}},"template":{"metadata":{"name":"pi","creationTimestamp":null,"labels":{"controller-uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","job-name":"pi"}},"spec":{"containers":[{"name":"pi","image":"perl","command":["perl","-Mbignum=bpi","-wle","print bpi(2000)"],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","securityContext":{},"schedulerName":"default-scheduler"}}},"status":{"conditions":[{"type":"Complete","status":"True","lastProbeTime":"2017-06-27T16:36:07Z","lastTransitionTime":"2017-06-27T16:36:07Z"}],"startTime":"2017-06-27T16:35:34Z","completionTime":"2017-06-27T16:36:07Z","succeeded":1}} 2 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var cfgFile string 27 | 28 | var RootCmd = &cobra.Command{ 29 | Use: "auger", 30 | Short: "Inspect and analyze kubernetes storage data.", 31 | Long: `Inspect and analyze kubernetes objects in binary storage 32 | encoding used with etcd 3+ and boltdb.`, 33 | } 34 | 35 | // Execute adds all child commands to the root command and sets flags appropriately. 36 | // This is called by main.main(). It only needs to happen once to the rootCmd. 37 | func Execute() { 38 | if err := RootCmd.Execute(); err != nil { 39 | fmt.Println(err) 40 | os.Exit(1) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jpbetz/auger 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/coreos/bbolt v1.3.1-coreos.3 7 | github.com/coreos/etcd v3.1.11+incompatible 8 | github.com/google/safetext v0.0.0-20220914124124-e18e3fe012bf 9 | github.com/spf13/cobra v0.0.1 10 | gopkg.in/yaml.v2 v2.4.0 11 | k8s.io/api v0.0.0-20180521142803-feb48db456a5 12 | k8s.io/apimachinery v0.0.0-20180515182440-31dade610c05 13 | ) 14 | 15 | require ( 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 // indirect 18 | github.com/gogo/protobuf v1.3.2 // indirect 19 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect 20 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7 // indirect 21 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 // indirect 22 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 23 | github.com/json-iterator/go v0.0.0-20171212105241-13f86432b882 // indirect 24 | github.com/spf13/pflag v1.0.0 // indirect 25 | github.com/stretchr/testify v1.7.0 // indirect 26 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect 27 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect 28 | golang.org/x/text v0.3.3 // indirect 29 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 30 | gopkg.in/inf.v0 v0.9.0 // indirect 31 | gopkg.in/yaml.v3 v3.0.1 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /REVIEWING.md: -------------------------------------------------------------------------------- 1 | # Reviewing and Merging Pull Requests 2 | 3 | This document is a guideline for how core contributors should review and merge 4 | pull requests (PRs). It is intended to outline the lightweight process that 5 | we’ll use for now. It’s assumed that we’ll operate on good faith for now in 6 | situations for which process is not specified. 7 | 8 | PRs may only be merged after the following criteria are met: 9 | 10 | 1. It has been open for at least 1 business day 11 | 1. It has no `veto` label on it 12 | 1. It has been 'LGTM'-ed by 3 different reviewers, each from a different 13 | organization 14 | 1. It has all appropriate corresponding documentation and testcases 15 | 16 | ## LGTMs 17 | 18 | When a reviewer deems a PR good enough to merge, they should add a comment to the PR 19 | thread that simply reads 'LGTM'. If they do not deem it ready for merge, 20 | they should add comments -- either inline or on the PR thread -- that indicate 21 | changes they believe should be made before merge. 22 | 23 | ## Vetoing 24 | 25 | If a reviewer decides that a PR should not be merged in its current state, 26 | even if it gets 3 'LGTM' approvals from others, they should mark that PR with 27 | `veto` label. 28 | 29 | This label should only be used by a reviewer when that person believes there 30 | is a fundamental problem with the PR. The reviewer should summarize that problem 31 | in the PR comments and a longer discussion may be required. 32 | 33 | We expect this label to be used infrequently. 34 | -------------------------------------------------------------------------------- /cmd/checksum.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | 22 | "github.com/jpbetz/auger/pkg/data" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var checksumCmd = &cobra.Command{ 27 | Use: "checksum", 28 | Short: "Checksum a etcd keyspace.", 29 | RunE: func(cmd *cobra.Command, args []string) error { 30 | return checksum() 31 | }, 32 | } 33 | 34 | type checksumOptions struct { 35 | filename string 36 | revision int64 37 | } 38 | 39 | var checksumOpts *checksumOptions = &checksumOptions{} 40 | 41 | func init() { 42 | RootCmd.AddCommand(checksumCmd) 43 | checksumCmd.Flags().StringVarP(&checksumOpts.filename, "file", "f", "", "Bolt DB '.db' filename") 44 | checksumCmd.Flags().Int64VarP(&checksumOpts.revision, "revision", "r", 0, "etcd revision to retrieve data at, if 0, the latest revision is used, defaults to 0") 45 | } 46 | 47 | func checksum() error { 48 | checksum, err := data.HashByRevision(checksumOpts.filename, checksumOpts.revision) 49 | if err != nil { 50 | return err 51 | } 52 | fmt.Printf("checksum: %d\n", checksum.Hash) 53 | fmt.Printf("compact-revision: %d\n", checksum.CompactRevision) 54 | fmt.Printf("revision: %d\n", checksum.Revision) 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: 4 | 5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ 6 | 7 | ## Getting Started 8 | 9 | The following outlines the general rules we follow: 10 | 11 | - All PRs must have the appropriate documentation changes made within the 12 | same PR. Note, not all PRs will necessarily require a documentation change 13 | but if it does please include it in the same PR so the PR is complete and 14 | not just a partial solution. 15 | - All PRs must have the appropriate testcases. For example, bug fixes should 16 | include tests that demonstrates the issue w/o your fix. New features should 17 | include as many testcases, within reason, to cover any variants of use of the 18 | feature. 19 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) - Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests 20 | - [Kubernetes Contributor Guide](https://k8s.dev/guide) - Main contributor documentation, or you can just jump directly to the [contributing page](https://k8s.dev/docs/guide/contributing/) 21 | - [Contributor Cheat Sheet](https://k8s.dev/cheatsheet) - Common resources for existing developers 22 | 23 | ## Contact Information 24 | 25 | - [SIG-etcd slack channel](https://kubernetes.slack.com/archives/C3HD8ARJ5) 26 | - [Mailing list](https://groups.google.com/g/etcd-dev) 27 | -------------------------------------------------------------------------------- /cmd/encode_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "bytes" 21 | "testing" 22 | 23 | "github.com/jpbetz/auger/pkg/encoding" 24 | ) 25 | 26 | var encodeTests = []struct { 27 | fileIn string 28 | fileExpected string 29 | inMediaType string 30 | }{ 31 | {"testdata/yaml/pod.yaml", "testdata/storage/pod.bin", encoding.YamlMediaType}, 32 | {"testdata/json/pod.json", "testdata/storage/pod.bin", encoding.JsonMediaType}, 33 | {"testdata/yaml/job.yaml", "testdata/storage/job.bin", encoding.YamlMediaType}, 34 | {"testdata/json/job.json", "testdata/storage/job.bin", encoding.JsonMediaType}, 35 | } 36 | 37 | func TestEncode(t *testing.T) { 38 | for _, test := range encodeTests { 39 | in := readTestFile(t, test.fileIn) 40 | out := new(bytes.Buffer) 41 | if err := encodeRun(test.inMediaType, in, out); err != nil { 42 | t.Errorf("%v for %+v", err, test) 43 | continue 44 | } 45 | rt := new(bytes.Buffer) 46 | if err := run(false, test.inMediaType, out.Bytes(), rt); err != nil { 47 | t.Errorf("%v for round trip of %+v", err, test) 48 | continue 49 | } 50 | b := rt.Bytes() 51 | if !bytes.Equal(b, in) { 52 | t.Errorf("for round trip of %s, got:\n%s\nwanted:\n%s\n", test.fileIn, b, in) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2022 The Kubernetes Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | NAME ?= auger 17 | PKG ?= github.com/jpbetz/$(NAME) 18 | GO_VERSION ?= 1.20.5 19 | GOOS ?= linux 20 | GOARCH ?= amd64 21 | TEMP_DIR := $(shell mktemp -d) 22 | 23 | # Local development build 24 | build: 25 | @mkdir -p build 26 | GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/$(NAME) 27 | @echo build/$(NAME) built! 28 | 29 | # Local development test 30 | # `go test` automatically manages the build, so no need to depend on the build target here in make 31 | test: 32 | @echo Vetting 33 | go vet ./... 34 | @echo Testing 35 | go test ./... 36 | 37 | # Dockerized build 38 | release: 39 | @cp -r $(CURDIR) $(TEMP_DIR) 40 | @echo Building release in temp directory $(TEMP_DIR) 41 | docker run \ 42 | -v $(TEMP_DIR)/$(NAME):/go/src/$(PKG) \ 43 | -w /go/src/$(PKG) \ 44 | golang:$(GO_VERSION) \ 45 | /bin/bash -c "make -f /go/src/$(PKG)/Makefile release-docker-build GOARCH=$(GOARCH) GOOS=$(GOOS)" 46 | @mkdir -p build 47 | @cp $(TEMP_DIR)/$(NAME)/$(NAME) build/$(NAME) 48 | @echo build/$(NAME) built! 49 | 50 | # Build used inside docker by 'release' 51 | release-docker-build: 52 | export GOPATH=/go 53 | GOOS=$(GOOS) GOARCH=$(GOARCH) GO111MODULE=on go build 54 | 55 | clean: 56 | rm -rf build 57 | 58 | .PHONY: build test release release-docker-build clean 59 | -------------------------------------------------------------------------------- /cmd/testdata/yaml/job.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: batch/v1 15 | kind: Job 16 | metadata: 17 | creationTimestamp: "2017-06-27T16:35:34Z" 18 | labels: 19 | controller-uid: a4acc46c-5b56-11e7-8d4b-42010a800002 20 | job-name: pi 21 | name: pi 22 | namespace: default 23 | selfLink: /apis/batch/v1/namespaces/default/jobs/pi 24 | uid: a4acc46c-5b56-11e7-8d4b-42010a800002 25 | spec: 26 | completions: 1 27 | parallelism: 1 28 | selector: 29 | matchLabels: 30 | controller-uid: a4acc46c-5b56-11e7-8d4b-42010a800002 31 | template: 32 | metadata: 33 | creationTimestamp: null 34 | labels: 35 | controller-uid: a4acc46c-5b56-11e7-8d4b-42010a800002 36 | job-name: pi 37 | name: pi 38 | spec: 39 | containers: 40 | - command: 41 | - perl 42 | - -Mbignum=bpi 43 | - -wle 44 | - print bpi(2000) 45 | image: perl 46 | imagePullPolicy: Always 47 | name: pi 48 | resources: {} 49 | terminationMessagePath: /dev/termination-log 50 | terminationMessagePolicy: File 51 | dnsPolicy: ClusterFirst 52 | restartPolicy: Never 53 | schedulerName: default-scheduler 54 | securityContext: {} 55 | terminationGracePeriodSeconds: 30 56 | status: 57 | completionTime: "2017-06-27T16:36:07Z" 58 | conditions: 59 | - lastProbeTime: "2017-06-27T16:36:07Z" 60 | lastTransitionTime: "2017-06-27T16:36:07Z" 61 | status: "True" 62 | type: Complete 63 | startTime: "2017-06-27T16:35:34Z" 64 | succeeded: 1 65 | -------------------------------------------------------------------------------- /pkg/encoding/encoding_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | import ( 20 | "fmt" 21 | "testing" 22 | ) 23 | 24 | var findProtoTests = []struct { 25 | in string 26 | ok bool 27 | expected string 28 | }{ 29 | {string(ProtoEncodingPrefix), true, string(ProtoEncodingPrefix)}, 30 | {fmt.Sprintf("xxxxxx%s...end", ProtoEncodingPrefix), true, fmt.Sprintf("%s...end", ProtoEncodingPrefix)}, 31 | {fmt.Sprintf("xxxxxx{}"), false, ""}, 32 | } 33 | 34 | func TestTryFindProto(t *testing.T) { 35 | for _, test := range findProtoTests { 36 | out, ok := tryFindProto([]byte(test.in)) 37 | if test.ok != ok { 38 | t.Errorf("got ok=%t, want ok=%t for %+v", ok, test.ok, test) 39 | } 40 | if ok { 41 | if string(out) != test.expected { 42 | t.Errorf("got %s, want %s for %+v", out, test.expected, test) 43 | } 44 | } 45 | } 46 | } 47 | 48 | var findJsonTests = []struct { 49 | in string 50 | ok bool 51 | expected string 52 | }{ 53 | {"{}", true, "{}"}, 54 | {"[]", true, "[]"}, 55 | {`{"key": "value"}`, true, `{"key": "value"}`}, 56 | {"[1,2,3]", true, "[1,2,3]"}, 57 | {"{}[]{}", true, "{}"}, 58 | {`{"1": {"2": []}}`, true, `{"1": {"2": []}}`}, 59 | {"xxxxxx{}", true, "{}"}, 60 | {"xx{x[x{}", true, "{}"}, 61 | {"xxxxxx[]", true, "[]"}, 62 | {fmt.Sprintf("xxxxxx%s...end", ProtoEncodingPrefix), false, ""}, 63 | {"{}xxxxxx", false, ""}, 64 | {"{}[", false, ""}, 65 | } 66 | 67 | func TestTryFindJson(t *testing.T) { 68 | for _, test := range findJsonTests { 69 | out, ok := tryFindJson([]byte(test.in)) 70 | if test.ok != ok { 71 | t.Errorf("got ok=%t, want ok=%t for %+v", ok, test.ok, test) 72 | } 73 | if ok { 74 | js, err := out.MarshalJSON() 75 | if err != nil { 76 | t.Errorf("unable to marshal json match: %v for %+v", err, test) 77 | } else if string(js) != test.expected { 78 | t.Errorf("got %s, want %s for %+v", string(js), test.expected, test) 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /cmd/encode.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "os" 23 | 24 | "github.com/jpbetz/auger/pkg/encoding" 25 | "github.com/spf13/cobra" 26 | ) 27 | 28 | var ( 29 | encodeLong = ` 30 | Encodes kubernetes objects to the binary key-value store encoding used 31 | with etcd 3+. 32 | ` 33 | 34 | encodeExample = ` 35 | cat pod.yaml | auger encode | \ 36 | ETCDCTL_API=3 etcdctl put /registry/pods/default/` 37 | ) 38 | 39 | var encodeCmd = &cobra.Command{ 40 | Use: "encode", 41 | Short: "Encode objects to the kubernetes binary key-value store encoding.", 42 | Long: encodeLong, 43 | Example: encodeExample, 44 | RunE: func(cmd *cobra.Command, args []string) error { 45 | return encodeValidateAndRun() 46 | }, 47 | } 48 | 49 | type encodeOptions struct { 50 | in string 51 | inputFilename string 52 | } 53 | 54 | var encodeOpts *encodeOptions = &encodeOptions{} 55 | 56 | func init() { 57 | RootCmd.AddCommand(encodeCmd) 58 | encodeCmd.Flags().StringVarP(&encodeOpts.in, "format", "f", "yaml", "Input format. One of: json|yaml|proto") 59 | encodeCmd.Flags().StringVar(&encodeOpts.inputFilename, "file", "", "Filename to read input data from") 60 | } 61 | 62 | // encodeValidateAndRun validates the command line flags and runs the command. 63 | func encodeValidateAndRun() error { 64 | // TODO: detect input file type by examining file extension? 65 | inMediaType, err := encoding.ToMediaType(encodeOpts.in) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | in, err := readInput(encodeOpts.inputFilename) 71 | if len(in) == 0 { 72 | return fmt.Errorf("no input data") 73 | } 74 | if err != nil { 75 | return err 76 | } 77 | 78 | return encodeRun(inMediaType, in, os.Stdout) 79 | } 80 | 81 | // encodeRun runs the encode command. 82 | func encodeRun(inMediaType string, in []byte, out io.Writer) error { 83 | _, err := encoding.Convert(inMediaType, encoding.StorageBinaryMediaType, in, out) 84 | return err 85 | } 86 | -------------------------------------------------------------------------------- /cmd/testdata/json/pod.json: -------------------------------------------------------------------------------- 1 | {"kind":"Pod","apiVersion":"v1","metadata":{"name":"pi-dqtsw","generateName":"pi-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/pi-dqtsw","uid":"a4adc7ca-5b56-11e7-8d4b-42010a800002","creationTimestamp":"2017-06-27T16:35:34Z","labels":{"controller-uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","job-name":"pi"},"annotations":{"kubernetes.io/created-by":"{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"Job\",\"namespace\":\"default\",\"name\":\"pi\",\"uid\":\"a4acc46c-5b56-11e7-8d4b-42010a800002\",\"apiVersion\":\"batch\",\"resourceVersion\":\"82647\"}}\n","kubernetes.io/limit-ranger":"LimitRanger plugin set: cpu request for container pi"},"ownerReferences":[{"apiVersion":"batch/v1","kind":"Job","name":"pi","uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-nm27w","secret":{"secretName":"default-token-nm27w","defaultMode":420}}],"containers":[{"name":"pi","image":"perl","command":["perl","-Mbignum=bpi","-wle","print bpi(2000)"],"resources":{"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"default-token-nm27w","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"kubernetes-minion-group-vlql","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.alpha.kubernetes.io/notReady","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.alpha.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Succeeded","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:35:34Z","reason":"PodCompleted"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:36:07Z","reason":"PodCompleted"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:35:34Z"}],"hostIP":"10.128.0.4","podIP":"10.244.2.8","startTime":"2017-06-27T16:35:34Z","containerStatuses":[{"name":"pi","state":{"terminated":{"exitCode":0,"reason":"Completed","startedAt":"2017-06-27T16:36:01Z","finishedAt":"2017-06-27T16:36:07Z","containerID":"docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117"}},"lastState":{},"ready":false,"restartCount":0,"image":"perl:latest","imageID":"docker://sha256:9fc8e8ba0b3a067188ac46cf51a25021a73e3b927e9637fab48663813c457612","containerID":"docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117"}],"qosClass":"Burstable"}} 2 | -------------------------------------------------------------------------------- /cmd/testdata/json/pod-with-key.txt: -------------------------------------------------------------------------------- 1 | /registry/pods/default/xyz 2 | {"kind":"Pod","apiVersion":"v1","metadata":{"name":"pi-dqtsw","generateName":"pi-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/pi-dqtsw","uid":"a4adc7ca-5b56-11e7-8d4b-42010a800002","creationTimestamp":"2017-06-27T16:35:34Z","labels":{"controller-uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","job-name":"pi"},"annotations":{"kubernetes.io/created-by":"{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"Job\",\"namespace\":\"default\",\"name\":\"pi\",\"uid\":\"a4acc46c-5b56-11e7-8d4b-42010a800002\",\"apiVersion\":\"batch\",\"resourceVersion\":\"82647\"}}\n","kubernetes.io/limit-ranger":"LimitRanger plugin set: cpu request for container pi"},"ownerReferences":[{"apiVersion":"batch/v1","kind":"Job","name":"pi","uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-nm27w","secret":{"secretName":"default-token-nm27w","defaultMode":420}}],"containers":[{"name":"pi","image":"perl","command":["perl","-Mbignum=bpi","-wle","print bpi(2000)"],"resources":{"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"default-token-nm27w","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Never","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"kubernetes-minion-group-vlql","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.alpha.kubernetes.io/notReady","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.alpha.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Succeeded","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:35:34Z","reason":"PodCompleted"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:36:07Z","reason":"PodCompleted"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2017-06-27T16:35:34Z"}],"hostIP":"10.128.0.4","podIP":"10.244.2.8","startTime":"2017-06-27T16:35:34Z","containerStatuses":[{"name":"pi","state":{"terminated":{"exitCode":0,"reason":"Completed","startedAt":"2017-06-27T16:36:01Z","finishedAt":"2017-06-27T16:36:07Z","containerID":"docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117"}},"lastState":{},"ready":false,"restartCount":0,"image":"perl:latest","imageID":"docker://sha256:9fc8e8ba0b3a067188ac46cf51a25021a73e3b927e9637fab48663813c457612","containerID":"docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117"}],"qosClass":"Burstable"}} 3 | -------------------------------------------------------------------------------- /cmd/extract_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "bytes" 21 | "strings" 22 | "testing" 23 | 24 | "github.com/jpbetz/auger/pkg/encoding" 25 | ) 26 | 27 | const ( 28 | dbFile = "testdata/boltdb/db" 29 | dbWithHistoryFile = "testdata/boltdb/db-with-history" 30 | ) 31 | 32 | func TestListKeys(t *testing.T) { 33 | out := new(bytes.Buffer) 34 | if err := printKeySummaries(dbFile, "", 0, []string{"key"}, out); err != nil { 35 | t.Fatal(err) 36 | } 37 | assertMatchesFile(t, out, "testdata/boltdb/keys.txt") 38 | } 39 | 40 | func TestListKeySummaries(t *testing.T) { 41 | out := new(bytes.Buffer) 42 | if err := printKeySummaries(dbWithHistoryFile, "", 0, []string{"key", "version-count", "value-size", "all-versions-value-size"}, out); err != nil { 43 | t.Fatal(err) 44 | } 45 | assertMatchesFile(t, out, "testdata/boltdb/keys-with-history.txt") 46 | } 47 | 48 | func TestListKeyVersions(t *testing.T) { 49 | out := new(bytes.Buffer) 50 | if err := printVersions(dbFile, "/registry/jobs/default/pi", out); err != nil { 51 | t.Fatal(err) 52 | } 53 | s := strings.Trim(out.String(), " \n") 54 | if s != "3" { 55 | t.Errorf("got %s, want 3", s) 56 | } 57 | } 58 | 59 | func TestExtractByKey(t *testing.T) { 60 | out := new(bytes.Buffer) 61 | if err := printValue(dbFile, "/registry/jobs/default/pi", "3", false, encoding.YamlMediaType, out); err != nil { 62 | t.Fatal(err) 63 | } 64 | assertMatchesFile(t, out, "testdata/yaml/job.yaml") 65 | } 66 | 67 | func TestExtractKeyFromLeaf(t *testing.T) { 68 | kv := readTestFileAsKv(t, "testdata/boltdb/page2item1.bin") 69 | out := new(bytes.Buffer) 70 | if err := printLeafItemKey(kv, out); err != nil { 71 | t.Fatal(err) 72 | } 73 | assertMatchesFile(t, out, "testdata/boltdb/page2item1-key.txt") 74 | } 75 | 76 | func TestExtractSummaryFromLeaf(t *testing.T) { 77 | kv := readTestFileAsKv(t, "testdata/boltdb/page2item1.bin") 78 | out := new(bytes.Buffer) 79 | if err := printLeafItemSummary(kv, out); err != nil { 80 | t.Fatal(err) 81 | } 82 | assertMatchesFile(t, out, "testdata/boltdb/page2item1-summary.txt") 83 | } 84 | 85 | // TODO: run a permutation grid of tests 86 | func TestExtractValueFromLeaf(t *testing.T) { 87 | kv := readTestFileAsKv(t, "testdata/boltdb/page2item1.bin") 88 | out := new(bytes.Buffer) 89 | if err := printLeafItemValue(kv, encoding.YamlMediaType, out); err != nil { 90 | t.Fatal(err) 91 | } 92 | assertMatchesFile(t, out, "testdata/yaml/pod.yaml") 93 | } 94 | -------------------------------------------------------------------------------- /cmd/analyze.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "sort" 22 | 23 | "github.com/jpbetz/auger/pkg/data" 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | var analyzeCmd = &cobra.Command{ 28 | Use: "analyze", 29 | Short: "Analyze kubernetes data from the boltdb '.db' files etcd persists to.", 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | return analyzeValidateAndRun() 32 | }, 33 | } 34 | 35 | type analyzeOptions struct { 36 | filename string 37 | } 38 | 39 | var analyzeOpts *analyzeOptions = &analyzeOptions{} 40 | 41 | func init() { 42 | RootCmd.AddCommand(analyzeCmd) 43 | analyzeCmd.Flags().StringVarP(&analyzeOpts.filename, "file", "f", "", "Bolt DB '.db' filename") 44 | } 45 | 46 | func analyzeValidateAndRun() error { 47 | summaries, err := data.ListKeySummaries(analyzeOpts.filename, []data.Filter{}, &data.KeySummaryProjection{HasKey: true, HasValue: false}, 0) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | objectCounts := map[string]uint{} 53 | for _, s := range summaries { 54 | if s.TypeMeta != nil { 55 | objectCounts[fmt.Sprintf("%s/%s", s.TypeMeta.APIVersion, s.TypeMeta.Kind)] += 1 56 | } 57 | } 58 | 59 | type entry struct { 60 | count uint 61 | gvk string 62 | } 63 | 64 | entries := []entry{} 65 | for k, v := range objectCounts { 66 | entries = append(entries, entry{gvk: k, count: v}) 67 | } 68 | 69 | sort.Slice(entries[:], func(i, j int) bool { 70 | return entries[i].count > entries[j].count 71 | }) 72 | 73 | var totalKeySize int 74 | var totalValueSize int 75 | var totalAllVersionsKeySize int 76 | var totalAllVersionsValueSize int 77 | for _, summary := range summaries { 78 | totalKeySize += summary.Stats.KeySize 79 | totalValueSize += summary.Stats.ValueSize 80 | totalAllVersionsKeySize += summary.Stats.AllVersionsKeySize 81 | totalAllVersionsValueSize += summary.Stats.AllVersionsValueSize 82 | } 83 | 84 | fmt.Printf("Total kubernetes objects: %d\n", len(summaries)) 85 | fmt.Printf("Total (all revisions) storage used by kubernetes objects: %d\n", totalAllVersionsKeySize+totalAllVersionsValueSize) 86 | fmt.Printf("Current (latest revision) storage used by kubernetes objects: %d\n", totalKeySize+totalValueSize) 87 | fmt.Printf("\n") 88 | fmt.Printf("Most common kubernetes types:\n") 89 | for i, entry := range entries { 90 | fmt.Printf("\t%d\t%s\n", entry.count, entry.gvk) 91 | if i > 10 { 92 | break 93 | } 94 | } 95 | 96 | sort.Slice(summaries[:], func(i, j int) bool { 97 | return summaries[i].Stats.AllVersionsValueSize > summaries[j].Stats.AllVersionsValueSize 98 | }) 99 | fmt.Printf("\n") 100 | fmt.Printf("Largest objects (byte size sum of all revisions):\n") 101 | for i, summary := range summaries { 102 | fmt.Printf("\t%d\t%s (%s/%s)\n", summary.Stats.AllVersionsValueSize, summary.Key, summary.TypeMeta.APIVersion, summary.TypeMeta.Kind) 103 | if i > 10 { 104 | break 105 | } 106 | } 107 | 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /cmd/decode_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "bytes" 21 | "io/ioutil" 22 | "os" 23 | "regexp" 24 | "strings" 25 | "testing" 26 | 27 | "github.com/coreos/etcd/mvcc/mvccpb" 28 | "github.com/jpbetz/auger/pkg/encoding" 29 | ) 30 | 31 | var decodeTests = []struct { 32 | fileIn string 33 | fileExpected string 34 | 35 | metaOnly bool 36 | outMediaType string 37 | }{ 38 | // Pod is in the 'core' group 39 | // {"testdata/storage/pod.bin", "testdata/yaml/pod.yaml", false, encoding.YamlMediaType}, 40 | {"testdata/storage/pod.bin", "testdata/json/pod.json", false, encoding.JsonMediaType}, 41 | {"testdata/storage/pod.bin", "testdata/proto/pod.bin", false, encoding.ProtobufMediaType}, 42 | {"testdata/storage/pod.bin", "testdata/meta/pod.txt", true, encoding.YamlMediaType}, 43 | 44 | // Job is in the 'batch' group 45 | // {"testdata/storage/job.bin", "testdata/yaml/job.yaml", false, encoding.YamlMediaType}, 46 | {"testdata/storage/job.bin", "testdata/json/job.json", false, encoding.JsonMediaType}, 47 | {"testdata/storage/job.bin", "testdata/proto/job.bin", false, encoding.ProtobufMediaType}, 48 | {"testdata/storage/job.bin", "testdata/meta/job.txt", true, encoding.YamlMediaType}, 49 | 50 | // JSON 51 | {"testdata/json/pod.json", "testdata/json/pod.json", false, encoding.JsonMediaType}, 52 | 53 | // With etcd key 54 | // {"testdata/storage/pod-with-key.bin", "testdata/yaml/pod.yaml", false, encoding.YamlMediaType}, 55 | {"testdata/json/pod-with-key.txt", "testdata/json/pod.json", false, encoding.JsonMediaType}, 56 | } 57 | 58 | func TestDecode(t *testing.T) { 59 | for _, test := range decodeTests { 60 | in := readTestFile(t, test.fileIn) 61 | in = in[:len(in)-1] 62 | out := new(bytes.Buffer) 63 | if err := run(test.metaOnly, test.outMediaType, in, out); err != nil { 64 | t.Fatalf("%v for %+v", err, test) 65 | } 66 | assertMatchesFile(t, out, test.fileExpected) 67 | } 68 | } 69 | 70 | func assertMatchesFile(t *testing.T, out *bytes.Buffer, filename string) { 71 | b := out.Bytes() 72 | expected := readTestFile(t, filename) 73 | if !bytes.Equal(b, expected) { 74 | t.Errorf(`got input of length %d, wanted input of length %d 75 | ==== BEGIN RECIEVED FILE ==== 76 | %s 77 | ====END RECIEVED FILE ==== 78 | ====BEGIN EXPECTED FILE (%s) ==== 79 | %s 80 | ==== END EXPECTED FILE ==== 81 | `, len(b), len(expected), b, filename, expected) 82 | } 83 | } 84 | 85 | func readTestFile(t *testing.T, filename string) []byte { 86 | f, err := os.Open(filename) 87 | if err != nil { 88 | t.Fatalf("failed to open test data file: %v", err) 89 | } 90 | in, err := ioutil.ReadAll(f) 91 | if err != nil { 92 | t.Fatalf("failed to read test data file, %v", err) 93 | } 94 | if strings.HasSuffix(filename, "yaml") { 95 | r, err := regexp.Compile("#[^\n]*\n") 96 | if err != nil { 97 | t.Fatalf("failed to strip comments from yaml file, %v", err) 98 | } 99 | return r.ReplaceAll(in, []byte("")) 100 | } 101 | return in 102 | } 103 | 104 | func readTestFileAsKv(t *testing.T, filename string) *mvccpb.KeyValue { 105 | kv, err := extractKvFromLeafItem(readTestFile(t, filename)) 106 | if err != nil { 107 | t.Fatalf("failed to extract etcd key-value record from boltdb leaf item: %s", err) 108 | } 109 | return kv 110 | } 111 | -------------------------------------------------------------------------------- /cmd/testdata/yaml/pod.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Pod 16 | metadata: 17 | annotations: 18 | kubernetes.io/created-by: | 19 | {"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"Job","namespace":"default","name":"pi","uid":"a4acc46c-5b56-11e7-8d4b-42010a800002","apiVersion":"batch","resourceVersion":"82647"}} 20 | kubernetes.io/limit-ranger: 'LimitRanger plugin set: cpu request for container 21 | pi' 22 | creationTimestamp: "2017-06-27T16:35:34Z" 23 | generateName: pi- 24 | labels: 25 | controller-uid: a4acc46c-5b56-11e7-8d4b-42010a800002 26 | job-name: pi 27 | name: pi-dqtsw 28 | namespace: default 29 | ownerReferences: 30 | - apiVersion: batch/v1 31 | blockOwnerDeletion: true 32 | controller: true 33 | kind: Job 34 | name: pi 35 | uid: a4acc46c-5b56-11e7-8d4b-42010a800002 36 | selfLink: /api/v1/namespaces/default/pods/pi-dqtsw 37 | uid: a4adc7ca-5b56-11e7-8d4b-42010a800002 38 | spec: 39 | containers: 40 | - command: 41 | - perl 42 | - -Mbignum=bpi 43 | - -wle 44 | - print bpi(2000) 45 | image: perl 46 | imagePullPolicy: Always 47 | name: pi 48 | resources: 49 | requests: 50 | cpu: 100m 51 | terminationMessagePath: /dev/termination-log 52 | terminationMessagePolicy: File 53 | volumeMounts: 54 | - mountPath: /var/run/secrets/kubernetes.io/serviceaccount 55 | name: default-token-nm27w 56 | readOnly: true 57 | dnsPolicy: ClusterFirst 58 | nodeName: kubernetes-minion-group-vlql 59 | restartPolicy: Never 60 | schedulerName: default-scheduler 61 | securityContext: {} 62 | serviceAccount: default 63 | serviceAccountName: default 64 | terminationGracePeriodSeconds: 30 65 | tolerations: 66 | - effect: NoExecute 67 | key: node.alpha.kubernetes.io/notReady 68 | operator: Exists 69 | tolerationSeconds: 300 70 | - effect: NoExecute 71 | key: node.alpha.kubernetes.io/unreachable 72 | operator: Exists 73 | tolerationSeconds: 300 74 | volumes: 75 | - name: default-token-nm27w 76 | secret: 77 | defaultMode: 420 78 | secretName: default-token-nm27w 79 | status: 80 | conditions: 81 | - lastProbeTime: null 82 | lastTransitionTime: "2017-06-27T16:35:34Z" 83 | reason: PodCompleted 84 | status: "True" 85 | type: Initialized 86 | - lastProbeTime: null 87 | lastTransitionTime: "2017-06-27T16:36:07Z" 88 | reason: PodCompleted 89 | status: "False" 90 | type: Ready 91 | - lastProbeTime: null 92 | lastTransitionTime: "2017-06-27T16:35:34Z" 93 | status: "True" 94 | type: PodScheduled 95 | containerStatuses: 96 | - containerID: docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117 97 | image: perl:latest 98 | imageID: docker://sha256:9fc8e8ba0b3a067188ac46cf51a25021a73e3b927e9637fab48663813c457612 99 | lastState: {} 100 | name: pi 101 | ready: false 102 | restartCount: 0 103 | state: 104 | terminated: 105 | containerID: docker://ef1d307d0a12f232b88037c61f577411b618f52975a2143801e5e49c4d0b0117 106 | exitCode: 0 107 | finishedAt: "2017-06-27T16:36:07Z" 108 | reason: Completed 109 | startedAt: "2017-06-27T16:36:01Z" 110 | hostIP: 10.128.0.4 111 | phase: Succeeded 112 | podIP: 10.244.2.8 113 | qosClass: Burstable 114 | startTime: "2017-06-27T16:35:34Z" 115 | -------------------------------------------------------------------------------- /pkg/encoding/scheme.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | import ( 20 | "os" 21 | 22 | admissionv1beta1 "k8s.io/api/admission/v1beta1" 23 | 24 | admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" 25 | admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" 26 | 27 | appsv1 "k8s.io/api/apps/v1" 28 | appsv1beta1 "k8s.io/api/apps/v1beta1" 29 | appsv1beta2 "k8s.io/api/apps/v1beta2" 30 | 31 | authenticationv1 "k8s.io/api/authentication/v1" 32 | authenticationv1beta1 "k8s.io/api/authentication/v1beta1" 33 | 34 | authorizationv1 "k8s.io/api/authorization/v1" 35 | authorizationv1beta1 "k8s.io/api/authorization/v1beta1" 36 | 37 | autoscalingv1 "k8s.io/api/autoscaling/v1" 38 | autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1" 39 | 40 | batchv1 "k8s.io/api/batch/v1" 41 | batchv1beta1 "k8s.io/api/batch/v1beta1" 42 | batchv2alpha1 "k8s.io/api/batch/v2alpha1" 43 | 44 | certificatesv1beta1 "k8s.io/api/certificates/v1beta1" 45 | 46 | corev1 "k8s.io/api/core/v1" 47 | 48 | eventsv1beta1 "k8s.io/api/events/v1beta1" 49 | 50 | extensionsv1beta1 "k8s.io/api/extensions/v1beta1" 51 | 52 | imagepolicyv1alpha1 "k8s.io/api/imagepolicy/v1alpha1" 53 | 54 | networkingv1 "k8s.io/api/networking/v1" 55 | 56 | policyv1beta1 "k8s.io/api/policy/v1beta1" 57 | 58 | rbacv1 "k8s.io/api/rbac/v1" 59 | rbacv1alpha1 "k8s.io/api/rbac/v1alpha1" 60 | rbacv1beta1 "k8s.io/api/rbac/v1beta1" 61 | 62 | schedulingv1alpha1 "k8s.io/api/scheduling/v1alpha1" 63 | 64 | settingsv1alpha1 "k8s.io/api/settings/v1alpha1" 65 | 66 | storagev1 "k8s.io/api/storage/v1" 67 | storagev1alpha1 "k8s.io/api/storage/v1alpha1" 68 | storagev1beta1 "k8s.io/api/storage/v1beta1" 69 | 70 | "k8s.io/apimachinery/pkg/apimachinery/registered" 71 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 72 | runtime "k8s.io/apimachinery/pkg/runtime" 73 | schema "k8s.io/apimachinery/pkg/runtime/schema" 74 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 75 | ) 76 | 77 | var Registry = registered.NewOrDie(os.Getenv("KUBE_API_VERSIONS")) 78 | var Scheme = runtime.NewScheme() 79 | var Codecs = serializer.NewCodecFactory(Scheme) 80 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 81 | 82 | func init() { 83 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 84 | AddToScheme(Scheme) 85 | } 86 | 87 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 88 | // of clientsets, like in: 89 | // 90 | // import ( 91 | // "k8s.io/client-go/kubernetes" 92 | // clientsetscheme "k8s.io/client-go/kuberentes/scheme" 93 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 94 | // ) 95 | // 96 | // kclientset, _ := kubernetes.NewForConfig(c) 97 | // aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 98 | // 99 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 100 | // correctly. 101 | func AddToScheme(scheme *runtime.Scheme) { 102 | admissionv1beta1.AddToScheme(scheme) 103 | 104 | admissionregistrationv1alpha1.AddToScheme(scheme) 105 | admissionregistrationv1beta1.AddToScheme(scheme) 106 | 107 | appsv1.AddToScheme(scheme) 108 | appsv1beta1.AddToScheme(scheme) 109 | appsv1beta2.AddToScheme(scheme) 110 | 111 | authenticationv1.AddToScheme(scheme) 112 | authenticationv1beta1.AddToScheme(scheme) 113 | 114 | authorizationv1.AddToScheme(scheme) 115 | authorizationv1beta1.AddToScheme(scheme) 116 | 117 | autoscalingv1.AddToScheme(scheme) 118 | autoscalingv2beta1.AddToScheme(scheme) 119 | 120 | batchv1.AddToScheme(scheme) 121 | batchv1beta1.AddToScheme(scheme) 122 | batchv2alpha1.AddToScheme(scheme) 123 | 124 | certificatesv1beta1.AddToScheme(scheme) 125 | 126 | corev1.AddToScheme(scheme) 127 | 128 | eventsv1beta1.AddToScheme(scheme) 129 | 130 | extensionsv1beta1.AddToScheme(scheme) 131 | 132 | imagepolicyv1alpha1.AddToScheme(scheme) 133 | 134 | networkingv1.AddToScheme(scheme) 135 | 136 | policyv1beta1.AddToScheme(scheme) 137 | 138 | rbacv1.AddToScheme(scheme) 139 | rbacv1alpha1.AddToScheme(scheme) 140 | rbacv1beta1.AddToScheme(scheme) 141 | 142 | schedulingv1alpha1.AddToScheme(scheme) 143 | 144 | settingsv1alpha1.AddToScheme(scheme) 145 | 146 | storagev1.AddToScheme(scheme) 147 | storagev1alpha1.AddToScheme(scheme) 148 | storagev1beta1.AddToScheme(scheme) 149 | } 150 | -------------------------------------------------------------------------------- /pkg/data/data_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package data 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | const ( 24 | dbFile = "testdata/boltdb/db" 25 | dbWithCrdFile = "testdata/boltdb/db-with-crd" 26 | ) 27 | 28 | func TestListKeySummariesFilters(t *testing.T) { 29 | cases := []struct { 30 | file string 31 | name string 32 | filters []Filter 33 | expectedKeys []string 34 | }{ 35 | { 36 | file: dbFile, 37 | name: "nofilters", 38 | filters: []Filter{}, 39 | expectedKeys: []string{"/registry/jobs/default/pi", "/registry/namespaces/default", "/registry/pods/default/pi-dqtsw", "compact_rev_key"}, 40 | }, 41 | { 42 | file: dbFile, 43 | name: "prefixfilter", 44 | filters: []Filter{NewPrefixFilter("/registry/jobs")}, 45 | expectedKeys: []string{"/registry/jobs/default/pi"}, 46 | }, 47 | { 48 | file: dbFile, 49 | name: "namespacefilter", 50 | filters: []Filter{mustBuildFilter(&FieldConstraint{lhs: ".Value.metadata.namespace", op: Equals, rhs: "default"})}, 51 | expectedKeys: []string{"/registry/jobs/default/pi", "/registry/pods/default/pi-dqtsw"}, 52 | }, 53 | { 54 | file: dbFile, 55 | name: "allfilters", 56 | filters: []Filter{NewPrefixFilter("/registry/jobs"), mustBuildFilter(&FieldConstraint{lhs: ".Value.metadata.namespace", op: Equals, rhs: "default"})}, 57 | expectedKeys: []string{"/registry/jobs/default/pi"}, 58 | }, 59 | { 60 | file: dbWithCrdFile, 61 | name: "crd", 62 | filters: []Filter{ 63 | NewPrefixFilter("/registry/apiextensions.k8s.io/customresourcedefinitions"), 64 | mustBuildFilter(&FieldConstraint{lhs: ".TypeMeta.APIVersion", op: Equals, rhs: "apiextensions.k8s.io/v1beta1"}), 65 | mustBuildFilter(&FieldConstraint{lhs: ".TypeMeta.Kind", op: Equals, rhs: "CustomResourceDefinition"}), 66 | }, 67 | expectedKeys: []string{"/registry/apiextensions.k8s.io/customresourcedefinitions/crontabs.stable.example.com"}, 68 | }, 69 | { 70 | file: dbWithCrdFile, 71 | name: "cr", 72 | filters: []Filter{ 73 | NewPrefixFilter("/registry/stable.example.com/crontabs"), 74 | mustBuildFilter(&FieldConstraint{lhs: ".TypeMeta.APIVersion", op: Equals, rhs: "stable.example.com/v1"}), 75 | mustBuildFilter(&FieldConstraint{lhs: ".TypeMeta.Kind", op: Equals, rhs: "CronTab"}), 76 | }, 77 | expectedKeys: []string{"/registry/stable.example.com/crontabs/default/my-new-cron-object"}, 78 | }, 79 | } 80 | for _, tt := range cases { 81 | t.Run(tt.name, func(t *testing.T) { 82 | missingKeys := map[string]struct{}{} 83 | for _, key := range tt.expectedKeys { 84 | missingKeys[key] = struct{}{} 85 | } 86 | unexpectedKeys := map[string]struct{}{} 87 | results, err := ListKeySummaries(tt.file, tt.filters, ProjectEverything, 0) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | for _, result := range results { 92 | if _, ok := missingKeys[result.Key]; ok { 93 | delete(missingKeys, result.Key) 94 | } else { 95 | unexpectedKeys[result.Key] = struct{}{} 96 | } 97 | } 98 | if len(unexpectedKeys) != 0 { 99 | t.Errorf("got %d unexpected keys: %v, expected none", len(unexpectedKeys), unexpectedKeys) 100 | } 101 | if len(missingKeys) != 0 { 102 | t.Errorf("got %d missing keys: %v, expected none", len(missingKeys), missingKeys) 103 | } 104 | }) 105 | } 106 | } 107 | 108 | func TestParseFilters(t *testing.T) { 109 | cases := []struct { 110 | name string 111 | rawFilter string 112 | expected []*FieldConstraint 113 | }{ 114 | { 115 | name: "namespace-equals", 116 | rawFilter: ".Value.metadata.namespace=default", 117 | expected: []*FieldConstraint{&FieldConstraint{lhs: ".Value.metadata.namespace", op: Equals, rhs: "default"}}, 118 | }, 119 | { 120 | name: "2-filters", 121 | rawFilter: ".Value.metadata.namespace=default,.Value.metadata.name=example", 122 | expected: []*FieldConstraint{ 123 | &FieldConstraint{lhs: ".Value.metadata.namespace", op: Equals, rhs: "default"}, 124 | &FieldConstraint{lhs: ".Value.metadata.name", op: Equals, rhs: "example"}, 125 | }, 126 | }, 127 | { 128 | name: "whitespace", 129 | rawFilter: " .Value.metadata.namespace=default\t, .Value.metadata.name=example\n", 130 | expected: []*FieldConstraint{ 131 | &FieldConstraint{lhs: ".Value.metadata.namespace", op: Equals, rhs: "default"}, 132 | &FieldConstraint{lhs: ".Value.metadata.name", op: Equals, rhs: "example"}, 133 | }, 134 | }, 135 | } 136 | for _, tt := range cases { 137 | t.Run(tt.name, func(t *testing.T) { 138 | filters, err := ParseFilters(tt.rawFilter) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | unexpected := map[FieldConstraint]struct{}{} 143 | missing := map[FieldConstraint]struct{}{} 144 | 145 | for _, expected := range tt.expected { 146 | missing[*expected] = struct{}{} 147 | } 148 | 149 | for _, filter := range filters { 150 | fc := filter.(*FieldFilter).FieldConstraint 151 | if _, ok := missing[*fc]; ok { 152 | delete(missing, *fc) 153 | } else { 154 | unexpected[*fc] = struct{}{} 155 | } 156 | } 157 | if len(unexpected) != 0 { 158 | t.Errorf("got %d unexpected filters: %#+v, expected none", len(unexpected), unexpected) 159 | } 160 | if len(missing) != 0 { 161 | t.Errorf("got %d missing filters: %#+v, expected none", len(missing), missing) 162 | } 163 | }) 164 | } 165 | } 166 | 167 | func mustBuildFilter(fc *FieldConstraint) Filter { 168 | filter, err := fc.BuildFilter() 169 | if err != nil { 170 | panic(err) 171 | } 172 | return filter 173 | } 174 | -------------------------------------------------------------------------------- /cmd/decode.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "io/ioutil" 23 | "os" 24 | 25 | "bufio" 26 | "bytes" 27 | "encoding/hex" 28 | 29 | "github.com/jpbetz/auger/pkg/encoding" 30 | "github.com/spf13/cobra" 31 | ) 32 | 33 | var ( 34 | decodeLong = ` 35 | Decodes kubernetes objects from the binary key-value store encoding used 36 | with etcd 3+. 37 | 38 | Outputs data in a variety of formats including YAML, JSON and Protobuf. 39 | 40 | YAML and JSON output conversion is performed using api-machinery 41 | serializers and type declarations from the version of kubernetes 42 | source this tool was built against. As a result, output may differ 43 | from what is stored if the type declarations from the kubernetes 44 | version that stored the data and the version of this tool are not 45 | identical. If a type declaration has an incompatible change, or if a 46 | type has been removed entirely, conversion may fail. 47 | 48 | The binary format contains an "encoding prefix", an envelope (defined 49 | by runtime.Unknown), and a protobuf payload. The envelope contains 50 | 'meta' fields identifying the payload type. 51 | 52 | Protobuf output requires no conversions and returns the exact bytes 53 | of the protobuf payload.` 54 | 55 | decodeExample = ` 56 | ETCDCTL_API=3 etcdctl get /registry/pods/default/ \ 57 | --print-value-only | auger decode` 58 | ) 59 | 60 | var decodeCmd = &cobra.Command{ 61 | Use: "decode", 62 | Short: "Decode objects from the kubernetes binary key-value store encoding.", 63 | Long: decodeLong, 64 | Example: decodeExample, 65 | RunE: func(cmd *cobra.Command, args []string) error { 66 | return validateAndRun() 67 | }, 68 | } 69 | 70 | type decodeOptions struct { 71 | out string 72 | metaOnly bool 73 | inputFilename string 74 | batchProcess bool // special flag to handle incoming etcd-dump-logs output 75 | } 76 | 77 | var options *decodeOptions = &decodeOptions{} 78 | 79 | func init() { 80 | RootCmd.AddCommand(decodeCmd) 81 | decodeCmd.Flags().StringVarP(&options.out, "output", "o", "yaml", "Output format. One of: json|yaml|proto") 82 | decodeCmd.Flags().BoolVar(&options.metaOnly, "meta-only", false, "Output only content type and metadata fields") 83 | decodeCmd.Flags().StringVar(&options.inputFilename, "file", "", "Filename to read storage encoded data from") 84 | decodeCmd.Flags().BoolVar(&options.batchProcess, "batch-process", false, "If set, deccode batch of objects from os.Stdin") 85 | } 86 | 87 | // Validate the command line flags and run the command. 88 | func validateAndRun() error { 89 | outMediaType, err := encoding.ToMediaType(options.out) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | if options.batchProcess { 95 | return runInBatchMode(options.metaOnly, outMediaType, os.Stdout) 96 | } 97 | 98 | in, err := readInput(options.inputFilename) 99 | if len(in) == 0 { 100 | return fmt.Errorf("no input data") 101 | } 102 | if err != nil { 103 | return err 104 | } 105 | 106 | return run(options.metaOnly, outMediaType, in, os.Stdout) 107 | } 108 | 109 | // runInBatchMode runs when batchProcess is set to true. 110 | // Original requirement is from etcd WAL log analysis tool etcd-dump-logs, 111 | // (etcd-dump-logs: add decoder support #9790 https://github.com/coreos/etcd/pull/9790) 112 | // to serve as a decoder to decode k8s objects from each entry of the etcd WAL log. 113 | // Read data from os.Stdin, decode object line by line, and print out to os.Stdout. 114 | // 115 | // In etcd-dump-logs, when batchProcess mode is on, two columns are listed for decoder: 116 | // "decoder_status" and "decoded_data". In this case, Auger has two possible output: 117 | // 1. "OK" for decoder_status, actual decoded data for decoded_data 118 | // 2. "ERROR:Auger error" for decoder_status, decoded_data column would be empty in this case. 119 | func runInBatchMode(metaOnly bool, outMediaType string, out io.Writer) (err error) { 120 | inputReader := bufio.NewReader(os.Stdin) 121 | lineNum := 0 122 | for { 123 | input, err := inputReader.ReadBytes(byte('\n')) 124 | if err == io.EOF { 125 | return nil 126 | } 127 | if err != nil { 128 | return fmt.Errorf("error reading --batch-process input: %v\n", err) 129 | } 130 | 131 | input = stripNewline(input) 132 | decodedinput, err := hex.DecodeString(string(input)) 133 | if err != nil { 134 | return fmt.Errorf("error decoding input on line %d of --batch-process input: %v", lineNum, err) 135 | } 136 | inMediaType, decodedinput, err := encoding.DetectAndExtract(decodedinput) 137 | if err != nil { 138 | fmt.Fprintf(out, "ERROR:%v|\n", err) 139 | continue 140 | } 141 | 142 | if metaOnly { 143 | buf := bytes.NewBufferString("") 144 | err = encoding.DecodeSummary(inMediaType, decodedinput, buf) 145 | if err != nil { 146 | 147 | fmt.Fprintf(out, "ERROR:%v|\n", err) 148 | } else { 149 | fmt.Fprintf(out, "OK|%s\n", buf.String()) 150 | } 151 | continue 152 | } 153 | 154 | buf := bytes.NewBufferString("") 155 | _, err = encoding.Convert(inMediaType, outMediaType, decodedinput, buf) 156 | if err != nil { 157 | fmt.Fprintf(out, "ERROR:%v|\n", err) 158 | } else { 159 | fmt.Fprintf(out, "OK|%s\n", buf.String()) 160 | } 161 | 162 | lineNum++ 163 | } 164 | } 165 | 166 | // Run the decode command line. 167 | func run(metaOnly bool, outMediaType string, in []byte, out io.Writer) error { 168 | inMediaType, in, err := encoding.DetectAndExtract(in) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | if metaOnly { 174 | return encoding.DecodeSummary(inMediaType, in, out) 175 | } 176 | 177 | _, err = encoding.Convert(inMediaType, outMediaType, in, out) 178 | return err 179 | } 180 | 181 | // Readinput reads command line input, either from a provided input file or from stdin. 182 | func readInput(inputFilename string) ([]byte, error) { 183 | if inputFilename != "" { 184 | data, err := ioutil.ReadFile(inputFilename) 185 | if err != nil { 186 | return nil, fmt.Errorf("error reading input file %s: %v", inputFilename, err) 187 | } 188 | data = stripNewline(data) 189 | return data, nil 190 | } 191 | 192 | stat, err := os.Stdin.Stat() 193 | if err != nil { 194 | return nil, fmt.Errorf("stdin error: %s", err) 195 | } 196 | if (stat.Mode() & os.ModeCharDevice) != 0 { 197 | fmt.Fprintln(os.Stderr, "warn: waiting on stdin from tty") 198 | } 199 | 200 | stdin, err := ioutil.ReadAll(os.Stdin) 201 | if err != nil { 202 | return nil, fmt.Errorf("unable to read data from stdin: %v", err) 203 | } 204 | // Etcd --print-value-only includes an extranous newline even for binary values. 205 | // We can safely strip it off since none of the valid inputs this code processes 206 | // have trailing newline byte. 207 | stdin = stripNewline(stdin) 208 | return stdin, nil 209 | } 210 | 211 | func stripNewline(d []byte) []byte { 212 | if len(d) > 0 && d[len(d)-1] == '\n' { 213 | return d[:len(d)-1] 214 | } else { 215 | return d 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/bbolt v1.3.1-coreos.3 h1:sP70znHBV8469pbVsmR2G6wvd2oQwgxtgWyZvV3KVBo= 2 | github.com/coreos/bbolt v1.3.1-coreos.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 3 | github.com/coreos/etcd v3.1.11+incompatible h1:8Zwb+fsI+3wXGp23Pjn1eEFE/3P5ib5yXVoAMRxFOgM= 4 | github.com/coreos/etcd v3.1.11+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 h1:ZktWZesgun21uEDrwW7iEV1zPCGQldM2atlJZ3TdvVM= 9 | github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 10 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 11 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 12 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 13 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 14 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7 h1:VoRHccBV7zFWlO/I85D1ke/x5Ak9Y7gKm9gkf7LHNhI= 15 | github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 h1:ScAXWS+TR6MZKex+7Z8rneuSJH+FSDqd6ocQyl+ZHo4= 17 | github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= 18 | github.com/google/safetext v0.0.0-20220914124124-e18e3fe012bf h1:F57w3YBUKQis5sQo3d8O6JhlvF4/p3zRfRtpIfSnBu4= 19 | github.com/google/safetext v0.0.0-20220914124124-e18e3fe012bf/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= 20 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 21 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 22 | github.com/json-iterator/go v0.0.0-20171212105241-13f86432b882 h1:Xqf4pbZo/Iztd32Rrir1VZsAvz7r85JmWPHR95V+Flo= 23 | github.com/json-iterator/go v0.0.0-20171212105241-13f86432b882/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 24 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 25 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 26 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 27 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 28 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 29 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 30 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 31 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 32 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 33 | github.com/spf13/cobra v0.0.1 h1:zZh3X5aZbdnoj+4XkaBxKfhO4ot82icYdhhREIAXIj8= 34 | github.com/spf13/cobra v0.0.1/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 35 | github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI= 36 | github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 39 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 40 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 41 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 42 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 43 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 44 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 45 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 46 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 47 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 48 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 49 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 50 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= 51 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 52 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 53 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 56 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 58 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 59 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 60 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 61 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 62 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 63 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 64 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 65 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 66 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 67 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 69 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 70 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 71 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 72 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 73 | gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= 74 | gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 75 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 76 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | k8s.io/api v0.0.0-20180521142803-feb48db456a5 h1:ZkJvJIvl22AqkIYbow7+ZkJCZ/Vf5TnLyJ1Q5UpFXEI= 81 | k8s.io/api v0.0.0-20180521142803-feb48db456a5/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= 82 | k8s.io/apimachinery v0.0.0-20180515182440-31dade610c05 h1:IxbzCht0hGNBVprna3ou1lB+jvFGT2Sh83htT2jL4sk= 83 | k8s.io/apimachinery v0.0.0-20180515182440-31dade610c05/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= 84 | -------------------------------------------------------------------------------- /pkg/encoding/encoding.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "fmt" 23 | "io" 24 | 25 | yaml "gopkg.in/yaml.v2" 26 | 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "k8s.io/apimachinery/pkg/runtime/serializer" 30 | ) 31 | 32 | const ( 33 | StorageBinaryMediaType = "application/vnd.kubernetes.storagebinary" 34 | ProtobufMediaType = "application/vnd.kubernetes.protobuf" 35 | YamlMediaType = "application/yaml" 36 | JsonMediaType = "application/json" 37 | 38 | ProtobufShortname = "proto" 39 | YamlShortname = "yaml" 40 | JsonShortname = "json" 41 | ) 42 | 43 | // See k8s.io/apimachinery/pkg/runtime/serializer/protobuf.go 44 | var ProtoEncodingPrefix = []byte{0x6b, 0x38, 0x73, 0x00} 45 | 46 | // ToMediaType maps 'out' flag values to corresponding mime types. 47 | func ToMediaType(out string) (string, error) { 48 | switch out { 49 | case YamlShortname: 50 | return YamlMediaType, nil 51 | case JsonShortname: 52 | return JsonMediaType, nil 53 | case ProtobufShortname: 54 | return ProtobufMediaType, nil 55 | default: 56 | return "", fmt.Errorf("unrecognized 'out' flag value: %v", out) 57 | } 58 | } 59 | 60 | // DetectAndConvert first detects the media type of the in param data and then converts it from 61 | // the kv store encoded data to the given output format using kubernetes' api machinery to 62 | // perform the conversion. 63 | func DetectAndConvert(outMediaType string, in []byte, out io.Writer) (*runtime.TypeMeta, error) { 64 | inMediaType, in, err := DetectAndExtract(in) 65 | if err != nil { 66 | return nil, err 67 | } 68 | return Convert(inMediaType, outMediaType, in, out) 69 | } 70 | 71 | // Convert from kv store encoded data to the given output format using kubernetes' api machinery to 72 | // perform the conversion. 73 | func Convert(inMediaType, outMediaType string, in []byte, out io.Writer) (*runtime.TypeMeta, error) { 74 | if inMediaType == StorageBinaryMediaType && outMediaType == ProtobufMediaType { 75 | return nil, DecodeRaw(in, out) 76 | } 77 | 78 | if inMediaType == ProtobufMediaType && outMediaType == StorageBinaryMediaType { 79 | return nil, fmt.Errorf("unsupported conversion: protobuf to kubernetes binary storage representation") 80 | } 81 | 82 | typeMeta, err := decodeTypeMeta(inMediaType, in) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | var encoded []byte 88 | if inMediaType == outMediaType { 89 | // Assumes that the stored version is "correct". Primarily a short cut to allow CRDs to work. 90 | encoded = in 91 | if outMediaType == JsonMediaType { 92 | encoded = append(encoded, '\n') 93 | } 94 | } else { 95 | inCodec, err := newCodec(typeMeta, inMediaType) 96 | if err != nil { 97 | return nil, err 98 | } 99 | outCodec, err := newCodec(typeMeta, outMediaType) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | obj, err := runtime.Decode(inCodec, in) 105 | if err != nil { 106 | return nil, fmt.Errorf("error decoding from %s: %s", inMediaType, err) 107 | } 108 | 109 | encoded, err = runtime.Encode(outCodec, obj) 110 | if err != nil { 111 | return nil, fmt.Errorf("error encoding to %s: %s", outMediaType, err) 112 | } 113 | } 114 | 115 | _, err = out.Write(encoded) 116 | if err != nil { 117 | return nil, err 118 | } 119 | return typeMeta, nil 120 | } 121 | 122 | // DetectAndExtract searches the the start of either json of protobuf data, and, if found, returns the mime type and data. 123 | func DetectAndExtract(in []byte) (string, []byte, error) { 124 | if pb, ok := tryFindProto(in); ok { 125 | return StorageBinaryMediaType, pb, nil 126 | } 127 | if rawJs, ok := tryFindJson(in); ok { 128 | js, err := rawJs.MarshalJSON() 129 | if err != nil { 130 | return "", nil, err 131 | } 132 | return JsonMediaType, js, nil 133 | } 134 | return "", nil, fmt.Errorf("error reading input, does not appear to contain valid JSON or binary data") 135 | } 136 | 137 | // TryFindProto searches for the 'k8s\0' prefix, and, if found, returns the data starting with the prefix. 138 | func tryFindProto(in []byte) ([]byte, bool) { 139 | i := bytes.Index(in, ProtoEncodingPrefix) 140 | if i >= 0 && i < len(in) { 141 | return in[i:], true 142 | } 143 | return nil, false 144 | } 145 | 146 | const jsonStartChars = "{[" 147 | 148 | // TryFindJson searches for the start of a valid json substring, and, if found, returns the json. 149 | func tryFindJson(in []byte) (*json.RawMessage, bool) { 150 | var js json.RawMessage 151 | 152 | i := bytes.IndexAny(in, jsonStartChars) 153 | for i >= 0 && i < len(in) { 154 | in = in[i:] 155 | if len(in) < 2 { 156 | break 157 | } 158 | err := json.Unmarshal(in, &js) 159 | if err == nil { 160 | return &js, true 161 | } 162 | in = in[1:] 163 | i = bytes.IndexAny(in, jsonStartChars) 164 | } 165 | return nil, false 166 | } 167 | 168 | // DecodeRaw decodes the raw payload bytes contained within the 'Unknown' protobuf envelope of 169 | // the given storage data. 170 | func DecodeRaw(in []byte, out io.Writer) error { 171 | unknown, err := DecodeUnknown(in) 172 | if err != nil { 173 | return err 174 | } 175 | _, err = out.Write(unknown.Raw) 176 | if err != nil { 177 | return fmt.Errorf("failed to write output: %v", err) 178 | } 179 | return nil 180 | } 181 | 182 | // DecodeSummary decodes the TypeMeta, ContentEncoding and ContentType fields from the 'Unknown' 183 | // protobuf envelope of the given storage data. 184 | func DecodeSummary(inMediaType string, in []byte, out io.Writer) error { 185 | typeMeta, err := decodeTypeMeta(inMediaType, in) 186 | if err != nil { 187 | return err 188 | } 189 | fmt.Fprintf(out, "TypeMeta.APIVersion: %s\n", typeMeta.APIVersion) 190 | fmt.Fprintf(out, "TypeMeta.Kind: %s\n", typeMeta.Kind) 191 | return nil 192 | } 193 | 194 | // DecodeUnknown decodes the Unknown protobuf type from the given storage data. 195 | func DecodeUnknown(in []byte) (*runtime.Unknown, error) { 196 | if len(in) < 4 { 197 | return nil, fmt.Errorf("input too short, expected 4 byte proto encoding prefix but got %v", in) 198 | } 199 | if !bytes.Equal(in[:4], ProtoEncodingPrefix) { 200 | return nil, fmt.Errorf("first 4 bytes %v, do not match proto encoding prefix of %v", in[:4], ProtoEncodingPrefix) 201 | } 202 | data := in[4:] 203 | 204 | unknown := &runtime.Unknown{} 205 | if err := unknown.Unmarshal(data); err != nil { 206 | return nil, err 207 | } 208 | return unknown, nil 209 | } 210 | 211 | // NewCodec creates a new kubernetes storage codec for encoding and decoding persisted data. 212 | func newCodec(typeMeta *runtime.TypeMeta, mediaType string) (runtime.Codec, error) { 213 | // For api machinery purposes, we treat StorageBinaryMediaType as ProtobufMediaType 214 | if mediaType == StorageBinaryMediaType { 215 | mediaType = ProtobufMediaType 216 | } 217 | mediaTypes := Codecs.SupportedMediaTypes() 218 | 219 | info, ok := runtime.SerializerInfoForMediaType(mediaTypes, mediaType) 220 | if !ok { 221 | if len(mediaTypes) == 0 { 222 | return nil, fmt.Errorf("no serializers registered for %v", mediaTypes) 223 | } 224 | info = mediaTypes[0] 225 | } 226 | cfactory := serializer.DirectCodecFactory{CodecFactory: Codecs} 227 | gv, err := schema.ParseGroupVersion(typeMeta.APIVersion) 228 | if err != nil { 229 | return nil, fmt.Errorf("unable to parse meta APIVersion '%s': %s", typeMeta.APIVersion, err) 230 | } 231 | encoder := cfactory.EncoderForVersion(info.Serializer, gv) 232 | decoder := cfactory.DecoderToVersion(info.Serializer, gv) 233 | codec := cfactory.CodecForVersions(encoder, decoder, gv, gv) 234 | return codec, nil 235 | } 236 | 237 | // getTypeMeta gets the TypeMeta from the given data, either as JSON or Protobuf. 238 | func decodeTypeMeta(inMediaType string, in []byte) (*runtime.TypeMeta, error) { 239 | switch inMediaType { 240 | case JsonMediaType: 241 | return typeMetaFromJson(in) 242 | case StorageBinaryMediaType: 243 | return typeMetaFromBinaryStorage(in) 244 | case YamlMediaType: 245 | return typeMetaFromYaml(in) 246 | default: 247 | return nil, fmt.Errorf("unsupported inMediaType %s", inMediaType) 248 | } 249 | } 250 | 251 | func typeMetaFromJson(in []byte) (*runtime.TypeMeta, error) { 252 | var meta runtime.TypeMeta 253 | json.Unmarshal(in, &meta) 254 | return &meta, nil 255 | } 256 | 257 | func typeMetaFromBinaryStorage(in []byte) (*runtime.TypeMeta, error) { 258 | unknown, err := DecodeUnknown(in) 259 | if err != nil { 260 | return nil, err 261 | } 262 | return &unknown.TypeMeta, nil 263 | } 264 | 265 | func typeMetaFromYaml(in []byte) (*runtime.TypeMeta, error) { 266 | var meta runtime.TypeMeta 267 | yaml.Unmarshal(in, &meta) 268 | return &meta, nil 269 | } 270 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cmd/extract.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "os" 23 | "reflect" 24 | "strconv" 25 | "strings" 26 | 27 | "github.com/coreos/etcd/mvcc/mvccpb" 28 | "github.com/google/safetext/yamltemplate" 29 | "github.com/jpbetz/auger/pkg/data" 30 | "github.com/jpbetz/auger/pkg/encoding" 31 | "github.com/spf13/cobra" 32 | ) 33 | 34 | var ( 35 | extractLong = ` 36 | Extracts kubernetes data stored by etcd into boltdb '.db' files. 37 | 38 | May be used both to inspect the contents of a botldb file and to 39 | extract specific data entries. Data may be looked up either by etcd 40 | key and version, or by bolt page and item coordinates. 41 | 42 | Etcd must stopped when using this tool, or it will wait indefinitely 43 | for the '.db' file lock.` 44 | 45 | extractExample = ` 46 | # Find an etcd value by it's key and extract it from a boltdb file: 47 | auger extract -f -k /registry/pods/default/ 48 | 49 | # List the keys and size of all entries in etcd 50 | auger extract -f --fields=key,value-size 51 | 52 | # Extract a specific field from each kubernetes object 53 | auger extract -f --template="{{.Value.metadata.creationTimestamp}}" 54 | 55 | # Extract kubernetes objects using a filter 56 | auger extract -f --filter=".Value.metadata.namespace=kube-system" 57 | 58 | # Extract the etcd value stored in page 10, item 0 of a boltdb file: 59 | bolt page --item 0 --value-only 10 | auger extract --leaf-item 60 | 61 | # Extract the etcd key stored in page 10, item 0 of a boltdb file: 62 | bolt page --item 0 --value-only 10 | auger extract --leaf-item --print-key 63 | ` 64 | ) 65 | 66 | var extractCmd = &cobra.Command{ 67 | Use: "extract", 68 | Short: "Extracts kubernetes data from the boltdb '.db' files etcd persists to.", 69 | Long: extractLong, 70 | Example: extractExample, 71 | RunE: func(cmd *cobra.Command, args []string) error { 72 | return extractValidateAndRun() 73 | }, 74 | } 75 | 76 | type extractOptions struct { 77 | out string 78 | filename string 79 | key string 80 | version string 81 | revision int64 82 | keyPrefix string 83 | listVersions bool 84 | leafItem bool 85 | printKey bool 86 | metaSummary bool 87 | raw bool 88 | fields string 89 | template string 90 | filter string 91 | } 92 | 93 | var opts *extractOptions = &extractOptions{} 94 | 95 | func init() { 96 | RootCmd.AddCommand(extractCmd) 97 | extractCmd.Flags().StringVarP(&opts.out, "output", "o", "yaml", "Output format. One of: json|yaml|proto") 98 | extractCmd.Flags().StringVarP(&opts.filename, "file", "f", "", "Bolt DB '.db' filename") 99 | extractCmd.Flags().StringVarP(&opts.key, "key", "k", "", "Etcd object key to find in boltdb file") 100 | extractCmd.Flags().StringVarP(&opts.version, "version", "v", "", "Version of etcd key to find, defaults to latest version") 101 | extractCmd.Flags().Int64VarP(&opts.revision, "revision", "r", 0, "etcd revision to retrieve data at, if 0, the latest revision is used, defaults to 0") 102 | extractCmd.Flags().StringVar(&opts.keyPrefix, "keys-by-prefix", "", "List out all keys with the given prefix") 103 | extractCmd.Flags().BoolVar(&opts.listVersions, "list-versions", false, "List out all versions of the key, requires --key") 104 | extractCmd.Flags().BoolVar(&opts.leafItem, "leaf-item", false, "Read the input as a boltdb leaf page item.") 105 | extractCmd.Flags().BoolVar(&opts.printKey, "print-key", false, "Print the key of the matching entry") 106 | extractCmd.Flags().BoolVar(&opts.metaSummary, "meta-summary", false, "Print a summary of the metadata of the matching entry") 107 | extractCmd.Flags().BoolVar(&opts.raw, "raw", false, "Don't attempt to decode the etcd value") 108 | extractCmd.Flags().StringVar(&opts.fields, "fields", Key, fmt.Sprintf("Fields to include when listing entries, comma separated list of: %v", SummaryFields)) 109 | extractCmd.Flags().StringVar(&opts.template, "template", "", fmt.Sprintf("golang template to use when listing entries, see https://golang.org/pkg/text/template, template is provided an object with the fields: %v. The Value field contains the entire kubernetes resource object which also may be dereferenced using a dot seperated path.", templateFields())) 110 | extractCmd.Flags().StringVar(&opts.filter, "filter", "", fmt.Sprintf("Filter entries using a comma separated list of '=value' constraints. Fields used in filters use the same naming as --template fields, e.g. .Value.metadata.namespace")) 111 | } 112 | 113 | const ( 114 | Key = "key" 115 | ValueSize = "value-size" 116 | AllVersionsValueSize = "all-versions-value-size" 117 | VersionCount = "version-count" 118 | Value = "value" 119 | ) 120 | 121 | var SummaryFields = []string{Key, ValueSize, AllVersionsValueSize, VersionCount, Value} 122 | 123 | func templateFields() string { 124 | t := reflect.ValueOf(data.KeySummary{}).Type() 125 | names := []string{} 126 | for i := 0; i < t.NumField(); i++ { 127 | names = append(names, t.Field(i).Name) 128 | } 129 | return strings.Join(names, ", ") 130 | } 131 | 132 | func extractValidateAndRun() error { 133 | outMediaType, err := encoding.ToMediaType(opts.out) 134 | if err != nil { 135 | return fmt.Errorf("invalid --output %s: %s", opts.out, err) 136 | } 137 | out := os.Stdout 138 | hasKey := opts.key != "" 139 | hasVersion := opts.version != "" 140 | hasKeyPrefix := opts.keyPrefix != "" 141 | hasFields := opts.fields != Key 142 | hasTemplate := opts.template != "" 143 | 144 | switch { 145 | case opts.leafItem: 146 | raw, err := readInput(opts.filename) 147 | if err != nil { 148 | return fmt.Errorf("unable to read input: %s", err) 149 | } 150 | kv, err := extractKvFromLeafItem(raw) 151 | if err != nil { 152 | return fmt.Errorf("failed to extract etcd key-value record from boltdb leaf item: %s", err) 153 | } 154 | if opts.metaSummary { 155 | return printLeafItemSummary(kv, out) 156 | } else if opts.printKey { 157 | return printLeafItemKey(kv, out) 158 | } else { 159 | return printLeafItemValue(kv, outMediaType, out) 160 | } 161 | case hasKey && hasKeyPrefix: 162 | return fmt.Errorf("--keys-by-prefix and --key may not be used together") 163 | case hasKey && opts.listVersions: 164 | return printVersions(opts.filename, opts.key, out) 165 | case hasKey: 166 | return printValue(opts.filename, opts.key, opts.version, opts.raw, outMediaType, out) 167 | case !hasKey && opts.listVersions: 168 | return fmt.Errorf("--list-versions may only be used with --key") 169 | case !hasKey && hasVersion: 170 | return fmt.Errorf("--version may only be used with --key") 171 | case hasTemplate && hasFields: 172 | return fmt.Errorf("--template and --fields may not be used together") 173 | case hasTemplate: 174 | return printTemplateSummaries(opts.filename, opts.keyPrefix, opts.revision, opts.template, opts.filter, out) 175 | default: 176 | fields := strings.Split(opts.fields, ",") 177 | return printKeySummaries(opts.filename, opts.keyPrefix, opts.revision, fields, out) 178 | } 179 | } 180 | 181 | // printVersions writes all versions of the given key. 182 | func printVersions(filename string, key string, out io.Writer) error { 183 | versions, err := data.ListVersions(filename, key) 184 | if err != nil { 185 | return err 186 | } 187 | for _, v := range versions { 188 | fmt.Fprintf(out, "%d\n", v) 189 | } 190 | return nil 191 | } 192 | 193 | // printValue writes the value, in the desired media type, of the given key version. 194 | func printValue(filename string, key string, version string, raw bool, outMediaType string, out io.Writer) error { 195 | var v int64 196 | var err error 197 | if version == "" { 198 | versions, err := data.ListVersions(filename, key) 199 | if err != nil { 200 | return err 201 | } 202 | if len(versions) == 0 { 203 | return fmt.Errorf("No versions found for key: %s", key) 204 | } 205 | 206 | v = maxInSlice(versions) 207 | } else { 208 | v, err = strconv.ParseInt(version, 10, 64) 209 | if err != nil { 210 | return fmt.Errorf("version must be an int64, but got %s: %s", version, err) 211 | } 212 | } 213 | in, err := data.GetValue(filename, key, v) 214 | if err != nil { 215 | return err 216 | } 217 | if len(in) == 0 { 218 | return fmt.Errorf("0 byte value") 219 | } 220 | if raw { 221 | fmt.Fprintf(out, "%s\n", string(in)) 222 | return nil 223 | } 224 | _, err = encoding.DetectAndConvert(outMediaType, in, out) 225 | return err 226 | } 227 | 228 | // printLeafItemKey prints an etcd key for a given boltdb leaf item. 229 | func printLeafItemKey(kv *mvccpb.KeyValue, out io.Writer) error { 230 | fmt.Fprintf(out, "%s\n", kv.Key) 231 | return nil 232 | } 233 | 234 | // printLeafItemSummary prints etcd metadata summary 235 | func printLeafItemSummary(kv *mvccpb.KeyValue, out io.Writer) error { 236 | fmt.Fprintf(out, "Key: %s\n", kv.Key) 237 | fmt.Fprintf(out, "Version: %d\n", kv.Version) 238 | fmt.Fprintf(out, "CreateRevision: %d\n", kv.CreateRevision) 239 | fmt.Fprintf(out, "ModRevision: %d\n", kv.ModRevision) 240 | fmt.Fprintf(out, "Lease: %d\n", kv.Lease) 241 | return nil 242 | } 243 | 244 | // printLeafItemValue prints an etcd value for a given boltdb leaf item. 245 | func printLeafItemValue(kv *mvccpb.KeyValue, outMediaType string, out io.Writer) error { 246 | _, err := encoding.DetectAndConvert(outMediaType, kv.Value, out) 247 | return err 248 | } 249 | 250 | // printKeySummaries prints all keys in the db file with the given key prefix. 251 | func printKeySummaries(filename string, keyPrefix string, revision int64, fields []string, out io.Writer) error { 252 | if len(fields) == 0 { 253 | return fmt.Errorf("no fields provided, nothing to output.") 254 | } 255 | 256 | var hasKey bool 257 | var hasValue bool 258 | for _, field := range fields { 259 | switch field { 260 | case Key: 261 | hasKey = true 262 | case Value: 263 | hasValue = true 264 | } 265 | } 266 | proj := &data.KeySummaryProjection{HasKey: hasKey, HasValue: hasValue} 267 | summaries, err := data.ListKeySummaries(filename, []data.Filter{data.NewPrefixFilter(keyPrefix)}, proj, revision) 268 | if err != nil { 269 | return err 270 | } 271 | for _, s := range summaries { 272 | summary, err := summarize(s, fields) 273 | if err != nil { 274 | return err 275 | } 276 | fmt.Fprintf(out, "%s\n", summary) 277 | } 278 | return nil 279 | } 280 | 281 | // printTemplateSummaries prints out each KeySummary according to the given golang template. 282 | // See https://golang.org/pkg/text/template for details on the template format. 283 | func printTemplateSummaries(filename string, keyPrefix string, revision int64, templatestr string, filterstr string, out io.Writer) error { 284 | var err error 285 | t, err := yamltemplate.New("template").Parse(templatestr) 286 | if err != nil { 287 | return err 288 | } 289 | 290 | if len(templatestr) == 0 { 291 | return fmt.Errorf("no template provided, nothing to output.") 292 | } 293 | 294 | filters := []data.Filter{} 295 | if filterstr != "" { 296 | filters, err = data.ParseFilters(filterstr) 297 | if err != nil { 298 | return err 299 | } 300 | } 301 | 302 | // We don't have a simple way to determine if the template uses the key or value or not 303 | summaries, err := data.ListKeySummaries(filename, append(filters, data.NewPrefixFilter(keyPrefix)), &data.KeySummaryProjection{HasKey: true, HasValue: true}, revision) 304 | if err != nil { 305 | return err 306 | } 307 | for _, s := range summaries { 308 | err := t.Execute(out, s) 309 | if err != nil { 310 | return err 311 | } 312 | fmt.Fprintf(out, "\n") 313 | } 314 | return nil 315 | } 316 | 317 | func summarize(s *data.KeySummary, fields []string) (string, error) { 318 | values := make([]string, len(fields)) 319 | for i, field := range fields { 320 | switch field { 321 | case Key: 322 | values[i] = fmt.Sprintf("%s", s.Key) 323 | case ValueSize: 324 | values[i] = fmt.Sprintf("%d", s.Stats.ValueSize) 325 | case AllVersionsValueSize: 326 | values[i] = fmt.Sprintf("%d", s.Stats.AllVersionsValueSize) 327 | case VersionCount: 328 | values[i] = fmt.Sprintf("%d", s.Stats.VersionCount) 329 | case Value: 330 | values[i] = fmt.Sprintf("%s", s.ValueJson()) 331 | default: 332 | return "", fmt.Errorf("unrecognized field: %s", field) 333 | } 334 | } 335 | return strings.Join(values, " "), nil 336 | } 337 | 338 | func extractKvFromLeafItem(raw []byte) (*mvccpb.KeyValue, error) { 339 | kv := &mvccpb.KeyValue{} 340 | err := kv.Unmarshal(raw) 341 | if err != nil { 342 | return nil, err 343 | } 344 | return kv, nil 345 | } 346 | 347 | func maxInSlice(s []int64) int64 { 348 | r := int64(0) 349 | for _, e := range s { 350 | if e > r { 351 | r = e 352 | } 353 | } 354 | return r 355 | } 356 | -------------------------------------------------------------------------------- /pkg/data/data.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package data 18 | 19 | import ( 20 | "bytes" 21 | "encoding/binary" 22 | "encoding/json" 23 | "fmt" 24 | "hash/crc32" 25 | "sort" 26 | "strings" 27 | "text/template" 28 | 29 | bolt "github.com/coreos/bbolt" 30 | "github.com/coreos/etcd/mvcc/mvccpb" 31 | "github.com/jpbetz/auger/pkg/encoding" 32 | "k8s.io/apimachinery/pkg/runtime" 33 | ) 34 | 35 | // See etcd/mvcc/kvstore.go:keyBucketName 36 | var ( 37 | keyBucket = []byte("key") 38 | metaBucket = []byte("meta") 39 | 40 | consistentIndexKeyName = []byte("consistent_index") 41 | scheduledCompactKeyName = []byte("scheduledCompactRev") 42 | finishedCompactKeyName = []byte("finishedCompactRev") 43 | ) 44 | 45 | // KeySummary represents a kubernetes object stored in etcd. 46 | type KeySummary struct { 47 | Key string 48 | Version int64 49 | Value interface{} 50 | TypeMeta *runtime.TypeMeta 51 | Stats *KeySummaryStats 52 | } 53 | 54 | // KeySummaryStats provides high level statistics on a a particular kubernetes object stored in etcd. 55 | type KeySummaryStats struct { 56 | VersionCount int 57 | KeySize int 58 | ValueSize int 59 | AllVersionsKeySize int 60 | AllVersionsValueSize int 61 | } 62 | 63 | // ValueJson gets the json representation of a kubernetes object stored in etcd. 64 | func (ks *KeySummary) ValueJson() string { 65 | return rawJsonMarshal(ks.Value) 66 | } 67 | 68 | // KeySummaryProjection declares which optional fields to include in KeySummary results. 69 | type KeySummaryProjection struct { 70 | HasKey bool 71 | HasValue bool 72 | } 73 | 74 | var ProjectEverything = &KeySummaryProjection{HasKey: true, HasValue: true} 75 | 76 | // Filter declares an interface for filtering KeySummary results. 77 | type Filter interface { 78 | Accept(ks *KeySummary) (bool, error) 79 | } 80 | 81 | // PrefixFilter filter by key prefix. 82 | type PrefixFilter struct { 83 | prefix string 84 | } 85 | 86 | func NewPrefixFilter(prefix string) *PrefixFilter { 87 | return &PrefixFilter{prefix} 88 | } 89 | 90 | func (ff *PrefixFilter) Accept(ks *KeySummary) (bool, error) { 91 | return strings.HasPrefix(ks.Key, ff.prefix), nil 92 | } 93 | 94 | type ConstraintOp int 95 | 96 | const ( 97 | Equals ConstraintOp = iota 98 | ) 99 | 100 | func (co ConstraintOp) String() string { 101 | switch co { 102 | case Equals: 103 | return "=" 104 | default: 105 | return fmt.Sprintf("unrecognized enum value %d", co) 106 | } 107 | } 108 | 109 | type FieldConstraint struct { 110 | lhs string 111 | op ConstraintOp 112 | rhs string 113 | } 114 | 115 | func (fc *FieldConstraint) String() string { 116 | return fmt.Sprintf("%s%s%s", fc.lhs, fc.op.String(), fc.rhs) 117 | } 118 | 119 | func (fc *FieldConstraint) BuildFilter() (*FieldFilter, error) { 120 | t, err := template.New("filter-param").Parse("{{" + fc.lhs + "}}") 121 | if err != nil { 122 | return nil, err 123 | } 124 | return &FieldFilter{fc, t}, nil 125 | } 126 | 127 | // FieldFilter filters according to a field constraint. 128 | type FieldFilter struct { 129 | *FieldConstraint 130 | lhsTemplate *template.Template 131 | } 132 | 133 | func (ff *FieldFilter) Accept(ks *KeySummary) (bool, error) { 134 | buf := new(bytes.Buffer) 135 | err := ff.lhsTemplate.Execute(buf, ks) 136 | if err != nil { 137 | return false, fmt.Errorf("failed to look up field in filter %s: %v", ff.lhs, err) 138 | } 139 | val := buf.String() 140 | switch ff.op { 141 | case Equals: 142 | return val == ff.rhs, nil 143 | default: 144 | return false, fmt.Errorf("Unsupported filter operator: %s", ff.op.String()) 145 | } 146 | } 147 | 148 | type Checksum struct { 149 | Hash uint32 150 | Revision int64 151 | CompactRevision int64 152 | } 153 | 154 | // HashByRevision returns the checksum and revision. The checksum is of the live keyspace at a 155 | // particular revision. It is equivalent to performing a range request of all key-value pairs can 156 | // computing a hash of the data. If revision is 0, the latest revision is checksumed, else revision 157 | // is checksumed. The resulting hash is consistent particular revision in the presence of 158 | // compactions; so long as the revions itself has not been compacted, the hash never changes. 159 | func HashByRevision(filename string, revision int64) (Checksum, error) { 160 | db, err := bolt.Open(filename, 0400, &bolt.Options{}) 161 | if err != nil { 162 | return Checksum{}, err 163 | } 164 | defer db.Close() 165 | 166 | h := crc32.New(crc32.MakeTable(crc32.Castagnoli)) 167 | h.Write(keyBucket) 168 | 169 | compactRevision, err := getCompactRevision(db) 170 | if err != nil { 171 | return Checksum{}, err 172 | } 173 | latestRevision := compactRevision 174 | err = walkRevision(db, revision, func(r revKey, kv *mvccpb.KeyValue) (bool, error) { 175 | if r.main > latestRevision { 176 | latestRevision = r.main 177 | } 178 | h.Write(kv.Key) 179 | h.Write(kv.Value) 180 | return false, nil 181 | }) 182 | if err != nil { 183 | return Checksum{}, err 184 | } 185 | return Checksum{h.Sum32(), latestRevision, compactRevision}, nil 186 | } 187 | 188 | func getCompactRevision(db *bolt.DB) (int64, error) { 189 | compactRev := int64(0) 190 | err := db.View(func(tx *bolt.Tx) error { 191 | b := tx.Bucket(metaBucket) 192 | finishedCompactBytes := b.Get(finishedCompactKeyName) 193 | if len(finishedCompactBytes) != 0 { 194 | compactRev = bytesToRev(finishedCompactBytes).main 195 | } 196 | return nil 197 | }) 198 | if err != nil { 199 | return 0, err 200 | } 201 | return compactRev, nil 202 | } 203 | 204 | func getConsistentIndex(db *bolt.DB) (int64, error) { 205 | consistentIndex := int64(0) 206 | err := db.View(func(tx *bolt.Tx) error { 207 | b := tx.Bucket(metaBucket) 208 | consistentIndexBytes := b.Get(consistentIndexKeyName) 209 | if len(consistentIndexBytes) != 0 { 210 | consistentIndex = int64(binary.BigEndian.Uint64(consistentIndexBytes[0:8])) 211 | } 212 | return nil 213 | }) 214 | if err != nil { 215 | return 0, err 216 | } 217 | return consistentIndex, nil 218 | } 219 | 220 | // ListKeySummaries returns a result set with all the provided filters and projections applied. 221 | func ListKeySummaries(filename string, filters []Filter, proj *KeySummaryProjection, revision int64) ([]*KeySummary, error) { 222 | var err error 223 | db, err := bolt.Open(filename, 0400, &bolt.Options{}) 224 | if err != nil { 225 | return nil, err 226 | } 227 | defer db.Close() 228 | 229 | prefixFilter, filters := separatePrefixFilter(filters) 230 | m := make(map[string]*KeySummary) 231 | err = walk(db, func(r revKey, kv *mvccpb.KeyValue) (bool, error) { 232 | if revision > 0 && r.main > revision { 233 | return false, nil 234 | } 235 | if r.tombstone { 236 | delete(m, string(kv.Key)) 237 | } 238 | prefixAllowed := true 239 | if prefixFilter != nil { 240 | prefixAllowed, err = prefixFilter.Accept(&KeySummary{Key: string(kv.Key)}) 241 | if err != nil { 242 | return false, err 243 | } 244 | } 245 | if prefixAllowed { 246 | ks, ok := m[string(kv.Key)] 247 | if !ok { 248 | buf := new(bytes.Buffer) 249 | var valJson string 250 | var typeMeta *runtime.TypeMeta 251 | var err error 252 | if typeMeta, err = encoding.DetectAndConvert(encoding.JsonMediaType, kv.Value, buf); err == nil { 253 | valJson = strings.TrimSpace(buf.String()) 254 | } 255 | var key string 256 | var value map[string]interface{} 257 | if proj.HasKey { 258 | key = string(kv.Key) 259 | } 260 | // If the caller or filters need the value, we need to deserialize it. 261 | // For filters we don't yet know if they need it, so if there are any filters we must include it. 262 | if proj.HasValue || len(filters) > 0 { 263 | value = rawJsonUnmarshal(valJson) 264 | } 265 | ks = &KeySummary{ 266 | Key: key, 267 | Version: kv.Version, 268 | Stats: &KeySummaryStats{ 269 | KeySize: len(kv.Key), 270 | ValueSize: len(kv.Value), 271 | AllVersionsKeySize: len(kv.Key), 272 | AllVersionsValueSize: len(kv.Value), 273 | VersionCount: 1, 274 | }, 275 | Value: value, 276 | TypeMeta: typeMeta, 277 | } 278 | for _, filter := range filters { 279 | ok, err := filter.Accept(ks) 280 | if err != nil { 281 | return false, err 282 | } 283 | if !ok { 284 | return false, nil 285 | } 286 | } 287 | m[string(kv.Key)] = ks 288 | } else { 289 | if kv.ModRevision > ks.Version { 290 | ks.Version = kv.ModRevision 291 | ks.Stats.ValueSize = len(kv.Value) 292 | } 293 | ks.Stats.VersionCount += 1 294 | ks.Stats.AllVersionsKeySize += len(kv.Key) 295 | ks.Stats.AllVersionsValueSize += len(kv.Value) 296 | } 297 | 298 | } 299 | return false, nil 300 | }) 301 | if err != nil { 302 | return nil, err 303 | } 304 | result := sortKeySummaries(m) 305 | return result, nil 306 | } 307 | 308 | func sortKeySummaries(m map[string]*KeySummary) []*KeySummary { 309 | result := make([]*KeySummary, len(m)) 310 | i := 0 311 | for _, v := range m { 312 | result[i] = v 313 | i++ 314 | } 315 | sort.Slice(result, func(i, j int) bool { 316 | return strings.Compare(result[i].Key, result[j].Key) <= 0 317 | }) 318 | 319 | return result 320 | } 321 | 322 | // ListVersions lists all versions of a object with the given key. 323 | func ListVersions(filename string, key string) ([]int64, error) { 324 | db, err := bolt.Open(filename, 0400, &bolt.Options{}) 325 | if err != nil { 326 | return nil, err 327 | } 328 | defer db.Close() 329 | 330 | var result []int64 331 | 332 | err = walk(db, func(r revKey, kv *mvccpb.KeyValue) (bool, error) { 333 | if string(kv.Key) == key { 334 | result = append(result, kv.Version) 335 | } 336 | return false, nil 337 | }) 338 | if err != nil { 339 | return nil, err 340 | } 341 | return result, nil 342 | } 343 | 344 | // GetValue scans the bucket of the bolt db file for a etcd v3 record with the given key and returns the value. 345 | // Because bolt db files are indexed by revision 346 | func GetValue(filename string, key string, version int64) ([]byte, error) { 347 | db, err := bolt.Open(filename, 0400, &bolt.Options{}) 348 | if err != nil { 349 | return nil, err 350 | } 351 | defer db.Close() 352 | var result []byte 353 | found := false 354 | err = walk(db, func(r revKey, kv *mvccpb.KeyValue) (bool, error) { 355 | if string(kv.Key) == key && kv.Version == version { 356 | result = kv.Value 357 | found = true 358 | return true, nil 359 | } 360 | return false, nil 361 | }) 362 | if err != nil { 363 | return nil, err 364 | } 365 | if !found { 366 | return nil, fmt.Errorf("key not found: %s", key) 367 | } 368 | return result, nil 369 | } 370 | 371 | type kvr struct { 372 | kv *mvccpb.KeyValue 373 | rev revKey 374 | } 375 | 376 | func walkRevision(db *bolt.DB, revision int64, f func(r revKey, kv *mvccpb.KeyValue) (bool, error)) error { 377 | compactRev, err := getCompactRevision(db) 378 | if err != nil { 379 | return err 380 | } 381 | if revision > 0 && revision < compactRev { 382 | return fmt.Errorf("required revision has been compacted") 383 | } 384 | 385 | m := map[string]kvr{} 386 | err = walk(db, func(r revKey, kv *mvccpb.KeyValue) (bool, error) { 387 | if revision > 0 && r.main > revision { 388 | return false, nil 389 | } 390 | k := string(kv.Key) 391 | if m[k].rev.main < r.main { 392 | m[k] = kvr{kv, r} 393 | } 394 | return false, nil 395 | }) 396 | if err != nil { 397 | return err 398 | } 399 | 400 | sorted := make([]kvr, len(m)) 401 | i := 0 402 | for _, v := range m { 403 | sorted[i] = v 404 | i++ 405 | } 406 | sort.Slice(sorted, func(i, j int) bool { 407 | return bytes.Compare(sorted[i].kv.Key, sorted[j].kv.Key) <= 0 408 | }) 409 | 410 | for _, s := range sorted { 411 | if !s.rev.tombstone { 412 | f(s.rev, s.kv) 413 | } 414 | } 415 | return nil 416 | } 417 | 418 | func walk(db *bolt.DB, f func(r revKey, kv *mvccpb.KeyValue) (bool, error)) error { 419 | err := db.View(func(tx *bolt.Tx) error { 420 | b := tx.Bucket(keyBucket) 421 | c := b.Cursor() 422 | 423 | for k, v := c.First(); k != nil; k, v = c.Next() { 424 | revision := bytesToRev(k) 425 | kv := &mvccpb.KeyValue{} 426 | err := kv.Unmarshal(v) 427 | if err != nil { 428 | return err 429 | } 430 | done, err := f(revision, kv) 431 | if err != nil { 432 | return fmt.Errorf("Error handling key %s", kv.Key) 433 | } 434 | if done { 435 | break 436 | } 437 | } 438 | return nil 439 | }) 440 | if err != nil { 441 | return err 442 | } 443 | return nil 444 | } 445 | 446 | // A revKey indicates modification of the key-value space. 447 | // The set of changes that share same main revision changes the key-value space atomically. 448 | type revKey struct { 449 | // main is the main revision of a set of changes that happen atomically. 450 | main int64 451 | // sub is the sub revision of a change in a set of changes that happen 452 | // atomically. Each change has different increasing sub revision in that 453 | // set. 454 | sub int64 455 | // https://github.com/etcd-io/etcd/blob/2c5162af5c096406fb991b4a15761d186b18afbf/mvcc/kvstore.go#L572 456 | tombstone bool 457 | } 458 | 459 | // revBytesLen is the byte length of a normal revision. 460 | // First 8 bytes is the revision.main in big-endian format. The 9th byte 461 | // is a '_'. The last 8 bytes is the revision.sub in big-endian format. 462 | 463 | const ( 464 | revBytesLen = 8 + 1 + 8 465 | markedRevBytesLen = revBytesLen + 1 466 | markBytePosition = markedRevBytesLen - 1 467 | markTombstone byte = 't' 468 | ) 469 | 470 | func bytesToRev(bytes []byte) revKey { 471 | r := revKey{ 472 | main: int64(binary.BigEndian.Uint64(bytes[0:8])), 473 | sub: int64(binary.BigEndian.Uint64(bytes[9:])), 474 | } 475 | if len(bytes) >= markedRevBytesLen { 476 | r.tombstone = bytes[markedRevBytesLen-1] == 't' 477 | } 478 | return r 479 | } 480 | 481 | // ParseFilters parses a comma separated list of '=' filters where each field is 482 | // specified in golang template format, e.g. '.Value.metadata.namespace'. 483 | func ParseFilters(filters string) ([]Filter, error) { 484 | var results []Filter 485 | for _, filter := range strings.Split(filters, ",") { 486 | parts := strings.Split(filter, "=") 487 | if len(parts) != 2 { 488 | return nil, fmt.Errorf("failed to parse filter '%s'", filter) 489 | } 490 | lhs := strings.TrimSpace(parts[0]) 491 | rhs := strings.TrimSpace(parts[1]) 492 | fc := &FieldConstraint{lhs: lhs, op: Equals, rhs: rhs} 493 | f, err := fc.BuildFilter() 494 | if err != nil { 495 | return nil, err 496 | } 497 | results = append(results, f) 498 | } 499 | return results, nil 500 | } 501 | 502 | func rawJsonMarshal(data interface{}) string { 503 | b, err := json.Marshal(data) 504 | if err != nil { 505 | return "" 506 | } 507 | return string(b) 508 | } 509 | 510 | func rawJsonUnmarshal(valJson string) map[string]interface{} { 511 | val := map[string]interface{}{} 512 | if err := json.Unmarshal([]byte(valJson), &val); err != nil { 513 | val = nil 514 | } 515 | return val 516 | } 517 | 518 | func separatePrefixFilter(filters []Filter) (*PrefixFilter, []Filter) { 519 | var prefixFilter *PrefixFilter 520 | remainingFilters := filters 521 | for i, filter := range filters { 522 | if pf, ok := filter.(*PrefixFilter); ok { 523 | prefixFilter = pf 524 | remainingFilters = append(filters[:i], filters[i+1:]...) 525 | } 526 | } 527 | return prefixFilter, remainingFilters 528 | } 529 | --------------------------------------------------------------------------------