├── api └── openapi │ └── violations.txt ├── pkg ├── runtime │ ├── doc.go │ ├── meta.go │ ├── identifiers.go │ └── zz_generated.deepcopy.go ├── util │ ├── fs.go │ ├── sync │ │ ├── monitor.go │ │ ├── batcher_test.go │ │ └── batcher.go │ ├── util.go │ ├── watcher │ │ ├── event.go │ │ ├── dir_traversal.go │ │ └── filewatcher_test.go │ └── patch │ │ ├── patch_test.go │ │ └── patch.go ├── storage │ ├── format.go │ ├── watch │ │ └── update │ │ │ ├── event.go │ │ │ └── update.go │ ├── transaction │ │ ├── storage.go │ │ ├── commit.go │ │ ├── pullrequest │ │ │ └── github │ │ │ │ └── github.go │ │ ├── pullrequest.go │ │ └── git.go │ ├── key.go │ ├── cache │ │ ├── object.go │ │ ├── index.go │ │ └── cache.go │ ├── mappedrawstorage.go │ ├── sync │ │ └── storage.go │ └── rawstorage.go ├── logs │ ├── flag │ │ └── flag.go │ └── logs.go ├── filter │ ├── options.go │ ├── interfaces.go │ ├── uid.go │ └── name.go ├── serializer │ ├── error_structs.go │ ├── frame_utils.go │ ├── frame_writer_test.go │ ├── defaulter.go │ ├── objectmeta.go │ ├── frame_reader_test.go │ ├── frame_writer.go │ ├── comments │ │ ├── comments.go │ │ ├── lost.go │ │ └── comments_test.go │ ├── error.go │ ├── comments_test.go │ ├── encode.go │ ├── frame_reader.go │ └── comments.go ├── gitdir │ └── transport.go └── client │ ├── client_dynamic.go │ └── client_resource_template.go ├── cmd ├── sample-app │ ├── apis │ │ └── sample │ │ │ ├── doc.go │ │ │ ├── v1alpha1 │ │ │ ├── doc.go │ │ │ ├── defaults.go │ │ │ ├── zz_generated.defaults.go │ │ │ ├── register.go │ │ │ ├── types.go │ │ │ └── zz_generated.deepcopy.go │ │ │ ├── zz_generated.defaults.go │ │ │ ├── register.go │ │ │ ├── scheme │ │ │ └── scheme.go │ │ │ ├── types.go │ │ │ └── zz_generated.deepcopy.go │ ├── version │ │ └── version.go │ ├── client │ │ ├── client.go │ │ ├── zz_generated.client_car.go │ │ └── zz_generated.client_motorcycle.go │ └── main.go ├── common │ └── common.go ├── sample-watch │ └── main.go └── sample-gitops │ └── main.go ├── .gitignore ├── docs └── images │ ├── storage_system_overview.png │ └── storage_system_transaction.png ├── .goreleaser.yml ├── CODE_OF_CONDUCT.md ├── .github └── workflows │ ├── golangci-lint.yaml │ ├── ci.yaml │ └── goreleaser.yaml ├── hack ├── generate-client.sh └── ldflags.sh ├── go.mod └── Makefile /api/openapi/violations.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/runtime/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | package runtime 3 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | package sample 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE-specific configuration 2 | .idea/ 3 | .vscode/ 4 | 5 | # Binary artifacts 6 | bin/ -------------------------------------------------------------------------------- /docs/images/storage_system_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaveworks/libgitops/HEAD/docs/images/storage_system_overview.png -------------------------------------------------------------------------------- /docs/images/storage_system_transaction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weaveworks/libgitops/HEAD/docs/images/storage_system_transaction.png -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | // +k8s:defaulter-gen=TypeMeta 3 | // +k8s:openapi-gen=true 4 | // +k8s:conversion-gen=github.com/weaveworks/libgitops/cmd/sample-app/apis/sample 5 | package v1alpha1 6 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # .goreleaser.yml 2 | project_name: libgitops 3 | 4 | # This is a library so we don't have any build targets, but having goreleaser automatically 5 | # generate release notes for us is nice. 6 | build: 7 | skip: true 8 | 9 | # This automatically closes any milestone named the same as the tag 10 | milestones: 11 | - close: true -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Community Code of Conduct 2 | 3 | Weaveworks follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | Instances of abusive, harassing, or otherwise unacceptable behavior 6 | may be reported by contacting a Weaveworks project maintainer, or 7 | Alexis Richardson (alexis@weave.works). 8 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/defaults.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | ) 6 | 7 | func addDefaultingFuncs(scheme *runtime.Scheme) error { 8 | return RegisterDefaults(scheme) 9 | } 10 | 11 | func SetDefaults_MotorcycleSpec(obj *MotorcycleSpec) { 12 | obj.Color = "blue" 13 | } 14 | 15 | func SetDefaults_CarSpec(obj *CarSpec) { 16 | obj.Brand = "Mercedes" 17 | } 18 | -------------------------------------------------------------------------------- /pkg/util/fs.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func PathExists(path string) (bool, os.FileInfo) { 8 | info, err := os.Stat(path) 9 | if os.IsNotExist(err) { 10 | return false, nil 11 | } 12 | 13 | return true, info 14 | } 15 | 16 | func FileExists(filename string) bool { 17 | exists, info := PathExists(filename) 18 | if !exists { 19 | return false 20 | } 21 | 22 | return !info.IsDir() 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yaml: -------------------------------------------------------------------------------- 1 | # https://github.com/marketplace/actions/run-golangci-lint 2 | name: golangci-lint 3 | 4 | on: 5 | pull_request: 6 | push: 7 | branches: [master] 8 | 9 | jobs: 10 | golangci: 11 | name: Linter 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: golangci-lint 16 | uses: golangci/golangci-lint-action@v1 17 | with: 18 | version: v1.28 -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Code generated by defaulter-gen. DO NOT EDIT. 4 | 5 | package sample 6 | 7 | import ( 8 | runtime "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // RegisterDefaults adds defaulters functions to the given scheme. 12 | // Public to allow building arbitrary schemes. 13 | // All generated defaulters are covering - they call all nested defaulters. 14 | func RegisterDefaults(scheme *runtime.Scheme) error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/storage/format.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "github.com/weaveworks/libgitops/pkg/serializer" 4 | 5 | // ContentTypes describes the connection between 6 | // file extensions and a content types. 7 | var ContentTypes = map[string]serializer.ContentType{ 8 | ".json": serializer.ContentTypeJSON, 9 | ".yaml": serializer.ContentTypeYAML, 10 | ".yml": serializer.ContentTypeYAML, 11 | } 12 | 13 | func extForContentType(wanted serializer.ContentType) string { 14 | for ext, ct := range ContentTypes { 15 | if ct == wanted { 16 | return ext 17 | } 18 | } 19 | return "" 20 | } 21 | -------------------------------------------------------------------------------- /pkg/util/sync/monitor.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import "sync" 4 | 5 | // Monitor is a convenience wrapper around 6 | // starting a goroutine with a wait group, 7 | // which can be used to wait for the 8 | // goroutine to stop. 9 | type Monitor struct { 10 | wg *sync.WaitGroup 11 | } 12 | 13 | func RunMonitor(f func()) (m *Monitor) { 14 | m = &Monitor{ 15 | wg: new(sync.WaitGroup), 16 | } 17 | 18 | m.wg.Add(1) 19 | go func() { 20 | f() 21 | m.wg.Done() 22 | }() 23 | 24 | return 25 | } 26 | 27 | func (m *Monitor) Wait() { 28 | if m != nil { 29 | m.wg.Wait() 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /hack/generate-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR=$( dirname "${BASH_SOURCE[0]}" ) 4 | cd ${SCRIPT_DIR}/.. 5 | 6 | RESOURCES="Car Motorcycle" 7 | CLIENT_NAME=SampleInternal 8 | OUT_DIR=cmd/sample-app/client 9 | API_DIR="github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 10 | mkdir -p ${OUT_DIR} 11 | for Resource in ${RESOURCES}; do 12 | resource=$(echo "${Resource}" | awk '{print tolower($0)}') 13 | sed -e "s|Resource|${Resource}|g;s|resource|${resource}|g;/build ignore/d;s|API_DIR|${API_DIR}|g;s|*Client|*${CLIENT_NAME}Client|g" \ 14 | pkg/client/client_resource_template.go > \ 15 | ${OUT_DIR}/zz_generated.client_${resource}.go 16 | done 17 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [master] 7 | 8 | jobs: 9 | unit-tests: 10 | name: Unit Tests 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout Sources 14 | uses: actions/checkout@v2 15 | - name: Docker Layer Caching 16 | uses: satackey/action-docker-layer-caching@v0.0.3 17 | - name: Restore Go Cache 18 | uses: actions/cache@v2 19 | with: 20 | path: bin/cache/go/pkg/mod 21 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 22 | restore-keys: ${{ runner.os }}-go- 23 | - name: Run Unit Tests 24 | run: make test -------------------------------------------------------------------------------- /pkg/storage/watch/update/event.go: -------------------------------------------------------------------------------- 1 | package update 2 | 3 | import "fmt" 4 | 5 | // ObjectEvent is an enum describing a change in an Object's state. 6 | type ObjectEvent byte 7 | 8 | var _ fmt.Stringer = ObjectEvent(0) 9 | 10 | const ( 11 | ObjectEventNone ObjectEvent = iota // 0 12 | ObjectEventCreate // 1 13 | ObjectEventModify // 2 14 | ObjectEventDelete // 3 15 | ) 16 | 17 | func (o ObjectEvent) String() string { 18 | switch o { 19 | case 0: 20 | return "NONE" 21 | case 1: 22 | return "CREATE" 23 | case 2: 24 | return "MODIFY" 25 | case 3: 26 | return "DELETE" 27 | } 28 | 29 | // Should never happen 30 | return "UNKNOWN" 31 | } 32 | -------------------------------------------------------------------------------- /pkg/logs/flag/flag.go: -------------------------------------------------------------------------------- 1 | package flag 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/spf13/pflag" 6 | ) 7 | 8 | type LogLevelFlag struct { 9 | value *logrus.Level 10 | } 11 | 12 | func (lf *LogLevelFlag) Set(val string) error { 13 | var err error 14 | *lf.value, err = logrus.ParseLevel(val) 15 | return err 16 | } 17 | 18 | func (lf *LogLevelFlag) String() string { 19 | if lf.value == nil { 20 | return "" 21 | } 22 | return lf.value.String() 23 | } 24 | 25 | func (lf *LogLevelFlag) Type() string { 26 | return "loglevel" 27 | } 28 | 29 | var _ pflag.Value = &LogLevelFlag{} 30 | 31 | func LogLevelFlagVar(fs *pflag.FlagSet, ptr *logrus.Level) { 32 | fs.Var(&LogLevelFlag{value: ptr}, "log-level", "Specify the loglevel for the program") 33 | } 34 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/zz_generated.defaults.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Code generated by defaulter-gen. DO NOT EDIT. 4 | 5 | package v1alpha1 6 | 7 | import ( 8 | runtime "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // RegisterDefaults adds defaulters functions to the given scheme. 12 | // Public to allow building arbitrary schemes. 13 | // All generated defaulters are covering - they call all nested defaulters. 14 | func RegisterDefaults(scheme *runtime.Scheme) error { 15 | scheme.AddTypeDefaultingFunc(&Car{}, func(obj interface{}) { SetObjectDefaults_Car(obj.(*Car)) }) 16 | scheme.AddTypeDefaultingFunc(&Motorcycle{}, func(obj interface{}) { SetObjectDefaults_Motorcycle(obj.(*Motorcycle)) }) 17 | return nil 18 | } 19 | 20 | func SetObjectDefaults_Car(in *Car) { 21 | SetDefaults_CarSpec(&in.Spec) 22 | } 23 | 24 | func SetObjectDefaults_Motorcycle(in *Motorcycle) { 25 | SetDefaults_MotorcycleSpec(&in.Spec) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/storage/watch/update/update.go: -------------------------------------------------------------------------------- 1 | package update 2 | 3 | import ( 4 | "github.com/weaveworks/libgitops/pkg/runtime" 5 | "github.com/weaveworks/libgitops/pkg/storage" 6 | ) 7 | 8 | // Update bundles an FileEvent with an 9 | // APIType for Storage retrieval. 10 | type Update struct { 11 | Event ObjectEvent 12 | PartialObject runtime.PartialObject 13 | Storage storage.Storage 14 | } 15 | 16 | // UpdateStream is a channel of updates. 17 | type UpdateStream chan Update 18 | 19 | // EventStorage is a storage that exposes an UpdateStream. 20 | type EventStorage interface { 21 | storage.Storage 22 | 23 | // SetUpdateStream gives the EventStorage a channel to send events to. 24 | // The caller is responsible for choosing a large enough buffer to avoid 25 | // blocking the underlying EventStorage implementation unnecessarily. 26 | // TODO: In the future maybe enable sending events to multiple listeners? 27 | SetUpdateStream(UpdateStream) 28 | } 29 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/register.go: -------------------------------------------------------------------------------- 1 | package sample 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | var ( 9 | // SchemeBuilder the schema builder 10 | SchemeBuilder = runtime.NewSchemeBuilder( 11 | addKnownTypes, 12 | ) 13 | 14 | localSchemeBuilder = &SchemeBuilder 15 | AddToScheme = localSchemeBuilder.AddToScheme 16 | ) 17 | 18 | const ( 19 | // GroupName is the group name use in this package 20 | GroupName = "sample-app.weave.works" 21 | ) 22 | 23 | // SchemeGroupVersion is group version used to register these objects 24 | var SchemeGroupVersion = schema.GroupVersion{ 25 | Group: GroupName, 26 | Version: runtime.APIVersionInternal, 27 | } 28 | 29 | // Adds the list of known types to the given scheme. 30 | func addKnownTypes(scheme *runtime.Scheme) error { 31 | scheme.AddKnownTypes(SchemeGroupVersion, 32 | &Car{}, 33 | &Motorcycle{}, 34 | ) 35 | 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | var ( 9 | // SchemeBuilder the schema builder 10 | SchemeBuilder = runtime.NewSchemeBuilder( 11 | addKnownTypes, 12 | addDefaultingFuncs, 13 | ) 14 | 15 | localSchemeBuilder = &SchemeBuilder 16 | AddToScheme = localSchemeBuilder.AddToScheme 17 | ) 18 | 19 | const ( 20 | // GroupName is the group name use in this package 21 | GroupName = "sample-app.weave.works" 22 | ) 23 | 24 | // SchemeGroupVersion is group version used to register these objects 25 | var SchemeGroupVersion = schema.GroupVersion{ 26 | Group: GroupName, 27 | Version: "v1alpha1", 28 | } 29 | 30 | // Adds the list of known types to the given scheme. 31 | func addKnownTypes(scheme *runtime.Scheme) error { 32 | scheme.AddKnownTypes(SchemeGroupVersion, 33 | &Car{}, 34 | &Motorcycle{}, 35 | ) 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/filter/options.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | // ListOptions is a generic struct for listing options. 4 | type ListOptions struct { 5 | // Filters contains a chain of ListFilters, which will be processed in order and pipe the 6 | // available objects through before returning. 7 | Filters []ListFilter 8 | } 9 | 10 | // ListOption is an interface which can be passed into e.g. List() methods as a variadic-length 11 | // argument list. 12 | type ListOption interface { 13 | // ApplyToListOptions applies the configuration of the current object into a target ListOptions struct. 14 | ApplyToListOptions(target *ListOptions) error 15 | } 16 | 17 | // MakeListOptions makes a completed ListOptions struct from a list of ListOption implementations. 18 | func MakeListOptions(opts ...ListOption) (*ListOptions, error) { 19 | o := &ListOptions{} 20 | for _, opt := range opts { 21 | // For every option, apply it into o, and check if there's an error 22 | if err := opt.ApplyToListOptions(o); err != nil { 23 | return nil, err 24 | } 25 | } 26 | return o, nil 27 | } 28 | -------------------------------------------------------------------------------- /pkg/storage/transaction/storage.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/weaveworks/libgitops/pkg/storage" 8 | ) 9 | 10 | var ( 11 | ErrAbortTransaction = errors.New("transaction aborted") 12 | ErrTransactionActive = errors.New("transaction is active") 13 | ErrNoPullRequestProvider = errors.New("no pull request provider given") 14 | ) 15 | 16 | type TransactionFunc func(ctx context.Context, s storage.Storage) (CommitResult, error) 17 | 18 | type TransactionStorage interface { 19 | storage.ReadStorage 20 | 21 | // Transaction creates a new "stream" (for Git: branch) with the given name, or 22 | // prefix if streamName ends with a dash (in that case, a 8-char hash will be appended). 23 | // The environment is made sure to be as up-to-date as possible before fn executes. When 24 | // fn executes, the given storage can be used to modify the desired state. If you want to 25 | // "commit" the changes made in fn, just return nil. If you want to abort, return ErrAbortTransaction. 26 | // If you want to 27 | Transaction(ctx context.Context, streamName string, fn TransactionFunc) error 28 | } 29 | -------------------------------------------------------------------------------- /pkg/serializer/error_structs.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | var _ ReadCloser = &errReadCloser{} 4 | 5 | type errReadCloser struct { 6 | err error 7 | } 8 | 9 | func (rc *errReadCloser) Read(p []byte) (n int, err error) { 10 | err = rc.err 11 | return 12 | } 13 | 14 | func (rc *errReadCloser) Close() error { 15 | return nil 16 | } 17 | 18 | var _ FrameReader = &errFrameReader{} 19 | 20 | type errFrameReader struct { 21 | err error 22 | contentType ContentType 23 | } 24 | 25 | func (fr *errFrameReader) ReadFrame() ([]byte, error) { 26 | return nil, fr.err 27 | } 28 | 29 | func (fr *errFrameReader) ContentType() ContentType { 30 | return fr.contentType 31 | } 32 | 33 | // Close implements io.Closer and closes the underlying ReadCloser 34 | func (fr *errFrameReader) Close() error { 35 | return nil 36 | } 37 | 38 | var _ FrameWriter = &errFrameWriter{} 39 | 40 | type errFrameWriter struct { 41 | err error 42 | contentType ContentType 43 | } 44 | 45 | func (fw *errFrameWriter) Write(_ []byte) (n int, err error) { 46 | err = fw.err 47 | return 48 | } 49 | 50 | func (fw *errFrameWriter) ContentType() ContentType { 51 | return fw.contentType 52 | } 53 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/weaveworks/libgitops 2 | 3 | go 1.14 4 | 5 | replace ( 6 | github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible 7 | github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.3.0 8 | ) 9 | 10 | require ( 11 | github.com/fluxcd/go-git-providers v0.0.2 12 | github.com/fluxcd/toolkit v0.0.1-beta.2 13 | github.com/go-git/go-git/v5 v5.1.0 14 | github.com/go-openapi/spec v0.19.8 15 | github.com/google/go-github/v32 v32.1.0 16 | github.com/labstack/echo v3.3.10+incompatible 17 | github.com/labstack/gommon v0.3.0 // indirect 18 | github.com/mattn/go-isatty v0.0.12 // indirect 19 | github.com/mitchellh/go-homedir v1.1.0 20 | github.com/rjeczalik/notify v0.9.2 21 | github.com/sirupsen/logrus v1.6.0 22 | github.com/spf13/pflag v1.0.5 23 | github.com/stretchr/testify v1.6.1 24 | golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect 25 | golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d 26 | k8s.io/apimachinery v0.18.6 27 | k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 28 | sigs.k8s.io/controller-runtime v0.6.0 29 | sigs.k8s.io/kustomize/kyaml v0.1.11 30 | sigs.k8s.io/yaml v1.2.0 31 | ) 32 | -------------------------------------------------------------------------------- /pkg/serializer/frame_utils.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import "io" 4 | 5 | // FrameList is a list of frames (byte arrays), used for convenience functions 6 | type FrameList [][]byte 7 | 8 | // ReadFrameList is a convenience method that reads all available frames from the FrameReader 9 | // into a returned FrameList 10 | func ReadFrameList(fr FrameReader) (FrameList, error) { 11 | // TODO: Create an unit test for this function 12 | var frameList [][]byte 13 | for { 14 | // Read until we get io.EOF or an error 15 | frame, err := fr.ReadFrame() 16 | if err == io.EOF { 17 | break 18 | } else if err != nil { 19 | return nil, err 20 | } 21 | // Append all frames to the returned list 22 | frameList = append(frameList, frame) 23 | } 24 | return frameList, nil 25 | } 26 | 27 | // WriteFrameList is a convenience method that writes a set of frames to a FrameWriter 28 | func WriteFrameList(fw FrameWriter, frameList FrameList) error { 29 | // TODO: Create an unit test for this function 30 | // Loop all frames in the list, and write them individually to the FrameWriter 31 | for _, frame := range frameList { 32 | if _, err := fw.Write(frame); err != nil { 33 | return err 34 | } 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/scheme/scheme.go: -------------------------------------------------------------------------------- 1 | package scheme 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | k8sserializer "k8s.io/apimachinery/pkg/runtime/serializer" 6 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 7 | 8 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 9 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/v1alpha1" 10 | "github.com/weaveworks/libgitops/pkg/serializer" 11 | ) 12 | 13 | var ( 14 | // Scheme is the runtime.Scheme to which all types are registered. 15 | Scheme = runtime.NewScheme() 16 | 17 | // codecs provides access to encoding and decoding for the scheme. 18 | // codecs is private, as Serializer will be used for all higher-level encoding/decoding 19 | codecs = k8sserializer.NewCodecFactory(Scheme) 20 | 21 | // Serializer provides high-level encoding/decoding functions 22 | Serializer = serializer.NewSerializer(Scheme, &codecs) 23 | ) 24 | 25 | func init() { 26 | AddToScheme(Scheme) 27 | } 28 | 29 | // AddToScheme builds the scheme using all known versions of the api. 30 | func AddToScheme(scheme *runtime.Scheme) { 31 | utilruntime.Must(v1alpha1.AddToScheme(Scheme)) 32 | utilruntime.Must(sample.AddToScheme(Scheme)) 33 | utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) 34 | } 35 | -------------------------------------------------------------------------------- /cmd/sample-app/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | var ( 9 | gitMajor = "" 10 | gitMinor = "" 11 | gitVersion = "" 12 | gitCommit = "" 13 | gitTreeState = "" 14 | buildDate = "" 15 | ) 16 | 17 | // Info stores information about a component's version 18 | type Info struct { 19 | Major string `json:"major"` 20 | Minor string `json:"minor"` 21 | GitVersion string `json:"gitVersion"` 22 | GitCommit string `json:"gitCommit"` 23 | GitTreeState string `json:"gitTreeState"` 24 | BuildDate string `json:"buildDate"` 25 | GoVersion string `json:"goVersion"` 26 | Compiler string `json:"compiler"` 27 | Platform string `json:"platform"` 28 | } 29 | 30 | // String returns info as a human-friendly version string 31 | func (info *Info) String() string { 32 | return info.GitVersion 33 | } 34 | 35 | // Get returns the version information of sample-app 36 | func Get() *Info { 37 | return &Info{ 38 | Major: gitMajor, 39 | Minor: gitMinor, 40 | GitVersion: gitVersion, 41 | GitCommit: gitCommit, 42 | GitTreeState: gitTreeState, 43 | BuildDate: buildDate, 44 | GoVersion: runtime.Version(), 45 | Compiler: runtime.Compiler, 46 | Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yaml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | - name: Unshallow 15 | run: git fetch --prune --unshallow 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.14 20 | # Similar to https://github.com/fluxcd/toolkit/blob/master/.github/workflows/release.yaml#L20-L27 21 | - name: Download release notes utility 22 | env: 23 | GH_REL_URL: https://github.com/buchanae/github-release-notes/releases/download/0.2.0/github-release-notes-linux-amd64-0.2.0.tar.gz 24 | run: cd /tmp && curl -sSL ${GH_REL_URL} | tar xz && sudo mv github-release-notes /usr/local/bin/ 25 | - name: Generate release notes 26 | run: | 27 | echo 'CHANGELOG' > /tmp/release.txt 28 | github-release-notes -org weaveworks -repo libgitops -since-latest-release -include-author -include-commits >> /tmp/release.txt 29 | - name: Run GoReleaser 30 | uses: goreleaser/goreleaser-action@v2 31 | with: 32 | version: latest 33 | args: release --release-notes=/tmp/release.txt --rm-dist 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "encoding/hex" 7 | "fmt" 8 | "os/exec" 9 | "strings" 10 | ) 11 | 12 | func ExecuteCommand(command string, args ...string) (string, error) { 13 | cmd := exec.Command(command, args...) 14 | out, err := cmd.CombinedOutput() 15 | if err != nil { 16 | return "", fmt.Errorf("command %q exited with %q: %v", cmd.Args, out, err) 17 | } 18 | 19 | return string(bytes.TrimSpace(out)), nil 20 | } 21 | 22 | func MatchPrefix(prefix string, fields ...string) ([]string, bool) { 23 | var prefixMatches, exactMatches []string 24 | 25 | for _, str := range fields { 26 | if str == prefix { 27 | exactMatches = append(exactMatches, str) 28 | } else if strings.HasPrefix(str, prefix) { 29 | prefixMatches = append(prefixMatches, str) 30 | } 31 | } 32 | 33 | // If we have exact matches, return them 34 | // and set the exact match boolean 35 | if len(exactMatches) > 0 { 36 | return exactMatches, true 37 | } 38 | 39 | return prefixMatches, false 40 | } 41 | 42 | func BoolPtr(b bool) *bool { 43 | return &b 44 | } 45 | 46 | // RandomSHA returns a hex-encoded string from {byteLen} random bytes. 47 | func RandomSHA(byteLen int) (string, error) { 48 | b := make([]byte, byteLen) 49 | _, err := rand.Read(b) 50 | if err != nil { 51 | return "", err 52 | } 53 | return hex.EncodeToString(b), nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/util/watcher/event.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // FileEvent is an enum describing a change in a file's state 9 | type FileEvent byte 10 | 11 | const ( 12 | FileEventNone FileEvent = iota // 0 13 | FileEventModify // 1 14 | FileEventDelete // 2 15 | FileEventMove // 3 16 | ) 17 | 18 | func (e FileEvent) String() string { 19 | switch e { 20 | case 0: 21 | return "NONE" 22 | case 1: 23 | return "MODIFY" 24 | case 2: 25 | return "DELETE" 26 | case 3: 27 | return "MOVE" 28 | } 29 | 30 | return "UNKNOWN" 31 | } 32 | 33 | // FileEvents is a slice of FileEvents 34 | type FileEvents []FileEvent 35 | 36 | var _ fmt.Stringer = FileEvents{} 37 | 38 | func (e FileEvents) String() string { 39 | strs := make([]string, 0, len(e)) 40 | for _, ev := range e { 41 | strs = append(strs, ev.String()) 42 | } 43 | 44 | return strings.Join(strs, ",") 45 | } 46 | 47 | func (e FileEvents) Bytes() []byte { 48 | b := make([]byte, 0, len(e)) 49 | for _, event := range e { 50 | b = append(b, byte(event)) 51 | } 52 | 53 | return b 54 | } 55 | 56 | // FileUpdates is a slice of FileUpdate pointers 57 | type FileUpdates []*FileUpdate 58 | 59 | // FileUpdate is used by watchers to 60 | // signal the state change of a file. 61 | type FileUpdate struct { 62 | Event FileEvent 63 | Path string 64 | } 65 | -------------------------------------------------------------------------------- /pkg/logs/logs.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "io/ioutil" 5 | golog "log" 6 | "os" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // Quiet specifies whether to only print machine-readable IDs 12 | var Quiet bool 13 | 14 | // Wrap the logrus logger together with the exit code 15 | // so we can control what log.Fatal returns 16 | type logger struct { 17 | *log.Logger 18 | ExitCode int 19 | } 20 | 21 | func newLogger() *logger { 22 | l := &logger{ 23 | Logger: log.StandardLogger(), // Use the standard logrus logger 24 | ExitCode: 1, 25 | } 26 | 27 | l.ExitFunc = func(_ int) { 28 | os.Exit(l.ExitCode) 29 | } 30 | 31 | return l 32 | } 33 | 34 | // Expose the logger 35 | var Logger *logger 36 | 37 | // Automatically initialize the logging system for libgitops 38 | func init() { 39 | // Initialize the logger 40 | Logger = newLogger() 41 | 42 | // Set the output to be stdout in the normal case, but discard all log output in quiet mode 43 | if Quiet { 44 | Logger.SetOutput(ioutil.Discard) 45 | } else { 46 | Logger.SetOutput(os.Stdout) 47 | } 48 | 49 | // Disable timestamp logging, but still output the seconds elapsed 50 | Logger.SetFormatter(&log.TextFormatter{ 51 | DisableTimestamp: false, 52 | FullTimestamp: false, 53 | }) 54 | 55 | // Disable the stdlib's automatic add of the timestamp in beginning of the log message, 56 | // as we stream the logs from stdlib log to this logrus instance. 57 | golog.SetFlags(0) 58 | golog.SetOutput(Logger.Writer()) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/serializer/frame_writer_test.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func Test_byteWriter_Write(t *testing.T) { 9 | type fields struct { 10 | to []byte 11 | index int 12 | } 13 | type args struct { 14 | from []byte 15 | } 16 | tests := []struct { 17 | name string 18 | fields fields 19 | args args 20 | wantN int 21 | wantErr bool 22 | }{ 23 | { 24 | name: "simple case", 25 | fields: fields{ 26 | to: make([]byte, 50), 27 | }, 28 | args: args{ 29 | from: []byte("Hello!\nFoobar"), 30 | }, 31 | wantN: 13, 32 | wantErr: false, 33 | }, 34 | { 35 | name: "target too short", 36 | fields: fields{ 37 | to: make([]byte, 10), 38 | }, 39 | args: args{ 40 | from: []byte("Hello!\nFoobar"), 41 | }, 42 | wantN: 0, 43 | wantErr: true, 44 | }, 45 | } 46 | for _, tt := range tests { 47 | t.Run(tt.name, func(t *testing.T) { 48 | w := &byteWriter{ 49 | to: tt.fields.to, 50 | index: tt.fields.index, 51 | } 52 | gotN, err := w.Write(tt.args.from) 53 | if (err != nil) != tt.wantErr { 54 | t.Errorf("byteWriter.Write() error = %v, wantErr %v", err, tt.wantErr) 55 | return 56 | } 57 | if gotN != tt.wantN { 58 | t.Errorf("byteWriter.Write() = %v, want %v", gotN, tt.wantN) 59 | return 60 | } 61 | if !tt.wantErr && !bytes.Equal(tt.fields.to[:gotN], tt.args.from) { 62 | t.Errorf("byteWriter.Write(): expected fields.to (%s) to equal args.from (%s), but didn't", tt.fields.to[:gotN], tt.args.from) 63 | } 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/util/patch/patch_test.go: -------------------------------------------------------------------------------- 1 | package patch 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | api "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 8 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" 9 | "github.com/weaveworks/libgitops/pkg/runtime" 10 | "github.com/weaveworks/libgitops/pkg/serializer" 11 | ) 12 | 13 | var ( 14 | basebytes = []byte(` 15 | { 16 | "kind": "Car", 17 | "apiVersion": "sample-app.weave.works/v1alpha1", 18 | "metadata": { 19 | "name": "foo", 20 | "uid": "0123456789101112" 21 | }, 22 | "spec": { 23 | "engine": "foo", 24 | "brand": "bar" 25 | } 26 | }`) 27 | overlaybytes = []byte(`{"status":{"speed":24.7}}`) 28 | ) 29 | 30 | var carGVK = api.SchemeGroupVersion.WithKind("Car") 31 | var p = NewPatcher(scheme.Serializer) 32 | 33 | func TestCreatePatch(t *testing.T) { 34 | car := &api.Car{ 35 | Spec: api.CarSpec{ 36 | Engine: "foo", 37 | Brand: "bar", 38 | }, 39 | } 40 | car.SetGroupVersionKind(carGVK) 41 | b, err := p.Create(car, func(obj runtime.Object) error { 42 | car2 := obj.(*api.Car) 43 | car2.Status.Speed = 24.7 44 | return nil 45 | }) 46 | if !bytes.Equal(b, overlaybytes) { 47 | t.Error(string(b), err, car.Status.Speed) 48 | } 49 | } 50 | 51 | func TestApplyPatch(t *testing.T) { 52 | result, err := p.Apply(basebytes, overlaybytes, carGVK) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | frameReader := serializer.NewJSONFrameReader(serializer.FromBytes(result)) 57 | if err := scheme.Serializer.Decoder().DecodeInto(frameReader, &api.Car{}); err != nil { 58 | t.Fatal(err) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pkg/util/sync/batcher_test.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var events = []string{"CREATE", "MODIFY", "DELETE"} 11 | 12 | type job struct { 13 | file string 14 | event string 15 | } 16 | 17 | func TestBatchWriter(t *testing.T) { 18 | ch := make(chan job) 19 | b := NewBatchWriter(1 * time.Second) 20 | go func() { 21 | for i := 0; i < 10; i++ { 22 | fmt.Println(i) 23 | if i == 4 { 24 | fmt.Println("sleep") 25 | time.Sleep(2 * time.Second) 26 | } 27 | 28 | file := fmt.Sprintf("foo%d", i%5) 29 | event := events[i%3] 30 | 31 | eventList := []string{} 32 | val, ok := b.Load(file) 33 | if ok { 34 | eventList = val.([]string) 35 | } 36 | eventList = append(eventList, event) 37 | 38 | // Store and batch all existing changes after the timeout duration 39 | b.Store(file, eventList) 40 | } 41 | time.Sleep(2 * time.Second) 42 | fmt.Println("stopping") 43 | b.Close() 44 | close(ch) 45 | }() 46 | go func() { 47 | for { 48 | // When the batch items are available, process the items from the map 49 | ok := b.ProcessBatch(func(key, val interface{}) bool { 50 | file := key.(string) 51 | events := val.([]string) 52 | ch <- job{ 53 | file: file, 54 | event: strings.Join(events, ","), 55 | } 56 | return true 57 | }) 58 | if !ok { 59 | return 60 | } 61 | fmt.Println("") 62 | fmt.Println("Map flushed") 63 | fmt.Println("") 64 | } 65 | }() 66 | 67 | for j := range ch { 68 | fmt.Println(j.file, j.event) 69 | } 70 | 71 | //t.Error("err") 72 | } 73 | -------------------------------------------------------------------------------- /pkg/util/watcher/dir_traversal.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | func (w *FileWatcher) getFiles() ([]string, error) { 10 | return WalkDirectoryForFiles(w.dir, w.opts.ValidExtensions, w.opts.ExcludeDirs) 11 | } 12 | 13 | func (w *FileWatcher) validFile(path string) bool { 14 | return isValidFile(path, w.opts.ValidExtensions, w.opts.ExcludeDirs) 15 | } 16 | 17 | // WalkDirectoryForFiles discovers all subdirectories and 18 | // returns a list of valid files in them 19 | func WalkDirectoryForFiles(dir string, validExts, excludeDirs []string) (files []string, err error) { 20 | err = filepath.Walk(dir, 21 | func(path string, info os.FileInfo, err error) error { 22 | if err != nil { 23 | return err 24 | } 25 | 26 | if !info.IsDir() { 27 | // Only include valid files 28 | if isValidFile(path, validExts, excludeDirs) { 29 | files = append(files, path) 30 | } 31 | } 32 | 33 | return nil 34 | }) 35 | 36 | return 37 | } 38 | 39 | // isValidFile is used to filter out all unsupported 40 | // files based on if their extension is unknown or 41 | // if their path contains an excluded directory 42 | func isValidFile(path string, validExts, excludeDirs []string) bool { 43 | parts := strings.Split(filepath.Clean(path), string(os.PathSeparator)) 44 | ext := filepath.Ext(parts[len(parts)-1]) 45 | for _, suffix := range validExts { 46 | if ext == suffix { 47 | return true 48 | } 49 | } 50 | 51 | for i := 0; i < len(parts)-1; i++ { 52 | for _, exclude := range excludeDirs { 53 | if parts[i] == exclude { 54 | return false 55 | } 56 | } 57 | } 58 | 59 | return false 60 | } 61 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/types.go: -------------------------------------------------------------------------------- 1 | package sample 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // Car represents a car 8 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 9 | type Car struct { 10 | metav1.TypeMeta 11 | // runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID 12 | // Name is available at the .metadata.name JSON path 13 | // ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string) 14 | metav1.ObjectMeta 15 | 16 | Spec CarSpec 17 | Status CarStatus 18 | } 19 | 20 | type CarSpec struct { 21 | Engine string 22 | YearModel string 23 | Brand string 24 | } 25 | 26 | type CarStatus struct { 27 | VehicleStatus 28 | Persons uint64 29 | } 30 | 31 | type VehicleStatus struct { 32 | Speed float64 33 | Acceleration float64 34 | Distance uint64 35 | } 36 | 37 | // Motorcycle represents a motorcycle 38 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 39 | type Motorcycle struct { 40 | metav1.TypeMeta 41 | // runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID 42 | // Name is available at the .metadata.name JSON path 43 | // ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string) 44 | metav1.ObjectMeta 45 | 46 | Spec MotorcycleSpec 47 | Status MotorcycleStatus 48 | } 49 | 50 | type MotorcycleSpec struct { 51 | Color string 52 | BodyType string 53 | } 54 | 55 | type MotorcycleStatus struct { 56 | VehicleStatus 57 | CurrentWeight float64 58 | } 59 | -------------------------------------------------------------------------------- /pkg/filter/interfaces.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "github.com/weaveworks/libgitops/pkg/runtime" 4 | 5 | // ListFilter is an interface for pipe-like list filtering behavior. 6 | type ListFilter interface { 7 | // Filter walks through all objects in obj, assesses whether the object 8 | // matches the filter parameters, and conditionally adds it to the return 9 | // slice or not. This method can be thought of like an UNIX pipe. 10 | Filter(objs ...runtime.Object) ([]runtime.Object, error) 11 | } 12 | 13 | // ObjectFilter is an interface for filtering objects one-by-one. 14 | type ObjectFilter interface { 15 | // Filter takes in one object (at once, per invocation), and returns a 16 | // boolean whether the object matches the filter parameters, or not. 17 | Filter(obj runtime.Object) (bool, error) 18 | } 19 | 20 | // ObjectToListFilter transforms an ObjectFilter into a ListFilter. If of is nil, 21 | // this function panics. 22 | func ObjectToListFilter(of ObjectFilter) ListFilter { 23 | if of == nil { 24 | panic("programmer error: of ObjectFilter must not be nil in ObjectToListFilter") 25 | } 26 | return &objectToListFilter{of} 27 | } 28 | 29 | type objectToListFilter struct { 30 | of ObjectFilter 31 | } 32 | 33 | // Filter implements ListFilter, but uses an ObjectFilter for the underlying logic. 34 | func (f objectToListFilter) Filter(objs ...runtime.Object) (retarr []runtime.Object, err error) { 35 | // Walk through all objects 36 | for _, obj := range objs { 37 | // Match them one-by-one against the ObjectFilter 38 | match, err := f.of.Filter(obj) 39 | if err != nil { 40 | return nil, err 41 | } 42 | // If the object matches, include it in the return array 43 | if match { 44 | retarr = append(retarr, obj) 45 | } 46 | } 47 | return 48 | } 49 | -------------------------------------------------------------------------------- /pkg/runtime/meta.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "sigs.k8s.io/yaml" 7 | ) 8 | 9 | // PartialObjectImpl is a struct implementing PartialObject, used for 10 | // unmarshalling unknown objects into this intermediate type 11 | // where .Name, .UID, .Kind and .APIVersion become easily available 12 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 13 | type PartialObjectImpl struct { 14 | metav1.TypeMeta `json:",inline"` 15 | metav1.ObjectMeta `json:"metadata"` 16 | } 17 | 18 | func (po *PartialObjectImpl) IsPartialObject() {} 19 | 20 | // This constructor ensures the PartialObjectImpl fields are not nil. 21 | // TODO: Make this multi-document-aware? 22 | func NewPartialObject(frame []byte) (PartialObject, error) { 23 | obj := &PartialObjectImpl{} 24 | 25 | // The yaml package supports both YAML and JSON. Don't use the serializer, as the APIType 26 | // wrapper is not registered in any scheme. 27 | if err := yaml.Unmarshal(frame, obj); err != nil { 28 | return nil, err 29 | } 30 | 31 | return obj, nil 32 | } 33 | 34 | var _ Object = &PartialObjectImpl{} 35 | var _ PartialObject = &PartialObjectImpl{} 36 | 37 | // Object is an union of the Object interfaces that are accessible for a 38 | // type that embeds both metav1.TypeMeta and metav1.ObjectMeta. 39 | type Object interface { 40 | runtime.Object 41 | metav1.ObjectMetaAccessor 42 | metav1.Object 43 | } 44 | 45 | // PartialObject is a partially-decoded object, where only metadata has been loaded. 46 | type PartialObject interface { 47 | Object 48 | 49 | // IsPartialObject is a dummy function for signalling that this is a partially-loaded object 50 | // i.e. only TypeMeta and ObjectMeta are stored in memory. 51 | IsPartialObject() 52 | } 53 | -------------------------------------------------------------------------------- /pkg/filter/uid.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/weaveworks/libgitops/pkg/runtime" 9 | "k8s.io/apimachinery/pkg/types" 10 | ) 11 | 12 | var ( 13 | // ErrInvalidFilterParams describes an error where invalid parameters were given 14 | // to a filter. 15 | ErrInvalidFilterParams = errors.New("invalid parameters given to filter") 16 | ) 17 | 18 | // UIDFilter implements ObjectFilter and ListOption. 19 | var _ ObjectFilter = UIDFilter{} 20 | var _ ListOption = UIDFilter{} 21 | 22 | // UIDFilter is an ObjectFilter that compares runtime.Object.GetUID() to 23 | // the UID field by either equality or prefix. The UID field is required, 24 | // otherwise ErrInvalidFilterParams is returned. 25 | type UIDFilter struct { 26 | // UID matches the object by .metadata.uid. 27 | // +required 28 | UID types.UID 29 | // MatchPrefix whether the UID-matching should be exact, or prefix-based. 30 | // +optional 31 | MatchPrefix bool 32 | } 33 | 34 | // Filter implements ObjectFilter 35 | func (f UIDFilter) Filter(obj runtime.Object) (bool, error) { 36 | // Require f.UID to always be set. 37 | if len(f.UID) == 0 { 38 | return false, fmt.Errorf("the UIDFilter.UID field must not be empty: %w", ErrInvalidFilterParams) 39 | } 40 | // If the UID should be matched by the prefix, use strings.HasPrefix 41 | if f.MatchPrefix { 42 | return strings.HasPrefix(string(obj.GetUID()), string(f.UID)), nil 43 | } 44 | // Otherwise, just use an equality check 45 | return f.UID == obj.GetUID(), nil 46 | } 47 | 48 | // ApplyToListOptions implements ListOption, and adds itself converted to 49 | // a ListFilter to ListOptions.Filters. 50 | func (f UIDFilter) ApplyToListOptions(target *ListOptions) error { 51 | target.Filters = append(target.Filters, ObjectToListFilter(f)) 52 | return nil 53 | } 54 | -------------------------------------------------------------------------------- /pkg/filter/name.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/weaveworks/libgitops/pkg/runtime" 8 | ) 9 | 10 | // NameFilter implements ObjectFilter and ListOption. 11 | var _ ObjectFilter = NameFilter{} 12 | var _ ListOption = NameFilter{} 13 | 14 | // NameFilter is an ObjectFilter that compares runtime.Object.GetName() 15 | // to the Name field by either equality or prefix. 16 | type NameFilter struct { 17 | // Name matches the object by .metadata.name. 18 | // +required 19 | Name string 20 | // Namespace matches the object by .metadata.namespace. If left as 21 | // an empty string, it is ignored when filtering. 22 | // +optional 23 | Namespace string 24 | // MatchPrefix whether the name (not namespace) matching should be exact, or prefix-based. 25 | // +optional 26 | MatchPrefix bool 27 | } 28 | 29 | // Filter implements ObjectFilter 30 | func (f NameFilter) Filter(obj runtime.Object) (bool, error) { 31 | // Require f.Name to always be set. 32 | if len(f.Name) == 0 { 33 | return false, fmt.Errorf("the NameFilter.Name field must not be empty: %w", ErrInvalidFilterParams) 34 | } 35 | 36 | // If f.Namespace is set, and it does not match the object, return false 37 | if len(f.Namespace) > 0 && f.Namespace != obj.GetNamespace() { 38 | return false, nil 39 | } 40 | 41 | // If the Name should be matched by the prefix, use strings.HasPrefix 42 | if f.MatchPrefix { 43 | return strings.HasPrefix(obj.GetName(), f.Name), nil 44 | } 45 | // Otherwise, just use an equality check 46 | return f.Name == obj.GetName(), nil 47 | } 48 | 49 | // ApplyToListOptions implements ListOption, and adds itself converted to 50 | // a ListFilter to ListOptions.Filters. 51 | func (f NameFilter) ApplyToListOptions(target *ListOptions) error { 52 | target.Filters = append(target.Filters, ObjectToListFilter(f)) 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/storage/key.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "github.com/weaveworks/libgitops/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | ) 7 | 8 | type kindKey schema.GroupVersionKind 9 | 10 | func (gvk kindKey) GetGroup() string { return gvk.Group } 11 | func (gvk kindKey) GetVersion() string { return gvk.Version } 12 | func (gvk kindKey) GetKind() string { return gvk.Kind } 13 | func (gvk kindKey) GetGVK() schema.GroupVersionKind { return schema.GroupVersionKind(gvk) } 14 | func (gvk kindKey) EqualsGVK(kind KindKey, respectVersion bool) bool { 15 | // Make sure kind and group match, otherwise return false 16 | if gvk.GetKind() != kind.GetKind() || gvk.GetGroup() != kind.GetGroup() { 17 | return false 18 | } 19 | // If we allow version mismatches (i.e. don't need to respect the version), return true 20 | if !respectVersion { 21 | return true 22 | } 23 | // Otherwise, return true if the version also is the same 24 | return gvk.GetVersion() == kind.GetVersion() 25 | } 26 | func (gvk kindKey) String() string { return gvk.GetGVK().String() } 27 | 28 | // kindKey implements KindKey. 29 | var _ KindKey = kindKey{} 30 | 31 | type KindKey interface { 32 | // String implements fmt.Stringer 33 | String() string 34 | 35 | GetGroup() string 36 | GetVersion() string 37 | GetKind() string 38 | GetGVK() schema.GroupVersionKind 39 | 40 | EqualsGVK(kind KindKey, respectVersion bool) bool 41 | } 42 | 43 | type ObjectKey interface { 44 | KindKey 45 | runtime.Identifyable 46 | } 47 | 48 | // objectKey implements ObjectKey. 49 | var _ ObjectKey = &objectKey{} 50 | 51 | type objectKey struct { 52 | KindKey 53 | runtime.Identifyable 54 | } 55 | 56 | func (key objectKey) String() string { return key.KindKey.String() + " " + key.GetIdentifier() } 57 | 58 | func NewKindKey(gvk schema.GroupVersionKind) KindKey { 59 | return kindKey(gvk) 60 | } 61 | 62 | func NewObjectKey(kind KindKey, id runtime.Identifyable) ObjectKey { 63 | return objectKey{kind, id} 64 | } 65 | -------------------------------------------------------------------------------- /pkg/runtime/identifiers.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "fmt" 5 | 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | ) 8 | 9 | // DefaultNamespace describes the default namespace name used for the system. 10 | const DefaultNamespace = "default" 11 | 12 | // Identifyable is an object which can be identified 13 | type Identifyable interface { 14 | // GetIdentifier can return e.g. a "namespace/name" combination, which is not guaranteed 15 | // to be unique world-wide, or alternatively a random SHA for instance 16 | GetIdentifier() string 17 | } 18 | 19 | type identifier string 20 | 21 | func (i identifier) GetIdentifier() string { return string(i) } 22 | 23 | type Metav1NameIdentifierFactory struct{} 24 | 25 | func (id Metav1NameIdentifierFactory) Identify(o interface{}) (Identifyable, bool) { 26 | switch obj := o.(type) { 27 | case metav1.Object: 28 | if len(obj.GetNamespace()) == 0 || len(obj.GetName()) == 0 { 29 | return nil, false 30 | } 31 | return NewIdentifier(fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName())), true 32 | } 33 | return nil, false 34 | } 35 | 36 | type ObjectUIDIdentifierFactory struct{} 37 | 38 | func (id ObjectUIDIdentifierFactory) Identify(o interface{}) (Identifyable, bool) { 39 | switch obj := o.(type) { 40 | case Object: 41 | if len(obj.GetUID()) == 0 { 42 | return nil, false 43 | } 44 | // TODO: Make sure that runtime.APIType works with this 45 | return NewIdentifier(string(obj.GetUID())), true 46 | } 47 | return nil, false 48 | } 49 | 50 | var ( 51 | // Metav1Identifier identifies an object using its metav1.ObjectMeta Name and Namespace 52 | Metav1NameIdentifier IdentifierFactory = Metav1NameIdentifierFactory{} 53 | // ObjectUIDIdentifier identifies an object using its libgitops/pkg/runtime.ObjectMeta UID field 54 | ObjectUIDIdentifier IdentifierFactory = ObjectUIDIdentifierFactory{} 55 | ) 56 | 57 | func NewIdentifier(str string) Identifyable { 58 | return identifier(str) 59 | } 60 | 61 | type IdentifierFactory interface { 62 | Identify(o interface{}) (id Identifyable, ok bool) 63 | } 64 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // Car represents a car 8 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 9 | type Car struct { 10 | metav1.TypeMeta `json:",inline"` 11 | // runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID 12 | // Name is available at the .metadata.name JSON path 13 | // ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string) 14 | metav1.ObjectMeta `json:"metadata"` 15 | 16 | Spec CarSpec `json:"spec"` 17 | Status CarStatus `json:"status"` 18 | } 19 | 20 | type CarSpec struct { 21 | Engine string `json:"engine"` 22 | YearModel string `json:"yearModel"` 23 | Brand string `json:"brand"` 24 | } 25 | 26 | type CarStatus struct { 27 | VehicleStatus `json:",inline"` 28 | Persons uint64 `json:"persons"` 29 | } 30 | 31 | type VehicleStatus struct { 32 | Speed float64 `json:"speed"` 33 | Acceleration float64 `json:"acceleration"` 34 | Distance uint64 `json:"distance"` 35 | } 36 | 37 | // Motorcycle represents a motorcycle 38 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 39 | type Motorcycle struct { 40 | metav1.TypeMeta `json:",inline"` 41 | // runtime.ObjectMeta is also embedded into the struct, and defines the human-readable name, and the machine-readable ID 42 | // Name is available at the .metadata.name JSON path 43 | // ID is available at the .metadata.uid JSON path (the Go type is k8s.io/apimachinery/pkg/types.UID, which is only a typed string) 44 | metav1.ObjectMeta `json:"metadata"` 45 | 46 | Spec MotorcycleSpec `json:"spec"` 47 | Status MotorcycleStatus `json:"status"` 48 | } 49 | 50 | type MotorcycleSpec struct { 51 | Color string `json:"color"` 52 | BodyType string `json:"bodyType"` 53 | } 54 | 55 | type MotorcycleStatus struct { 56 | VehicleStatus `json:",inline"` 57 | CurrentWeight float64 `json:"currentWeight"` 58 | } 59 | -------------------------------------------------------------------------------- /cmd/sample-app/client/client.go: -------------------------------------------------------------------------------- 1 | // TODO: Docs 2 | 3 | // +build ignore 4 | 5 | package client 6 | 7 | import ( 8 | api "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 9 | "github.com/weaveworks/libgitops/pkg/client" 10 | "github.com/weaveworks/libgitops/pkg/runtime" 11 | "github.com/weaveworks/libgitops/pkg/storage" 12 | 13 | "k8s.io/apimachinery/pkg/runtime/schema" 14 | ) 15 | 16 | // TODO: Autogenerate this! 17 | 18 | // NewClient creates a client for the specified storage 19 | func NewClient(s storage.Storage) *Client { 20 | return &Client{ 21 | SampleInternalClient: NewSampleInternalClient(s), 22 | } 23 | } 24 | 25 | // Client is a struct providing high-level access to objects in a storage 26 | // The resource-specific client interfaces are automatically generated based 27 | // off client_resource_template.go. The auto-generation can be done with hack/client.sh 28 | // At the moment SampleInternalClient is the default client. If more than this client 29 | // is created in the future, the SampleInternalClient will be accessible under 30 | // Client.SampleInternal() instead. 31 | type Client struct { 32 | *SampleInternalClient 33 | } 34 | 35 | func NewSampleInternalClient(s storage.Storage) *SampleInternalClient { 36 | return &SampleInternalClient{ 37 | storage: s, 38 | dynamicClients: map[schema.GroupVersionKind]client.DynamicClient{}, 39 | gv: api.SchemeGroupVersion, 40 | } 41 | } 42 | 43 | type SampleInternalClient struct { 44 | storage storage.Storage 45 | gv schema.GroupVersion 46 | carClient CarClient 47 | motorcycleClient MotorcycleClient 48 | dynamicClients map[schema.GroupVersionKind]client.DynamicClient 49 | } 50 | 51 | // Dynamic returns the DynamicClient for the Client instance, for the specific kind 52 | func (c *SampleInternalClient) Dynamic(kind runtime.Kind) (dc client.DynamicClient) { 53 | var ok bool 54 | gvk := c.gv.WithKind(kind.Title()) 55 | if dc, ok = c.dynamicClients[gvk]; !ok { 56 | dc = client.NewDynamicClient(c.storage, gvk) 57 | c.dynamicClients[gvk] = dc 58 | } 59 | 60 | return 61 | } 62 | -------------------------------------------------------------------------------- /pkg/serializer/defaulter.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "k8s.io/apimachinery/pkg/runtime" 5 | "k8s.io/apimachinery/pkg/runtime/schema" 6 | "k8s.io/apimachinery/pkg/util/errors" 7 | ) 8 | 9 | func newDefaulter(scheme *runtime.Scheme) *defaulter { 10 | return &defaulter{scheme} 11 | } 12 | 13 | type defaulter struct { 14 | scheme *runtime.Scheme 15 | } 16 | 17 | // NewDefaultedObject returns a new, defaulted object. It is essentially scheme.New() and 18 | // scheme.Default(obj), but with extra logic to also cover internal versions. 19 | // Important to note here is that the TypeMeta information is NOT applied automatically. 20 | func (d *defaulter) NewDefaultedObject(gvk schema.GroupVersionKind) (runtime.Object, error) { 21 | obj, err := d.scheme.New(gvk) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | // Default the new object, this will take care of internal defaulting automatically 27 | if err := d.Default(obj); err != nil { 28 | return nil, err 29 | } 30 | 31 | return obj, nil 32 | } 33 | 34 | func (d *defaulter) Default(objs ...runtime.Object) error { 35 | errs := []error{} 36 | for _, obj := range objs { 37 | errs = append(errs, d.runDefaulting(obj)) 38 | } 39 | return errors.NewAggregate(errs) 40 | } 41 | 42 | func (d *defaulter) runDefaulting(obj runtime.Object) error { 43 | // First, get the groupversionkind of the object 44 | gvk, err := GVKForObject(d.scheme, obj) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | // If the version is external, just default it and return. 50 | if gvk.Version != runtime.APIVersionInternal { 51 | d.scheme.Default(obj) 52 | return nil 53 | } 54 | 55 | // We know that the current object is internal 56 | // Get the preferred external version... 57 | gv, err := prioritizedVersionForGroup(d.scheme, gvk.Group) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | // ...and make a new object of it 63 | external, err := d.scheme.New(gv.WithKind(gvk.Kind)) 64 | if err != nil { 65 | return err 66 | } 67 | // Convert the internal object to the external 68 | if err := d.scheme.Convert(obj, external, nil); err != nil { 69 | return err 70 | } 71 | // Default the external 72 | d.scheme.Default(external) 73 | // And convert back to internal 74 | return d.scheme.Convert(external, obj, nil) 75 | 76 | } 77 | -------------------------------------------------------------------------------- /pkg/runtime/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Code generated by deepcopy-gen. DO NOT EDIT. 4 | 5 | package runtime 6 | 7 | import ( 8 | pkgruntime "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 12 | func (in *Metav1NameIdentifierFactory) DeepCopyInto(out *Metav1NameIdentifierFactory) { 13 | *out = *in 14 | return 15 | } 16 | 17 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metav1NameIdentifierFactory. 18 | func (in *Metav1NameIdentifierFactory) DeepCopy() *Metav1NameIdentifierFactory { 19 | if in == nil { 20 | return nil 21 | } 22 | out := new(Metav1NameIdentifierFactory) 23 | in.DeepCopyInto(out) 24 | return out 25 | } 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *ObjectUIDIdentifierFactory) DeepCopyInto(out *ObjectUIDIdentifierFactory) { 29 | *out = *in 30 | return 31 | } 32 | 33 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectUIDIdentifierFactory. 34 | func (in *ObjectUIDIdentifierFactory) DeepCopy() *ObjectUIDIdentifierFactory { 35 | if in == nil { 36 | return nil 37 | } 38 | out := new(ObjectUIDIdentifierFactory) 39 | in.DeepCopyInto(out) 40 | return out 41 | } 42 | 43 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 44 | func (in *PartialObjectImpl) DeepCopyInto(out *PartialObjectImpl) { 45 | *out = *in 46 | out.TypeMeta = in.TypeMeta 47 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 48 | return 49 | } 50 | 51 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PartialObjectImpl. 52 | func (in *PartialObjectImpl) DeepCopy() *PartialObjectImpl { 53 | if in == nil { 54 | return nil 55 | } 56 | out := new(PartialObjectImpl) 57 | in.DeepCopyInto(out) 58 | return out 59 | } 60 | 61 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new pkgruntime.Object. 62 | func (in *PartialObjectImpl) DeepCopyObject() pkgruntime.Object { 63 | if c := in.DeepCopy(); c != nil { 64 | return c 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /pkg/util/watcher/filewatcher_test.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/rjeczalik/notify" 7 | "golang.org/x/sys/unix" 8 | ) 9 | 10 | type testEventWrapper struct { 11 | event notify.Event 12 | } 13 | 14 | func (t *testEventWrapper) Event() notify.Event { 15 | return t.event 16 | } 17 | 18 | func (t *testEventWrapper) Path() string { return "" } 19 | func (t *testEventWrapper) Sys() interface{} { return &unix.InotifyEvent{} } 20 | 21 | var _ notify.EventInfo = &testEventWrapper{} 22 | 23 | func testEvent(event notify.Event) notify.EventInfo { 24 | return &testEventWrapper{event} 25 | } 26 | 27 | var testEvents = []notifyEvents{ 28 | { 29 | testEvent(notify.InDelete), 30 | testEvent(notify.InCloseWrite), 31 | testEvent(notify.InMovedTo), 32 | }, 33 | { 34 | testEvent(notify.InCloseWrite), 35 | testEvent(notify.InDelete), 36 | testEvent(notify.InDelete), 37 | }, 38 | { 39 | testEvent(notify.InCloseWrite), 40 | testEvent(notify.InMovedTo), 41 | testEvent(notify.InMovedFrom), 42 | testEvent(notify.InDelete), 43 | }, 44 | { 45 | testEvent(notify.InDelete), 46 | testEvent(notify.InCloseWrite), 47 | }, 48 | { 49 | testEvent(notify.InCloseWrite), 50 | testEvent(notify.InDelete), 51 | }, 52 | } 53 | 54 | var targets = []FileEvents{ 55 | { 56 | FileEventModify, 57 | }, 58 | { 59 | FileEventDelete, 60 | }, 61 | { 62 | FileEventModify, 63 | FileEventMove, 64 | FileEventDelete, 65 | }, 66 | { 67 | FileEventModify, 68 | }, 69 | {}, 70 | } 71 | 72 | func extractEvents(updates FileUpdates) (events FileEvents) { 73 | for _, update := range updates { 74 | events = append(events, update.Event) 75 | } 76 | 77 | return 78 | } 79 | 80 | func eventsEqual(a, b FileEvents) bool { 81 | if len(a) != len(b) { 82 | return false 83 | } 84 | 85 | for i := range a { 86 | if a[i] != b[i] { 87 | return false 88 | } 89 | } 90 | 91 | return true 92 | } 93 | 94 | func TestEventConcatenation(t *testing.T) { 95 | for i, e := range testEvents { 96 | result := extractEvents((&FileWatcher{}).concatenateEvents(e)) 97 | if !eventsEqual(result, targets[i]) { 98 | t.Errorf("wrong concatenation result: %v != %v", result, targets[i]) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cmd/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "time" 11 | 12 | "github.com/labstack/echo" 13 | "github.com/spf13/pflag" 14 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/v1alpha1" 15 | "github.com/weaveworks/libgitops/cmd/sample-app/version" 16 | "github.com/weaveworks/libgitops/pkg/runtime" 17 | "github.com/weaveworks/libgitops/pkg/storage" 18 | ) 19 | 20 | var ( 21 | CarGVK = v1alpha1.SchemeGroupVersion.WithKind("Car") 22 | ) 23 | 24 | func init() { 25 | rand.Seed(time.Now().UnixNano()) 26 | } 27 | 28 | func CarKeyForName(name string) storage.ObjectKey { 29 | return storage.NewObjectKey(storage.NewKindKey(CarGVK), runtime.NewIdentifier("default/"+name)) 30 | } 31 | 32 | func NewCar(name string) *v1alpha1.Car { 33 | obj := &v1alpha1.Car{} 34 | obj.Name = name 35 | obj.Namespace = "default" 36 | obj.Spec.Brand = fmt.Sprintf("Acura-%03d", rand.Intn(1000)) 37 | 38 | return obj 39 | } 40 | 41 | func SetNewCarStatus(s storage.Storage, key storage.ObjectKey) error { 42 | obj, err := s.Get(key) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | car := obj.(*v1alpha1.Car) 48 | car.Status.Distance = rand.Uint64() 49 | car.Status.Speed = rand.Float64() * 100 50 | 51 | return s.Update(car) 52 | } 53 | 54 | func ParseVersionFlag() { 55 | var showVersion bool 56 | 57 | pflag.BoolVar(&showVersion, "version", showVersion, "Show version information and exit") 58 | pflag.Parse() 59 | if showVersion { 60 | fmt.Printf("sample-app version: %#v\n", version.Get()) 61 | os.Exit(0) 62 | } 63 | } 64 | 65 | func NewEcho() *echo.Echo { 66 | // Set up the echo server 67 | e := echo.New() 68 | e.Debug = true 69 | e.GET("/", func(c echo.Context) error { 70 | return c.String(http.StatusOK, "Welcome!") 71 | }) 72 | return e 73 | } 74 | 75 | func StartEcho(e *echo.Echo) error { 76 | // Start the server 77 | go func() { 78 | if err := e.Start(":8888"); err != nil { 79 | e.Logger.Info("shutting down the server") 80 | } 81 | }() 82 | 83 | quit := make(chan os.Signal, 1) 84 | signal.Notify(quit, os.Interrupt) 85 | 86 | // Wait for interrupt signal to gracefully shutdown the application with a timeout of 10 seconds 87 | <-quit 88 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 89 | defer cancel() 90 | 91 | return e.Shutdown(ctx) 92 | } 93 | -------------------------------------------------------------------------------- /pkg/gitdir/transport.go: -------------------------------------------------------------------------------- 1 | package gitdir 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/fluxcd/go-git-providers/gitprovider" 7 | "github.com/fluxcd/toolkit/pkg/ssh/knownhosts" 8 | "github.com/go-git/go-git/v5/plumbing/transport" 9 | "github.com/go-git/go-git/v5/plumbing/transport/http" 10 | "github.com/go-git/go-git/v5/plumbing/transport/ssh" 11 | ) 12 | 13 | // AuthMethod specifies the authentication method and related credentials for connecting 14 | // to a Git repository. 15 | type AuthMethod interface { 16 | // This AuthMethod is a superset of the go-git AuthMethod 17 | transport.AuthMethod 18 | // TransportType defines what transport type should be used with this method 19 | TransportType() gitprovider.TransportType 20 | } 21 | 22 | // NewSSHAuthMethod creates a new AuthMethod for the Git SSH protocol, using a given 23 | // identity and known_hosts file. 24 | // 25 | // identityFile is the bytes of e.g. ~/.ssh/id_rsa, given that ~/.ssh/id_rsa.pub is 26 | // registered with and trusted by the Git provider. 27 | // 28 | // knownHostsFile should be the file content of the known_hosts file to use for remote (e.g. GitHub) 29 | // public key verification. 30 | // If you want to use the default git CLI behavior, populate this byte slice with contents from 31 | // ioutil.ReadFile("~/.ssh/known_hosts"). 32 | func NewSSHAuthMethod(identityFile, knownHostsFile []byte) (AuthMethod, error) { 33 | if len(identityFile) == 0 || len(knownHostsFile) == 0 { 34 | return nil, errors.New("invalid identityFile, knownHostsFile options") 35 | } 36 | 37 | pk, err := ssh.NewPublicKeys("git", identityFile, "") 38 | if err != nil { 39 | return nil, err 40 | } 41 | callback, err := knownhosts.New(knownHostsFile) 42 | if err != nil { 43 | return nil, err 44 | } 45 | pk.HostKeyCallback = callback 46 | 47 | return &authMethod{ 48 | AuthMethod: pk, 49 | t: gitprovider.TransportTypeGit, 50 | }, nil 51 | } 52 | 53 | func NewHTTPSAuthMethod(username, password string) (AuthMethod, error) { 54 | if len(username) == 0 || len(password) == 0 { 55 | return nil, errors.New("invalid username, password options") 56 | } 57 | return &authMethod{ 58 | AuthMethod: &http.BasicAuth{ 59 | Username: username, 60 | Password: password, 61 | }, 62 | t: gitprovider.TransportTypeHTTPS, 63 | }, nil 64 | } 65 | 66 | type authMethod struct { 67 | transport.AuthMethod 68 | t gitprovider.TransportType 69 | } 70 | 71 | func (a *authMethod) TransportType() gitprovider.TransportType { 72 | return a.t 73 | } 74 | -------------------------------------------------------------------------------- /pkg/storage/transaction/commit.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/fluxcd/go-git-providers/validation" 7 | ) 8 | 9 | // CommitResult describes a result of a transaction. 10 | type CommitResult interface { 11 | // GetAuthorName describes the author's name (as per git config) 12 | // +required 13 | GetAuthorName() string 14 | // GetAuthorEmail describes the author's email (as per git config) 15 | // +required 16 | GetAuthorEmail() string 17 | // GetTitle describes the change concisely, so it can be used as a commit message or PR title. 18 | // +required 19 | GetTitle() string 20 | // GetDescription contains optional extra information about the change. 21 | // +optional 22 | GetDescription() string 23 | 24 | // GetMessage returns GetTitle() followed by a newline and GetDescription(), if set. 25 | GetMessage() string 26 | // Validate validates that all required fields are set, and given data is valid. 27 | Validate() error 28 | } 29 | 30 | // GenericCommitResult implements CommitResult. 31 | var _ CommitResult = &GenericCommitResult{} 32 | 33 | // GenericCommitResult implements CommitResult. 34 | type GenericCommitResult struct { 35 | // AuthorName describes the author's name (as per git config) 36 | // +required 37 | AuthorName string 38 | // AuthorEmail describes the author's email (as per git config) 39 | // +required 40 | AuthorEmail string 41 | // Title describes the change concisely, so it can be used as a commit message or PR title. 42 | // +required 43 | Title string 44 | // Description contains optional extra information about the change. 45 | // +optional 46 | Description string 47 | } 48 | 49 | func (r *GenericCommitResult) GetAuthorName() string { 50 | return r.AuthorName 51 | } 52 | func (r *GenericCommitResult) GetAuthorEmail() string { 53 | return r.AuthorEmail 54 | } 55 | func (r *GenericCommitResult) GetTitle() string { 56 | return r.Title 57 | } 58 | func (r *GenericCommitResult) GetDescription() string { 59 | return r.Description 60 | } 61 | func (r *GenericCommitResult) GetMessage() string { 62 | if len(r.Description) == 0 { 63 | return r.Title 64 | } 65 | return fmt.Sprintf("%s\n%s", r.Title, r.Description) 66 | } 67 | func (r *GenericCommitResult) Validate() error { 68 | v := validation.New("GenericCommitResult") 69 | if len(r.AuthorName) == 0 { 70 | v.Required("AuthorName") 71 | } 72 | if len(r.AuthorEmail) == 0 { 73 | v.Required("AuthorEmail") 74 | } 75 | if len(r.Title) == 0 { 76 | v.Required("Title") 77 | } 78 | return v.Error() 79 | } 80 | -------------------------------------------------------------------------------- /pkg/serializer/objectmeta.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | ) 7 | 8 | // toMetaObject converts a runtime.Object to a metav1.Object (containing methods that allow modification of 9 | // e.g. annotations, labels, name, namespaces, etc.), and reports whether this cast was successful 10 | func toMetaObject(obj runtime.Object) (metav1.Object, bool) { 11 | // Check if the object has ObjectMeta embedded. If it does, it can be casted to 12 | // an ObjectMetaAccessor, which allows us to get operate directly on the ObjectMeta field 13 | acc, ok := obj.(metav1.ObjectMetaAccessor) 14 | if !ok { 15 | return nil, false 16 | } 17 | 18 | // If, by accident, someone embedded ObjectMeta in their object as a pointer, and forgot to set it, this 19 | // check makes sure we don't get a nil dereference panic later 20 | om, ok := acc.GetObjectMeta().(*metav1.ObjectMeta) 21 | if !ok || om == nil { 22 | return nil, false 23 | } 24 | 25 | // Perform the cast, we can now be sure that ObjectMeta is embedded and non-nil 26 | metaObj, ok := obj.(metav1.Object) 27 | return metaObj, ok 28 | } 29 | 30 | func getAnnotation(metaObj metav1.Object, key string) (string, bool) { 31 | // Get the annotations map 32 | a := metaObj.GetAnnotations() 33 | if a == nil { 34 | return "", false 35 | } 36 | 37 | // Get the value like normal and return 38 | val, ok := a[key] 39 | return val, ok 40 | } 41 | 42 | func setAnnotation(metaObj metav1.Object, key, val string) { 43 | // Get the annotations map 44 | a := metaObj.GetAnnotations() 45 | 46 | // If the annotations are nil, create a new map 47 | if a == nil { 48 | a = map[string]string{} 49 | } 50 | 51 | // Set the key-value mapping and write back to the object 52 | a[key] = val 53 | // This wouldn't be needed if a was non-nil (as maps are reference types), but in the case 54 | // of the map being nil, this is a must in order to apply the change 55 | metaObj.SetAnnotations(a) 56 | } 57 | 58 | func deleteAnnotation(metaObj metav1.Object, key string) { 59 | // Get the annotations map 60 | a := metaObj.GetAnnotations() 61 | 62 | // If the object doesn't have any annotations or that specific one, never mind 63 | if a == nil { 64 | return 65 | } 66 | _, ok := a[key] 67 | if !ok { 68 | return 69 | } 70 | 71 | // Delete the internal annotation and write back to the object. 72 | // The map is passed by reference so this automatically updates the underlying annotations object. 73 | delete(a, key) 74 | } 75 | -------------------------------------------------------------------------------- /cmd/sample-watch/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | 9 | "github.com/labstack/echo" 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/pflag" 12 | "github.com/weaveworks/libgitops/cmd/common" 13 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" 14 | "github.com/weaveworks/libgitops/pkg/logs" 15 | "github.com/weaveworks/libgitops/pkg/serializer" 16 | "github.com/weaveworks/libgitops/pkg/storage/watch" 17 | "github.com/weaveworks/libgitops/pkg/storage/watch/update" 18 | ) 19 | 20 | var watchDirFlag = pflag.String("watch-dir", "/tmp/libgitops/watch", "Where to watch for YAML/JSON manifests") 21 | 22 | func main() { 23 | // Parse the version flag 24 | common.ParseVersionFlag() 25 | 26 | // Run the application 27 | if err := run(); err != nil { 28 | fmt.Println(err) 29 | os.Exit(1) 30 | } 31 | } 32 | 33 | func run() error { 34 | // Create the watch directory 35 | if err := os.MkdirAll(*watchDirFlag, 0755); err != nil { 36 | return err 37 | } 38 | 39 | // Set the log level 40 | logs.Logger.SetLevel(logrus.InfoLevel) 41 | 42 | watchStorage, err := watch.NewManifestStorage(*watchDirFlag, scheme.Serializer) 43 | if err != nil { 44 | return err 45 | } 46 | defer func() { _ = watchStorage.Close() }() 47 | 48 | updates := make(chan update.Update, 4096) 49 | watchStorage.SetUpdateStream(updates) 50 | 51 | go func() { 52 | for upd := range updates { 53 | logrus.Infof("Got %s update for: %v %v", upd.Event, upd.PartialObject.GetObjectKind().GroupVersionKind(), upd.PartialObject.GetObjectMeta()) 54 | } 55 | }() 56 | 57 | e := common.NewEcho() 58 | 59 | e.GET("/watch/:name", func(c echo.Context) error { 60 | name := c.Param("name") 61 | if len(name) == 0 { 62 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 63 | } 64 | 65 | obj, err := watchStorage.Get(common.CarKeyForName(name)) 66 | if err != nil { 67 | return err 68 | } 69 | var content bytes.Buffer 70 | if err := scheme.Serializer.Encoder().Encode(serializer.NewJSONFrameWriter(&content), obj); err != nil { 71 | return err 72 | } 73 | return c.JSONBlob(http.StatusOK, content.Bytes()) 74 | }) 75 | 76 | e.PUT("/watch/:name", func(c echo.Context) error { 77 | name := c.Param("name") 78 | if len(name) == 0 { 79 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 80 | } 81 | 82 | if err := common.SetNewCarStatus(watchStorage, common.CarKeyForName(name)); err != nil { 83 | return err 84 | } 85 | return c.String(200, "OK!") 86 | }) 87 | 88 | return common.StartEcho(e) 89 | } 90 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | UID_GID ?= $(shell id -u):$(shell id -g) 2 | GO_VERSION ?= 1.14.4 3 | GIT_VERSION := $(shell hack/ldflags.sh --version-only) 4 | PROJECT := github.com/weaveworks/libgitops 5 | BOUNDING_API_DIRS := ${PROJECT}/cmd/apis/sample 6 | API_DIRS := ${PROJECT}/cmd/sample-app/apis/sample,${PROJECT}/cmd/sample-app/apis/sample/v1alpha1 7 | SRC_PKGS := cmd pkg 8 | DOCKER_ARGS := --rm 9 | CACHE_DIR := $(shell pwd)/bin/cache 10 | API_DOCS := api/sample-app.md api/runtime.md 11 | BINARIES := bin/sample-app bin/sample-gitops bin/sample-watch 12 | 13 | # If we're not running in CI, run Docker interactively 14 | ifndef CI 15 | DOCKER_ARGS += -it 16 | endif 17 | 18 | all: docker-binaries 19 | 20 | binaries: $(BINARIES) 21 | $(BINARIES): bin/%: 22 | go mod download 23 | CGO_ENABLED=0 go build -ldflags "$(shell ./hack/ldflags.sh)" -o "./bin/$*" "./cmd/$*" 24 | 25 | docker-%: 26 | mkdir -p "${CACHE_DIR}/go" "${CACHE_DIR}/cache" 27 | docker run ${DOCKER_ARGS} \ 28 | -u "${UID_GID}" \ 29 | -v "${CACHE_DIR}/go":/go \ 30 | -v "${CACHE_DIR}/cache":/.cache/go-build \ 31 | -v "$(shell pwd):/go/src/${PROJECT}" \ 32 | -w "/go/src/${PROJECT}" \ 33 | "golang:${GO_VERSION}" make $* 34 | 35 | test: docker-test-internal 36 | test-internal: 37 | go test -v $(addsuffix /...,$(addprefix ./,${SRC_PKGS})) 38 | 39 | tidy: docker-tidy-internal 40 | tidy-internal: /go/bin/goimports 41 | go mod tidy 42 | hack/generate-client.sh 43 | gofmt -s -w ${SRC_PKGS} 44 | goimports -w ${SRC_PKGS} 45 | 46 | autogen: docker-autogen-internal 47 | autogen-internal: /go/bin/deepcopy-gen /go/bin/defaulter-gen /go/bin/conversion-gen /go/bin/openapi-gen 48 | # Let the boilerplate be empty 49 | touch /tmp/boilerplate 50 | 51 | /go/bin/deepcopy-gen \ 52 | --input-dirs ${API_DIRS},${PROJECT}/pkg/runtime \ 53 | --bounding-dirs ${BOUNDING_API_DIRS} \ 54 | -O zz_generated.deepcopy \ 55 | -h /tmp/boilerplate 56 | 57 | /go/bin/defaulter-gen \ 58 | --input-dirs ${API_DIRS} \ 59 | -O zz_generated.defaults \ 60 | -h /tmp/boilerplate 61 | 62 | /go/bin/conversion-gen \ 63 | --input-dirs ${API_DIRS} \ 64 | -O zz_generated.conversion \ 65 | -h /tmp/boilerplate 66 | 67 | /go/bin/openapi-gen \ 68 | --input-dirs ${API_DIRS} \ 69 | --output-package ${PROJECT}/api/openapi \ 70 | --report-filename api/openapi/violations.txt \ 71 | -h /tmp/boilerplate 72 | 73 | # These commands modify the environment, perform cleanup 74 | $(MAKE) tidy-internal 75 | 76 | /go/bin/deepcopy-gen /go/bin/defaulter-gen /go/bin/conversion-gen: /go/bin/%: 77 | go get k8s.io/code-generator/cmd/$* 78 | 79 | /go/bin/openapi-gen: 80 | go get k8s.io/kube-openapi/cmd/openapi-gen 81 | 82 | /go/bin/goimports: 83 | go get golang.org/x/tools/cmd/goimports 84 | 85 | .PHONY: $(BINARIES) -------------------------------------------------------------------------------- /cmd/sample-app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | 9 | "github.com/labstack/echo" 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/pflag" 12 | "github.com/weaveworks/libgitops/cmd/common" 13 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" 14 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/v1alpha1" 15 | "github.com/weaveworks/libgitops/pkg/logs" 16 | "github.com/weaveworks/libgitops/pkg/runtime" 17 | "github.com/weaveworks/libgitops/pkg/serializer" 18 | "github.com/weaveworks/libgitops/pkg/storage" 19 | ) 20 | 21 | var manifestDirFlag = pflag.String("data-dir", "/tmp/libgitops/manifest", "Where to store the YAML files") 22 | 23 | func main() { 24 | // Parse the version flag 25 | common.ParseVersionFlag() 26 | 27 | // Run the application 28 | if err := run(); err != nil { 29 | fmt.Println(err) 30 | os.Exit(1) 31 | } 32 | } 33 | 34 | func run() error { 35 | // Create the manifest directory 36 | if err := os.MkdirAll(*manifestDirFlag, 0755); err != nil { 37 | return err 38 | } 39 | 40 | // Set the log level 41 | logs.Logger.SetLevel(logrus.InfoLevel) 42 | 43 | plainStorage := storage.NewGenericStorage( 44 | storage.NewGenericRawStorage(*manifestDirFlag, v1alpha1.SchemeGroupVersion, serializer.ContentTypeYAML), 45 | scheme.Serializer, 46 | []runtime.IdentifierFactory{runtime.Metav1NameIdentifier}, 47 | ) 48 | defer func() { _ = plainStorage.Close() }() 49 | 50 | e := common.NewEcho() 51 | 52 | e.GET("/plain/:name", func(c echo.Context) error { 53 | name := c.Param("name") 54 | if len(name) == 0 { 55 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 56 | } 57 | 58 | obj, err := plainStorage.Get(common.CarKeyForName(name)) 59 | if err != nil { 60 | return err 61 | } 62 | var content bytes.Buffer 63 | if err := scheme.Serializer.Encoder().Encode(serializer.NewJSONFrameWriter(&content), obj); err != nil { 64 | return err 65 | } 66 | return c.JSONBlob(http.StatusOK, content.Bytes()) 67 | }) 68 | 69 | e.POST("/plain/:name", func(c echo.Context) error { 70 | name := c.Param("name") 71 | if len(name) == 0 { 72 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 73 | } 74 | 75 | if err := plainStorage.Create(common.NewCar(name)); err != nil { 76 | return err 77 | } 78 | return c.String(200, "OK!") 79 | }) 80 | 81 | e.PUT("/plain/:name", func(c echo.Context) error { 82 | name := c.Param("name") 83 | if len(name) == 0 { 84 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 85 | } 86 | 87 | if err := common.SetNewCarStatus(plainStorage, common.CarKeyForName(name)); err != nil { 88 | return err 89 | } 90 | return c.String(200, "OK!") 91 | }) 92 | 93 | return common.StartEcho(e) 94 | } 95 | -------------------------------------------------------------------------------- /pkg/serializer/frame_reader_test.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "reflect" 7 | "strings" 8 | "testing" 9 | 10 | "k8s.io/apimachinery/pkg/runtime/serializer/json" 11 | ) 12 | 13 | const ( 14 | fooYAML = `kind: Foo 15 | apiVersion: bar/v1 16 | a: b1234567890 17 | c: d1234567890 18 | e: f1234567890 19 | hello: true` 20 | 21 | barYAML = `kind: Bar 22 | apiVersion: foo/v1 23 | a: b1234567890 24 | c: d1234567890 25 | e: f1234567890 26 | hello: false` 27 | 28 | bazYAML = `baz: true` 29 | 30 | testYAML = "\n---\n" + fooYAML + "\n---\n" + barYAML + "\n---\n" + bazYAML 31 | ) 32 | 33 | func Test_FrameReader_ReadFrame(t *testing.T) { 34 | testYAMLReadCloser := json.YAMLFramer.NewFrameReader(ioutil.NopCloser(strings.NewReader(testYAML))) 35 | 36 | type fields struct { 37 | rc io.ReadCloser 38 | bufSize int 39 | maxFrameSize int 40 | } 41 | type result struct { 42 | wantB []byte 43 | wantErr bool 44 | } 45 | tests := []struct { 46 | name string 47 | fields fields 48 | wants []result 49 | }{ 50 | { 51 | name: "three-document YAML case", 52 | fields: fields{ 53 | rc: testYAMLReadCloser, 54 | bufSize: 16, 55 | maxFrameSize: 1024, 56 | }, 57 | wants: []result{ 58 | { 59 | wantB: []byte(fooYAML), 60 | wantErr: false, 61 | }, 62 | { 63 | wantB: []byte(barYAML), 64 | wantErr: false, 65 | }, 66 | { 67 | wantB: []byte(bazYAML), 68 | wantErr: false, 69 | }, 70 | { 71 | wantB: nil, 72 | wantErr: true, 73 | }, 74 | }, 75 | }, 76 | { 77 | name: "maximum size reached", 78 | fields: fields{ 79 | rc: testYAMLReadCloser, 80 | bufSize: 16, 81 | maxFrameSize: 32, 82 | }, 83 | wants: []result{ 84 | { 85 | wantB: nil, 86 | wantErr: true, 87 | }, 88 | }, 89 | }, 90 | } 91 | for _, tt := range tests { 92 | t.Run(tt.name, func(t *testing.T) { 93 | rf := &frameReader{ 94 | rc: tt.fields.rc, 95 | bufSize: tt.fields.bufSize, 96 | maxFrameSize: tt.fields.maxFrameSize, 97 | } 98 | for _, expected := range tt.wants { 99 | gotB, err := rf.ReadFrame() 100 | if (err != nil) != expected.wantErr { 101 | t.Errorf("frameReader.ReadFrame() error = %v, wantErr %v", err, expected.wantErr) 102 | return 103 | } 104 | if len(gotB) < len(expected.wantB) { 105 | t.Errorf("frameReader.ReadFrame(): got smaller slice %v than expected %v", gotB, expected.wantB) 106 | return 107 | } 108 | if !reflect.DeepEqual(gotB[:len(expected.wantB)], expected.wantB) { 109 | t.Errorf("frameReader.ReadFrame() = %v, want %v", gotB, expected.wantB) 110 | } 111 | } 112 | }) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /pkg/storage/cache/object.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | /* 4 | 5 | TODO: Revisit if we need this file/package in the future. 6 | 7 | import ( 8 | log "github.com/sirupsen/logrus" 9 | "github.com/weaveworks/libgitops/pkg/runtime" 10 | "github.com/weaveworks/libgitops/pkg/storage" 11 | ) 12 | 13 | type cacheObject struct { 14 | storage storage.Storage 15 | object runtime.Object 16 | checksum string 17 | apiType bool 18 | } 19 | 20 | func newCacheObject(s storage.Storage, object runtime.Object, apiType bool) (c *cacheObject, err error) { 21 | c = &cacheObject{ 22 | storage: s, 23 | object: object, 24 | apiType: apiType, 25 | } 26 | 27 | if c.checksum, err = s.Checksum(c.object.GroupVersionKind(), c.object.GetUID()); err != nil { 28 | c = nil 29 | } 30 | 31 | return 32 | } 33 | 34 | // loadFull returns the full Object, loading it only if it hasn't been cached before or the checksum has changed 35 | func (c *cacheObject) loadFull() (runtime.Object, error) { 36 | var checksum string 37 | reload := c.apiType 38 | 39 | if !reload { 40 | if chk, err := c.storage.Checksum(c.object.GroupVersionKind(), c.object.GetUID()); err != nil { 41 | return nil, err 42 | } else if chk != c.checksum { 43 | log.Tracef("cacheObject: %q invalidated, checksum mismatch: %q -> %q", c.object.GetName(), c.checksum, chk) 44 | checksum = chk 45 | reload = true 46 | } else { 47 | log.Tracef("cacheObject: %q checksum: %q", c.object.GetName(), c.checksum) 48 | } 49 | } 50 | 51 | if reload { 52 | log.Tracef("cacheObject: full load triggered for %q", c.object.GetName()) 53 | obj, err := c.storage.Get(c.object.GroupVersionKind(), c.object.GetUID()) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | // Only apply the change after a successful Get 59 | c.object = obj 60 | c.apiType = false 61 | 62 | if len(checksum) > 0 { 63 | c.checksum = checksum 64 | } 65 | } 66 | 67 | return c.object, nil 68 | } 69 | 70 | // loadAPI returns the APIType of the Object, loading it only if the checksum has changed 71 | func (c *cacheObject) loadAPI() (runtime.Object, error) { 72 | if chk, err := c.storage.Checksum(c.object.GroupVersionKind(), c.object.GetUID()); err != nil { 73 | return nil, err 74 | } else if chk != c.checksum { 75 | log.Tracef("cacheObject: %q invalidated, checksum mismatch: %q -> %q", c.object.GetName(), c.checksum, chk) 76 | log.Tracef("cacheObject: API load triggered for %q", c.object.GetName()) 77 | obj, err := c.storage.GetMeta(c.object.GroupVersionKind(), c.object.GetUID()) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | // Only apply the change after a successful GetMeta 83 | c.object = obj 84 | c.checksum = chk 85 | c.apiType = true 86 | } else { 87 | log.Tracef("cacheObject: %q checksum: %q", c.object.GetName(), c.checksum) 88 | } 89 | 90 | if c.apiType { 91 | return c.object, nil 92 | } 93 | 94 | return runtime.PartialObjectFrom(c.object), nil 95 | } 96 | */ 97 | -------------------------------------------------------------------------------- /pkg/util/sync/batcher.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // NewBatchWriter creates a new BatchWriter 11 | func NewBatchWriter(duration time.Duration) *BatchWriter { 12 | return &BatchWriter{ 13 | duration: duration, 14 | flushCh: make(chan struct{}), 15 | syncMap: &sync.Map{}, 16 | } 17 | } 18 | 19 | // BatchWriter is a struct that wraps a concurrent sync.Map 20 | // and dispatches all writes to it at once, a specific 21 | // duration (e.g. 1s) after the last write was performed. This 22 | // allows for 100s of concurrent writes in milliseconds to the 23 | // same map in one sending goroutine; and one receiving goroutine 24 | // which can process the result after all the writes are done. 25 | type BatchWriter struct { 26 | duration time.Duration 27 | timer *time.Timer 28 | flushCh chan struct{} 29 | syncMap *sync.Map 30 | } 31 | 32 | // Load reads the key from the map 33 | func (b *BatchWriter) Load(key interface{}) (value interface{}, ok bool) { 34 | return b.syncMap.Load(key) 35 | } 36 | 37 | // Store writes the value for the specified key to the map 38 | // If no other .Store call is made during the specified duration, 39 | // flushCh is invoked and ProcessBatch unblocks in the other goroutine 40 | func (b *BatchWriter) Store(key, value interface{}) { 41 | // prevent the timer from firing as we're manipulating it now 42 | b.cancelUnfiredTimer() 43 | // store the key and the value as requested 44 | log.Tracef("BatchWriter: Storing key %v and value %q, reset the timer.", key, value) 45 | b.syncMap.Store(key, value) 46 | // set the timer to fire after the duration, unless there's a new .Store call 47 | b.dispatchAfterTimeout() 48 | } 49 | 50 | // Close closes the underlying channel 51 | func (b *BatchWriter) Close() { 52 | log.Trace("BatchWriter: Closing the batch channel") 53 | close(b.flushCh) 54 | } 55 | 56 | // ProcessBatch is effectively a Range over the sync.Map, once a batch write is 57 | // released. This should be used in the receiving goroutine. The internal map is 58 | // reset after this call, so be sure to capture all the contents if needed. This 59 | // function returns false if Close() has been called. 60 | func (b *BatchWriter) ProcessBatch(fn func(key, val interface{}) bool) bool { 61 | if _, ok := <-b.flushCh; !ok { 62 | // channel is closed 63 | return false 64 | } 65 | log.Trace("BatchWriter: Received a flush for the batch. Dispatching it now.") 66 | b.syncMap.Range(fn) 67 | *b.syncMap = sync.Map{} 68 | return true 69 | } 70 | 71 | func (b *BatchWriter) cancelUnfiredTimer() { 72 | // If the timer already exists; stop it 73 | if b.timer != nil { 74 | log.Tracef("BatchWriter: Cancelled timer") 75 | b.timer.Stop() 76 | b.timer = nil 77 | } 78 | } 79 | 80 | func (b *BatchWriter) dispatchAfterTimeout() { 81 | b.timer = time.AfterFunc(b.duration, func() { 82 | log.Tracef("BatchWriter: Dispatching a batch job") 83 | b.flushCh <- struct{}{} 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /hack/ldflags.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GO_PROJECT="github.com/weaveworks/libgitops" 4 | 5 | # Note: This file is heavily inspired by https://github.com/kubernetes/kubernetes/blob/master/hack/lib/version.sh 6 | 7 | get_version_vars() { 8 | GIT_COMMIT=$(git rev-parse "HEAD^{commit}" 2>/dev/null) 9 | if git_status=$(git status --porcelain 2>/dev/null) && [[ -z ${git_status} ]]; then 10 | GIT_TREE_STATE="clean" 11 | else 12 | GIT_TREE_STATE="dirty" 13 | fi 14 | # Use git describe to find the version based on tags. 15 | GIT_VERSION=$(git describe --tags --abbrev=14 "${GIT_COMMIT}^{commit}" 2>/dev/null) 16 | 17 | # This translates the "git describe" to an actual semver.org 18 | # compatible semantic version that looks something like this: 19 | # v1.1.0-alpha.0.6+84c76d1142ea4d 20 | DASHES_IN_VERSION=$(echo "${GIT_VERSION}" | sed "s/[^-]//g") 21 | if [[ "${DASHES_IN_VERSION}" == "---" ]] ; then 22 | # We have distance to subversion (v1.1.0-subversion-1-gCommitHash) 23 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$/.\1\+\2/") 24 | elif [[ "${DASHES_IN_VERSION}" == "--" ]] ; then 25 | # We have distance to base tag (v1.1.0-1-gCommitHash) 26 | GIT_VERSION=$(echo "${GIT_VERSION}" | sed "s/-g\([0-9a-f]\{14\}\)$/+\1/") 27 | fi 28 | if [[ "${GIT_TREE_STATE}" == "dirty" ]]; then 29 | # git describe --dirty only considers changes to existing files, but 30 | # that is problematic since new untracked .go files affect the build, 31 | # so use our idea of "dirty" from git status instead. 32 | GIT_VERSION+="-dirty" 33 | fi 34 | 35 | # Try to match the "git describe" output to a regex to try to extract 36 | # the "major" and "minor" versions and whether this is the exact tagged 37 | # version or whether the tree is between two tagged versions. 38 | if [[ "${GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?([-].*)?([+].*)?$ ]]; then 39 | GIT_MAJOR=${BASH_REMATCH[1]} 40 | GIT_MINOR=${BASH_REMATCH[2]} 41 | if [[ -n "${BASH_REMATCH[4]}" ]]; then 42 | GIT_MINOR+="+" 43 | fi 44 | fi 45 | } 46 | 47 | ldflag() { 48 | local key=${1} 49 | local val=${2} 50 | echo "-X '${GO_PROJECT}/cmd/sample-app/version.${key}=${val}'" 51 | } 52 | 53 | # Prints the value that needs to be passed to the -ldflags parameter of go build 54 | # in order to set the version based on the git tree status. 55 | ldflags() { 56 | get_version_vars 57 | 58 | local buildDate= 59 | [[ -z ${SOURCE_DATE_EPOCH-} ]] || buildDate="--date=@${SOURCE_DATE_EPOCH}" 60 | local -a ldflags=($(ldflag "buildDate" "$(date ${buildDate} -u +'%Y-%m-%dT%H:%M:%SZ')")) 61 | if [[ -n ${GIT_COMMIT-} ]]; then 62 | ldflags+=($(ldflag "gitCommit" "${GIT_COMMIT}")) 63 | ldflags+=($(ldflag "gitTreeState" "${GIT_TREE_STATE}")) 64 | fi 65 | 66 | if [[ -n ${GIT_VERSION-} ]]; then 67 | ldflags+=($(ldflag "gitVersion" "${GIT_VERSION}")) 68 | fi 69 | 70 | if [[ -n ${GIT_MAJOR-} && -n ${GIT_MINOR-} ]]; then 71 | ldflags+=( 72 | $(ldflag "gitMajor" "${GIT_MAJOR}") 73 | $(ldflag "gitMinor" "${GIT_MINOR}") 74 | ) 75 | fi 76 | 77 | # Output only the version with this flag 78 | if [[ $1 == "--version-only" ]]; then 79 | echo "${GIT_VERSION}" 80 | exit 0 81 | fi 82 | 83 | # The -ldflags parameter takes a single string, so join the output. 84 | echo "${ldflags[*]-}" 85 | } 86 | 87 | ldflags $@ -------------------------------------------------------------------------------- /pkg/util/patch/patch.go: -------------------------------------------------------------------------------- 1 | package patch 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | 8 | "github.com/weaveworks/libgitops/pkg/runtime" 9 | "github.com/weaveworks/libgitops/pkg/serializer" 10 | "k8s.io/apimachinery/pkg/runtime/schema" 11 | "k8s.io/apimachinery/pkg/util/strategicpatch" 12 | ) 13 | 14 | type Patcher interface { 15 | Create(new runtime.Object, applyFn func(runtime.Object) error) ([]byte, error) 16 | Apply(original, patch []byte, gvk schema.GroupVersionKind) ([]byte, error) 17 | ApplyOnFile(filePath string, patch []byte, gvk schema.GroupVersionKind) error 18 | } 19 | 20 | func NewPatcher(s serializer.Serializer) Patcher { 21 | return &patcher{serializer: s} 22 | } 23 | 24 | type patcher struct { 25 | serializer serializer.Serializer 26 | } 27 | 28 | // Create is a helper that creates a patch out of the change made in applyFn 29 | func (p *patcher) Create(new runtime.Object, applyFn func(runtime.Object) error) (patchBytes []byte, err error) { 30 | var oldBytes, newBytes bytes.Buffer 31 | encoder := p.serializer.Encoder() 32 | old := new.DeepCopyObject().(runtime.Object) 33 | 34 | if err = encoder.Encode(serializer.NewJSONFrameWriter(&oldBytes), old); err != nil { 35 | return 36 | } 37 | 38 | if err = applyFn(new); err != nil { 39 | return 40 | } 41 | 42 | if err = encoder.Encode(serializer.NewJSONFrameWriter(&newBytes), new); err != nil { 43 | return 44 | } 45 | 46 | emptyObj, err := p.serializer.Scheme().New(old.GetObjectKind().GroupVersionKind()) 47 | if err != nil { 48 | return 49 | } 50 | 51 | patchBytes, err = strategicpatch.CreateTwoWayMergePatch(oldBytes.Bytes(), newBytes.Bytes(), emptyObj) 52 | if err != nil { 53 | return nil, fmt.Errorf("CreateTwoWayMergePatch failed: %v", err) 54 | } 55 | 56 | return patchBytes, nil 57 | } 58 | 59 | func (p *patcher) Apply(original, patch []byte, gvk schema.GroupVersionKind) ([]byte, error) { 60 | emptyObj, err := p.serializer.Scheme().New(gvk) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | b, err := strategicpatch.StrategicMergePatch(original, patch, emptyObj) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | return p.serializerEncode(b) 71 | } 72 | 73 | func (p *patcher) ApplyOnFile(filePath string, patch []byte, gvk schema.GroupVersionKind) error { 74 | oldContent, err := ioutil.ReadFile(filePath) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | newContent, err := p.Apply(oldContent, patch, gvk) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | return ioutil.WriteFile(filePath, newContent, 0644) 85 | } 86 | 87 | // StrategicMergePatch returns an unindented, unorganized JSON byte slice, 88 | // this helper takes that as an input and returns the same JSON re-encoded 89 | // with the serializer so it conforms to a runtime.Object 90 | // TODO: Just use encoding/json.Indent here instead? 91 | func (p *patcher) serializerEncode(input []byte) ([]byte, error) { 92 | obj, err := p.serializer.Decoder().Decode(serializer.NewJSONFrameReader(serializer.FromBytes(input))) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | var result bytes.Buffer 98 | if err := p.serializer.Encoder().Encode(serializer.NewJSONFrameWriter(&result), obj); err != nil { 99 | return nil, err 100 | } 101 | 102 | return result.Bytes(), err 103 | } 104 | -------------------------------------------------------------------------------- /pkg/client/client_dynamic.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package client 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/weaveworks/libgitops/pkg/runtime" 9 | "github.com/weaveworks/libgitops/pkg/storage" 10 | "github.com/weaveworks/libgitops/pkg/storage/filterer" 11 | "k8s.io/apimachinery/pkg/runtime/schema" 12 | ) 13 | 14 | // DynamicClient is an interface for accessing API types generically 15 | type DynamicClient interface { 16 | // New returns a new Object of its kind 17 | New() runtime.Object 18 | // Get returns an Object matching the UID from the storage 19 | Get(runtime.UID) (runtime.Object, error) 20 | // Set saves an Object into the persistent storage 21 | Set(runtime.Object) error 22 | // Patch performs a strategic merge patch on the object with 23 | // the given UID, using the byte-encoded patch given 24 | Patch(runtime.UID, []byte) error 25 | // Find returns an Object based on the given filter, filters can 26 | // match e.g. the Object's Name, UID or a specific property 27 | Find(filter filterer.BaseFilter) (runtime.Object, error) 28 | // FindAll returns multiple Objects based on the given filter, filters can 29 | // match e.g. the Object's Name, UID or a specific property 30 | FindAll(filter filterer.BaseFilter) ([]runtime.Object, error) 31 | // Delete deletes an Object from the storage 32 | Delete(uid runtime.UID) error 33 | // List returns a list of all Objects available 34 | List() ([]runtime.Object, error) 35 | } 36 | 37 | // dynamicClient is a struct implementing the DynamicClient interface 38 | // It uses a shared storage instance passed from the Client together with its own Filterer 39 | type dynamicClient struct { 40 | storage storage.Storage 41 | gvk schema.GroupVersionKind 42 | filterer *filterer.Filterer 43 | } 44 | 45 | // NewDynamicClient builds the dynamicClient struct using the storage implementation and a new Filterer 46 | func NewDynamicClient(s storage.Storage, gvk schema.GroupVersionKind) DynamicClient { 47 | return &dynamicClient{ 48 | storage: s, 49 | gvk: gvk, 50 | filterer: filterer.NewFilterer(s), 51 | } 52 | } 53 | 54 | // New returns a new Object of its kind 55 | func (c *dynamicClient) New() runtime.Object { 56 | obj, err := c.storage.New(c.gvk) 57 | if err != nil { 58 | panic(fmt.Sprintf("Client.New must not return an error: %v", err)) 59 | } 60 | return obj 61 | } 62 | 63 | // Get returns an Object based the given UID 64 | func (c *dynamicClient) Get(uid runtime.UID) (runtime.Object, error) { 65 | return c.storage.Get(c.gvk, uid) 66 | } 67 | 68 | // Set saves an Object into the persistent storage 69 | func (c *dynamicClient) Set(resource runtime.Object) error { 70 | return c.storage.Set(c.gvk, resource) 71 | } 72 | 73 | // Patch performs a strategic merge patch on the object with 74 | // the given UID, using the byte-encoded patch given 75 | func (c *dynamicClient) Patch(uid runtime.UID, patch []byte) error { 76 | return c.storage.Patch(c.gvk, uid, patch) 77 | } 78 | 79 | // Find returns an Object based on a given Filter 80 | func (c *dynamicClient) Find(filter filterer.BaseFilter) (runtime.Object, error) { 81 | return c.filterer.Find(c.gvk, filter) 82 | } 83 | 84 | // FindAll returns multiple Objects based on a given Filter 85 | func (c *dynamicClient) FindAll(filter filterer.BaseFilter) ([]runtime.Object, error) { 86 | return c.filterer.FindAll(c.gvk, filter) 87 | } 88 | 89 | // Delete deletes the Object from the storage 90 | func (c *dynamicClient) Delete(uid runtime.UID) error { 91 | return c.storage.Delete(c.gvk, uid) 92 | } 93 | 94 | // List returns a list of all Objects available 95 | func (c *dynamicClient) List() ([]runtime.Object, error) { 96 | return c.storage.List(c.gvk) 97 | } 98 | -------------------------------------------------------------------------------- /pkg/storage/transaction/pullrequest/github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/fluxcd/go-git-providers/github" 9 | "github.com/fluxcd/go-git-providers/gitprovider" 10 | gogithub "github.com/google/go-github/v32/github" 11 | "github.com/weaveworks/libgitops/pkg/storage/transaction" 12 | ) 13 | 14 | // TODO: This package should really only depend on go-git-providers' abstraction interface 15 | 16 | var ErrProviderNotSupported = errors.New("only the Github go-git-providers provider is supported at the moment") 17 | 18 | // NewGitHubPRProvider returns a new transaction.PullRequestProvider from a gitprovider.Client. 19 | func NewGitHubPRProvider(c gitprovider.Client) (transaction.PullRequestProvider, error) { 20 | // Make sure a Github client was passed 21 | if c.ProviderID() != github.ProviderID { 22 | return nil, ErrProviderNotSupported 23 | } 24 | return &prCreator{c}, nil 25 | } 26 | 27 | type prCreator struct { 28 | c gitprovider.Client 29 | } 30 | 31 | func (c *prCreator) CreatePullRequest(ctx context.Context, spec transaction.PullRequestSpec) error { 32 | // First, validate the input 33 | if err := spec.Validate(); err != nil { 34 | return fmt.Errorf("given PullRequestSpec wasn't valid") 35 | } 36 | 37 | // Use the "raw" go-github client to do this 38 | ghClient := c.c.Raw().(*gogithub.Client) 39 | 40 | // Helper variables 41 | owner := spec.GetRepositoryRef().GetIdentity() 42 | repo := spec.GetRepositoryRef().GetRepository() 43 | var body *string 44 | if spec.GetDescription() != "" { 45 | body = gogithub.String(spec.GetDescription()) 46 | } 47 | 48 | // Create the Pull Request 49 | pr, _, err := ghClient.PullRequests.Create(ctx, owner, repo, &gogithub.NewPullRequest{ 50 | Head: gogithub.String(spec.GetMergeBranch()), 51 | Base: gogithub.String(spec.GetMainBranch()), 52 | Title: gogithub.String(spec.GetTitle()), 53 | Body: body, 54 | }) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | // If spec.GetMilestone() is set, fetch the ID of the milestone 60 | // Only set milestoneID to non-nil if specified 61 | var milestoneID *int 62 | if len(spec.GetMilestone()) != 0 { 63 | milestoneID, err = getMilestoneID(ctx, ghClient, owner, repo, spec.GetMilestone()) 64 | if err != nil { 65 | return err 66 | } 67 | } 68 | 69 | // Only set assignees to non-nil if specified 70 | var assignees *[]string 71 | if a := spec.GetAssignees(); len(a) != 0 { 72 | assignees = &a 73 | } 74 | 75 | // Only set labels to non-nil if specified 76 | var labels *[]string 77 | if l := spec.GetLabels(); len(l) != 0 { 78 | labels = &l 79 | } 80 | 81 | // Only PATCH the PR if any of the fields were set 82 | if milestoneID != nil || assignees != nil || labels != nil { 83 | _, _, err := ghClient.Issues.Edit(ctx, owner, repo, pr.GetNumber(), &gogithub.IssueRequest{ 84 | Milestone: milestoneID, 85 | Assignees: assignees, 86 | Labels: labels, 87 | }) 88 | if err != nil { 89 | return err 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | 96 | func getMilestoneID(ctx context.Context, c *gogithub.Client, owner, repo, milestoneName string) (*int, error) { 97 | // List all milestones in the repo 98 | // TODO: This could/should use pagination 99 | milestones, _, err := c.Issues.ListMilestones(ctx, owner, repo, &gogithub.MilestoneListOptions{ 100 | State: "all", 101 | }) 102 | if err != nil { 103 | return nil, err 104 | } 105 | // Loop through all milestones, search for one with the right name 106 | for _, milestone := range milestones { 107 | // Only consider a milestone with the right name 108 | if milestone.GetTitle() != milestoneName { 109 | continue 110 | } 111 | // Validate nil to avoid panics 112 | if milestone.Number == nil { 113 | return nil, fmt.Errorf("didn't expect milestone Number to be nil: %v", milestone) 114 | } 115 | // Return the Milestone number 116 | return milestone.Number, nil 117 | } 118 | return nil, fmt.Errorf("couldn't find milestone with name: %s", milestoneName) 119 | } 120 | -------------------------------------------------------------------------------- /pkg/serializer/frame_writer.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | const ( 8 | yamlSeparator = "---\n" 9 | ) 10 | 11 | // Writer in this package is an alias for io.Writer. It helps in Godoc to locate 12 | // helpers in this package which returns writers (i.e. ToBytes) 13 | type Writer io.Writer 14 | 15 | // FrameWriter is a ContentType-specific io.Writer that writes given frames in an applicable way 16 | // to an underlying io.Writer stream 17 | type FrameWriter interface { 18 | ContentTyped 19 | Writer 20 | } 21 | 22 | // NewFrameWriter returns a new FrameWriter for the given Writer and ContentType 23 | func NewFrameWriter(contentType ContentType, w Writer) FrameWriter { 24 | switch contentType { 25 | case ContentTypeYAML: 26 | // Use our own implementation of the underlying YAML FrameWriter 27 | return &frameWriter{newYAMLWriter(w), contentType} 28 | case ContentTypeJSON: 29 | // Comment from k8s.io/apimachinery/pkg/runtime/serializer/json.Framer.NewFrameWriter: 30 | // "we can write JSON objects directly to the writer, because they are self-framing" 31 | // Hence, we directly use w without any modifications. 32 | return &frameWriter{w, contentType} 33 | default: 34 | return &errFrameWriter{ErrUnsupportedContentType, contentType} 35 | } 36 | } 37 | 38 | // NewYAMLFrameWriter returns a FrameWriter that writes YAML frames separated by "---\n" 39 | // 40 | // This call is the same as NewFrameWriter(ContentTypeYAML, w) 41 | func NewYAMLFrameWriter(w Writer) FrameWriter { 42 | return NewFrameWriter(ContentTypeYAML, w) 43 | } 44 | 45 | // NewJSONFrameWriter returns a FrameWriter that writes JSON frames without separation 46 | // (i.e. "{ ... }{ ... }{ ... }" on the wire) 47 | // 48 | // This call is the same as NewFrameWriter(ContentTypeYAML, w) 49 | func NewJSONFrameWriter(w Writer) FrameWriter { 50 | return NewFrameWriter(ContentTypeJSON, w) 51 | } 52 | 53 | // frameWriter is an implementation of the FrameWriter interface 54 | type frameWriter struct { 55 | Writer 56 | 57 | contentType ContentType 58 | 59 | // TODO: Maybe add mutexes for thread-safety (so no two goroutines write at the same time) 60 | } 61 | 62 | // ContentType returns the content type for the given FrameWriter 63 | func (wf *frameWriter) ContentType() ContentType { 64 | return wf.contentType 65 | } 66 | 67 | // newYAMLWriter returns a new yamlWriter implementation 68 | func newYAMLWriter(w Writer) *yamlWriter { 69 | return &yamlWriter{ 70 | w: w, 71 | hasWritten: false, 72 | } 73 | } 74 | 75 | // yamlWriter writes yamlSeparator between documents 76 | type yamlWriter struct { 77 | w io.Writer 78 | hasWritten bool 79 | } 80 | 81 | // Write implements io.Writer 82 | func (w *yamlWriter) Write(p []byte) (n int, err error) { 83 | // If we've already written some documents, add the separator in between 84 | if w.hasWritten { 85 | _, err = w.w.Write([]byte(yamlSeparator)) 86 | if err != nil { 87 | return 88 | } 89 | } 90 | 91 | // Write the given bytes to the underlying writer 92 | n, err = w.w.Write(p) 93 | if err != nil { 94 | return 95 | } 96 | 97 | // Mark that we've now written once and should write the separator in between 98 | w.hasWritten = true 99 | return 100 | } 101 | 102 | // ToBytes returns a Writer which can be passed to NewFrameWriter. The Writer writes directly 103 | // to an underlying byte array. The byte array must be of enough length in order to write. 104 | func ToBytes(p []byte) Writer { 105 | return &byteWriter{p, 0} 106 | } 107 | 108 | type byteWriter struct { 109 | to []byte 110 | // the next index to write to 111 | index int 112 | } 113 | 114 | func (w *byteWriter) Write(from []byte) (n int, err error) { 115 | // Check if we have space in to, in order to write bytes there 116 | if w.index+len(from) > len(w.to) { 117 | err = io.ErrShortBuffer 118 | return 119 | } 120 | // Copy over the bytes one by one 121 | for i := range from { 122 | w.to[w.index+i] = from[i] 123 | } 124 | // Increase the index for the next Write call's target position 125 | w.index += len(from) 126 | n += len(from) 127 | return 128 | } 129 | -------------------------------------------------------------------------------- /pkg/storage/cache/index.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | /* 4 | 5 | TODO: Revisit if we need this file/package in the future. 6 | 7 | import ( 8 | log "github.com/sirupsen/logrus" 9 | "github.com/weaveworks/libgitops/pkg/runtime" 10 | "github.com/weaveworks/libgitops/pkg/storage" 11 | "k8s.io/apimachinery/pkg/runtime/schema" 12 | ) 13 | 14 | type index struct { 15 | storage storage.Storage 16 | objects map[schema.GroupVersionKind]map[runtime.UID]*cacheObject 17 | } 18 | 19 | func newIndex(storage storage.Storage) *index { 20 | return &index{ 21 | storage: storage, 22 | objects: make(map[schema.GroupVersionKind]map[runtime.UID]*cacheObject), 23 | } 24 | } 25 | 26 | func (i *index) loadByID(gvk schema.GroupVersionKind, uid runtime.UID) (runtime.Object, error) { 27 | if uids, ok := i.objects[gvk]; ok { 28 | if obj, ok := uids[uid]; ok { 29 | log.Tracef("index: cache hit for %s with UID %q", gvk.Kind, uid) 30 | return obj.loadFull() 31 | } 32 | } 33 | 34 | log.Tracef("index: cache miss for %s with UID %q", gvk.Kind, uid) 35 | return nil, nil 36 | } 37 | 38 | func (i *index) loadAll() ([]runtime.Object, error) { 39 | var size uint64 40 | 41 | for gvk := range i.objects { 42 | size += i.count(gvk) 43 | } 44 | 45 | all := make([]runtime.Object, 0, size) 46 | 47 | for gvk := range i.objects { 48 | if objects, err := i.list(gvk); err == nil { 49 | all = append(all, objects...) 50 | } else { 51 | return nil, err 52 | } 53 | } 54 | 55 | return all, nil 56 | } 57 | 58 | func store(i *index, obj runtime.Object, apiType bool) error { 59 | // If store is called for an invalid Object lacking an UID, 60 | // panic and print the stack trace. This should never happen. 61 | if obj.GetUID() == "" { 62 | panic("Attempt to cache invalid Object: missing UID") 63 | } 64 | 65 | co, err := newCacheObject(i.storage, obj, apiType) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | gvk := co.object.GetObjectKind().GroupVersionKind() 71 | 72 | if _, ok := i.objects[gvk]; !ok { 73 | i.objects[gvk] = make(map[runtime.UID]*cacheObject) 74 | } 75 | 76 | log.Tracef("index: storing %s object with UID %q, meta: %t", gvk.Kind, obj.GetName(), apiType) 77 | i.objects[gvk][co.object.GetUID()] = co 78 | 79 | return nil 80 | } 81 | 82 | func (i *index) store(obj runtime.Object) error { 83 | return store(i, obj, false) 84 | } 85 | 86 | func (i *index) storeAll(objs []runtime.Object) (err error) { 87 | for _, obj := range objs { 88 | if err = i.store(obj); err != nil { 89 | break 90 | } 91 | } 92 | 93 | return 94 | } 95 | 96 | func (i *index) storeMeta(obj runtime.Object) error { 97 | return store(i, obj, true) 98 | } 99 | 100 | func (i *index) storeAllMeta(objs []runtime.Object) (err error) { 101 | for _, obj := range objs { 102 | if uids, ok := i.objects[obj.GetObjectKind().GroupVersionKind()]; ok { 103 | if _, ok := uids[obj.GetUID()]; ok { 104 | continue 105 | } 106 | } 107 | 108 | if err = i.storeMeta(obj); err != nil { 109 | break 110 | } 111 | } 112 | 113 | return 114 | } 115 | 116 | func (i *index) delete(gvk schema.GroupVersionKind, uid runtime.UID) { 117 | if uids, ok := i.objects[gvk]; ok { 118 | delete(uids, uid) 119 | } 120 | } 121 | 122 | func (i *index) count(gvk schema.GroupVersionKind) (count uint64) { 123 | count = uint64(len(i.objects[gvk])) 124 | log.Tracef("index: counted %d %s object(s)", count, gvk.Kind) 125 | return 126 | } 127 | 128 | func list(i *index, gvk schema.GroupVersionKind, apiTypes bool) ([]runtime.Object, error) { 129 | uids := i.objects[gvk] 130 | list := make([]runtime.Object, 0, len(uids)) 131 | 132 | log.Tracef("index: listing %s objects, meta: %t", gvk, apiTypes) 133 | for _, obj := range uids { 134 | loadFunc := obj.loadFull 135 | if apiTypes { 136 | loadFunc = obj.loadAPI 137 | } 138 | 139 | if result, err := loadFunc(); err != nil { 140 | return nil, err 141 | } else { 142 | list = append(list, result) 143 | } 144 | } 145 | 146 | return list, nil 147 | } 148 | 149 | func (i *index) list(gvk schema.GroupVersionKind) ([]runtime.Object, error) { 150 | return list(i, gvk, false) 151 | } 152 | 153 | func (i *index) listMeta(gvk schema.GroupVersionKind) ([]runtime.Object, error) { 154 | return list(i, gvk, true) 155 | } 156 | */ 157 | -------------------------------------------------------------------------------- /pkg/serializer/comments/comments.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Kubernetes Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // This package provides a means to copy over comments between 5 | // two kyaml/yaml.RNode trees. This code is derived from 6 | // the sigs.k8s.io/kustomize/kyaml/comments package, at revision 7 | // 600d4f2c0bf174abd76d03e49939ee0c34b2a019. 8 | // 9 | // It has been slightly modified and adapted to not lose any 10 | // comment from the old tree, although the node the comment is 11 | // attached to doesn't exist in the new tree. To solve this, 12 | // this package moves any such comments to the beginning of the 13 | // file. 14 | // This file is a temporary means as long as we're waiting for 15 | // these code changes to get upstreamed to its origin, the kustomize repo. 16 | // https://pkg.go.dev/sigs.k8s.io/kustomize/kyaml/comments?tab=doc#CopyComments 17 | 18 | package comments 19 | 20 | import ( 21 | "sigs.k8s.io/kustomize/kyaml/openapi" 22 | "sigs.k8s.io/kustomize/kyaml/yaml" 23 | "sigs.k8s.io/kustomize/kyaml/yaml/walk" 24 | ) 25 | 26 | // CopyComments recursively copies the comments on fields in from to fields in to 27 | func CopyComments(from, to *yaml.RNode, moveCommentsTop bool) error { 28 | // create the copier struct for the specified mode 29 | c := &copier{moveCommentsTop, nil, make(map[int]trackedKey)} 30 | 31 | // copy over comments for the root tree(s) 32 | c.copyFieldComments(from, to) 33 | 34 | // walk the fields copying comments 35 | _, err := walk.Walker{ 36 | Sources: []*yaml.RNode{from, to}, 37 | Visitor: c, 38 | VisitKeysAsScalars: true}.Walk() 39 | 40 | // restore lost comments to the top of the document, if applicable 41 | if moveCommentsTop { 42 | c.restoreLostComments(to) 43 | } 44 | 45 | return err 46 | } 47 | 48 | // copier implements walk.Visitor, and copies comments to fields shared between 2 instances 49 | // of a resource 50 | type copier struct { 51 | // moveCommentsTop specifies whether to recover lost comments or not 52 | moveCommentsTop bool 53 | // if moveCommentsTop is true, this slice will be populated with lost comment entries while iterating 54 | lostComments []lostComment 55 | // if moveCommentsTop is true, this map will be populated with tracked YAML keys for lines while iterating 56 | trackedKeys map[int]trackedKey 57 | } 58 | 59 | func (c *copier) VisitMap(s walk.Sources, _ *openapi.ResourceSchema) (*yaml.RNode, error) { 60 | c.copyFieldComments(s.Dest(), s.Origin()) 61 | return s.Dest(), nil 62 | } 63 | 64 | func (c *copier) VisitScalar(s walk.Sources, _ *openapi.ResourceSchema) (*yaml.RNode, error) { 65 | to := s.Origin() 66 | // TODO: File a bug with upstream yaml to handle comments for FoldedStyle scalar nodes 67 | // Hack: convert FoldedStyle scalar node to DoubleQuotedStyle as the line comments are 68 | // being serialized without space 69 | // https://github.com/GoogleContainerTools/kpt/issues/766 70 | if to != nil && to.Document().Style == yaml.FoldedStyle { 71 | to.Document().Style = yaml.DoubleQuotedStyle 72 | } 73 | 74 | c.copyFieldComments(s.Dest(), to) 75 | return s.Dest(), nil 76 | } 77 | 78 | func (c *copier) VisitList(s walk.Sources, _ *openapi.ResourceSchema, _ walk.ListKind) (*yaml.RNode, error) { 79 | c.copyFieldComments(s.Dest(), s.Origin()) 80 | destItems := s.Dest().Content() 81 | originItems := s.Origin().Content() 82 | 83 | for i := 0; i < len(destItems) && i < len(originItems); i++ { 84 | dest := destItems[i] 85 | origin := originItems[i] 86 | 87 | if dest.Value == origin.Value { 88 | c.copyFieldComments(yaml.NewRNode(dest), yaml.NewRNode(origin)) 89 | } 90 | } 91 | 92 | return s.Dest(), nil 93 | } 94 | 95 | // copyFieldComments copies the comment from one field to another 96 | func (c *copier) copyFieldComments(from, to *yaml.RNode) { 97 | // If either from or to doesn't exist, return quickly 98 | if from == nil || to == nil { 99 | 100 | // If we asked for moving lost comments (i.e. if from is non-nil and to is nil), 101 | // do it through the moveLostCommentToTop function 102 | if c.moveCommentsTop && from != nil && to == nil { 103 | c.rememberLostComments(from) 104 | } 105 | return 106 | } 107 | 108 | if to.Document().LineComment == "" { 109 | to.Document().LineComment = from.Document().LineComment 110 | } 111 | if to.Document().HeadComment == "" { 112 | to.Document().HeadComment = from.Document().HeadComment 113 | } 114 | if to.Document().FootComment == "" { 115 | to.Document().FootComment = from.Document().FootComment 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Code generated by deepcopy-gen. DO NOT EDIT. 4 | 5 | package sample 6 | 7 | import ( 8 | runtime "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 12 | func (in *Car) DeepCopyInto(out *Car) { 13 | *out = *in 14 | out.TypeMeta = in.TypeMeta 15 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 16 | out.Spec = in.Spec 17 | out.Status = in.Status 18 | return 19 | } 20 | 21 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Car. 22 | func (in *Car) DeepCopy() *Car { 23 | if in == nil { 24 | return nil 25 | } 26 | out := new(Car) 27 | in.DeepCopyInto(out) 28 | return out 29 | } 30 | 31 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 32 | func (in *Car) DeepCopyObject() runtime.Object { 33 | if c := in.DeepCopy(); c != nil { 34 | return c 35 | } 36 | return nil 37 | } 38 | 39 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 40 | func (in *CarSpec) DeepCopyInto(out *CarSpec) { 41 | *out = *in 42 | return 43 | } 44 | 45 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CarSpec. 46 | func (in *CarSpec) DeepCopy() *CarSpec { 47 | if in == nil { 48 | return nil 49 | } 50 | out := new(CarSpec) 51 | in.DeepCopyInto(out) 52 | return out 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *CarStatus) DeepCopyInto(out *CarStatus) { 57 | *out = *in 58 | out.VehicleStatus = in.VehicleStatus 59 | return 60 | } 61 | 62 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CarStatus. 63 | func (in *CarStatus) DeepCopy() *CarStatus { 64 | if in == nil { 65 | return nil 66 | } 67 | out := new(CarStatus) 68 | in.DeepCopyInto(out) 69 | return out 70 | } 71 | 72 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 73 | func (in *Motorcycle) DeepCopyInto(out *Motorcycle) { 74 | *out = *in 75 | out.TypeMeta = in.TypeMeta 76 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 77 | out.Spec = in.Spec 78 | out.Status = in.Status 79 | return 80 | } 81 | 82 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Motorcycle. 83 | func (in *Motorcycle) DeepCopy() *Motorcycle { 84 | if in == nil { 85 | return nil 86 | } 87 | out := new(Motorcycle) 88 | in.DeepCopyInto(out) 89 | return out 90 | } 91 | 92 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 93 | func (in *Motorcycle) DeepCopyObject() runtime.Object { 94 | if c := in.DeepCopy(); c != nil { 95 | return c 96 | } 97 | return nil 98 | } 99 | 100 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 101 | func (in *MotorcycleSpec) DeepCopyInto(out *MotorcycleSpec) { 102 | *out = *in 103 | return 104 | } 105 | 106 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MotorcycleSpec. 107 | func (in *MotorcycleSpec) DeepCopy() *MotorcycleSpec { 108 | if in == nil { 109 | return nil 110 | } 111 | out := new(MotorcycleSpec) 112 | in.DeepCopyInto(out) 113 | return out 114 | } 115 | 116 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 117 | func (in *MotorcycleStatus) DeepCopyInto(out *MotorcycleStatus) { 118 | *out = *in 119 | out.VehicleStatus = in.VehicleStatus 120 | return 121 | } 122 | 123 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MotorcycleStatus. 124 | func (in *MotorcycleStatus) DeepCopy() *MotorcycleStatus { 125 | if in == nil { 126 | return nil 127 | } 128 | out := new(MotorcycleStatus) 129 | in.DeepCopyInto(out) 130 | return out 131 | } 132 | 133 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 134 | func (in *VehicleStatus) DeepCopyInto(out *VehicleStatus) { 135 | *out = *in 136 | return 137 | } 138 | 139 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VehicleStatus. 140 | func (in *VehicleStatus) DeepCopy() *VehicleStatus { 141 | if in == nil { 142 | return nil 143 | } 144 | out := new(VehicleStatus) 145 | in.DeepCopyInto(out) 146 | return out 147 | } 148 | -------------------------------------------------------------------------------- /cmd/sample-app/apis/sample/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | // Code generated by deepcopy-gen. DO NOT EDIT. 4 | 5 | package v1alpha1 6 | 7 | import ( 8 | runtime "k8s.io/apimachinery/pkg/runtime" 9 | ) 10 | 11 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 12 | func (in *Car) DeepCopyInto(out *Car) { 13 | *out = *in 14 | out.TypeMeta = in.TypeMeta 15 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 16 | out.Spec = in.Spec 17 | out.Status = in.Status 18 | return 19 | } 20 | 21 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Car. 22 | func (in *Car) DeepCopy() *Car { 23 | if in == nil { 24 | return nil 25 | } 26 | out := new(Car) 27 | in.DeepCopyInto(out) 28 | return out 29 | } 30 | 31 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 32 | func (in *Car) DeepCopyObject() runtime.Object { 33 | if c := in.DeepCopy(); c != nil { 34 | return c 35 | } 36 | return nil 37 | } 38 | 39 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 40 | func (in *CarSpec) DeepCopyInto(out *CarSpec) { 41 | *out = *in 42 | return 43 | } 44 | 45 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CarSpec. 46 | func (in *CarSpec) DeepCopy() *CarSpec { 47 | if in == nil { 48 | return nil 49 | } 50 | out := new(CarSpec) 51 | in.DeepCopyInto(out) 52 | return out 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *CarStatus) DeepCopyInto(out *CarStatus) { 57 | *out = *in 58 | out.VehicleStatus = in.VehicleStatus 59 | return 60 | } 61 | 62 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CarStatus. 63 | func (in *CarStatus) DeepCopy() *CarStatus { 64 | if in == nil { 65 | return nil 66 | } 67 | out := new(CarStatus) 68 | in.DeepCopyInto(out) 69 | return out 70 | } 71 | 72 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 73 | func (in *Motorcycle) DeepCopyInto(out *Motorcycle) { 74 | *out = *in 75 | out.TypeMeta = in.TypeMeta 76 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 77 | out.Spec = in.Spec 78 | out.Status = in.Status 79 | return 80 | } 81 | 82 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Motorcycle. 83 | func (in *Motorcycle) DeepCopy() *Motorcycle { 84 | if in == nil { 85 | return nil 86 | } 87 | out := new(Motorcycle) 88 | in.DeepCopyInto(out) 89 | return out 90 | } 91 | 92 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 93 | func (in *Motorcycle) DeepCopyObject() runtime.Object { 94 | if c := in.DeepCopy(); c != nil { 95 | return c 96 | } 97 | return nil 98 | } 99 | 100 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 101 | func (in *MotorcycleSpec) DeepCopyInto(out *MotorcycleSpec) { 102 | *out = *in 103 | return 104 | } 105 | 106 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MotorcycleSpec. 107 | func (in *MotorcycleSpec) DeepCopy() *MotorcycleSpec { 108 | if in == nil { 109 | return nil 110 | } 111 | out := new(MotorcycleSpec) 112 | in.DeepCopyInto(out) 113 | return out 114 | } 115 | 116 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 117 | func (in *MotorcycleStatus) DeepCopyInto(out *MotorcycleStatus) { 118 | *out = *in 119 | out.VehicleStatus = in.VehicleStatus 120 | return 121 | } 122 | 123 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MotorcycleStatus. 124 | func (in *MotorcycleStatus) DeepCopy() *MotorcycleStatus { 125 | if in == nil { 126 | return nil 127 | } 128 | out := new(MotorcycleStatus) 129 | in.DeepCopyInto(out) 130 | return out 131 | } 132 | 133 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 134 | func (in *VehicleStatus) DeepCopyInto(out *VehicleStatus) { 135 | *out = *in 136 | return 137 | } 138 | 139 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VehicleStatus. 140 | func (in *VehicleStatus) DeepCopy() *VehicleStatus { 141 | if in == nil { 142 | return nil 143 | } 144 | out := new(VehicleStatus) 145 | in.DeepCopyInto(out) 146 | return out 147 | } 148 | -------------------------------------------------------------------------------- /pkg/serializer/comments/lost.go: -------------------------------------------------------------------------------- 1 | package comments 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "sigs.k8s.io/kustomize/kyaml/yaml" 8 | ) 9 | 10 | // lostComment specifies a mapping between a fieldName (in the old structure), which doesn't exist in the 11 | // new tree, and its related comment. It optionally specifies the line number of the comment, a positive 12 | // line number is used to distinguish inline comments, which require special handling to resolve the 13 | // correct field name, since they are attached to the value and not the key of a YAML key-value pair. 14 | type lostComment struct { 15 | fieldName string 16 | comment string 17 | line int 18 | } 19 | 20 | // Since the YAML walker needs to visit all keys as scalar nodes, we have no way of distinguishing keys from 21 | // values when trying to resolve the field names for inline comments. By tracking the leftmost key (lowest 22 | // column value, be it a key or value) for each row, we can figure out the actual key for inline comments 23 | // and not accidentally use a value as the field name, since keys are guaranteed to come before values. 24 | type trackedKey struct { 25 | name string 26 | column int 27 | } 28 | 29 | // trackKey compares the column position of the given node to the stored best (lowest) column position for the 30 | // node's line and replaces the best if the given node is more likely to be a key (has a smaller column value). 31 | func (c *copier) trackKey(node *yaml.Node) { 32 | // If the given key doesn't have a smaller column value, return. 33 | if key, ok := c.trackedKeys[node.Line]; ok { 34 | if key.column < node.Column { 35 | return 36 | } 37 | } 38 | 39 | // Store the new best tracked key for the line. 40 | c.trackedKeys[node.Line] = trackedKey{ 41 | name: node.Value, 42 | column: node.Column, 43 | } 44 | } 45 | 46 | // parseComments parses the line, head and foot comments of the given node in this 47 | // order and cleans them up (removes the potential "#" prefix and trims whitespace). 48 | func parseComments(node *yaml.Node) (comments []string) { 49 | for _, comment := range []string{node.LineComment, node.HeadComment, node.FootComment} { 50 | comments = append(comments, strings.TrimSpace(strings.TrimPrefix(comment, "#"))) 51 | } 52 | 53 | return 54 | } 55 | 56 | // rememberLostComments goes through the comments attached to the 'from' node and adds 57 | // them to the internal lostComments slice for usage after the tree walk. It also 58 | // stores the line numbers for inline comments for resolving the correct field names. 59 | func (c *copier) rememberLostComments(from *yaml.RNode) { 60 | // Track the given node as a potential key for inline comments. 61 | c.trackKey(from.Document()) 62 | 63 | // Get the field name, for head/foot comments this is the correct key, 64 | // but for inline comments this resolves to the value of the field instead. 65 | fieldName := from.Document().Value 66 | comments := parseComments(from.Document()) 67 | line := -1 // Don't store the line number of the comment by default, this is reserved for inline comments. 68 | 69 | for i, comment := range comments { 70 | // If the line number is set (positive), an inline comment 71 | // has been registered for this node and we can stop parsing. 72 | if line >= 0 { 73 | break 74 | } 75 | 76 | // Do not store blank comment entries (nonexistent comments). 77 | if len(comment) == 0 { 78 | continue 79 | } 80 | 81 | if i == 0 { 82 | // If this node has an inline comment, store its line 83 | // number for resolving the correct field name later. 84 | line = from.Document().Line 85 | } 86 | 87 | // Append the lost comment to the slice of copier. 88 | c.lostComments = append(c.lostComments, lostComment{ 89 | fieldName: fieldName, 90 | comment: comment, 91 | line: line, 92 | }) 93 | } 94 | } 95 | 96 | // restoreLostComments writes the cached lost comments to the top of the to YAML tree. 97 | // If it encounters inline comments, it will check the cached tracked keys for the 98 | // best key for the line on which the comment resided. If no key is found for some 99 | // reason, it will use the stored field name (the field value) as the key. 100 | func (c *copier) restoreLostComments(to *yaml.RNode) { 101 | for i, lc := range c.lostComments { 102 | if i == 0 { 103 | to.Document().HeadComment += "\nComments lost during file manipulation:" 104 | } 105 | 106 | fieldName := lc.fieldName 107 | if lc.line >= 0 { 108 | // This is an inline comment, resolve the field name from the tracked keys. 109 | if key, ok := c.trackedKeys[lc.line]; ok { 110 | fieldName = key.name 111 | } 112 | } 113 | 114 | to.Document().HeadComment += fmt.Sprintf("\n# Field %q: %q", fieldName, lc.comment) 115 | } 116 | 117 | to.Document().HeadComment = strings.TrimPrefix(to.Document().HeadComment, "\n") 118 | } 119 | -------------------------------------------------------------------------------- /pkg/storage/transaction/pullrequest.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/fluxcd/go-git-providers/gitprovider" 7 | "github.com/fluxcd/go-git-providers/validation" 8 | ) 9 | 10 | // PullRequestResult can be returned from a TransactionFunc instead of a CommitResult, if 11 | // a PullRequest is desired to be created by the PullRequestProvider. 12 | type PullRequestResult interface { 13 | // PullRequestResult is a superset of CommitResult 14 | CommitResult 15 | 16 | // GetLabels specifies what labels should be applied on the PR. 17 | // +optional 18 | GetLabels() []string 19 | // GetAssignees specifies what user login names should be assigned to this PR. 20 | // Note: Only users with "pull" access or more can be assigned. 21 | // +optional 22 | GetAssignees() []string 23 | // GetMilestone specifies what milestone this should be attached to. 24 | // +optional 25 | GetMilestone() string 26 | } 27 | 28 | // GenericPullRequestResult implements PullRequestResult. 29 | var _ PullRequestResult = &GenericPullRequestResult{} 30 | 31 | // GenericPullRequestResult implements PullRequestResult. 32 | type GenericPullRequestResult struct { 33 | // GenericPullRequestResult is a superset of a CommitResult. 34 | CommitResult 35 | 36 | // Labels specifies what labels should be applied on the PR. 37 | // +optional 38 | Labels []string 39 | // Assignees specifies what user login names should be assigned to this PR. 40 | // Note: Only users with "pull" access or more can be assigned. 41 | // +optional 42 | Assignees []string 43 | // Milestone specifies what milestone this should be attached to. 44 | // +optional 45 | Milestone string 46 | } 47 | 48 | func (r *GenericPullRequestResult) GetLabels() []string { 49 | return r.Labels 50 | } 51 | func (r *GenericPullRequestResult) GetAssignees() []string { 52 | return r.Assignees 53 | } 54 | func (r *GenericPullRequestResult) GetMilestone() string { 55 | return r.Milestone 56 | } 57 | func (r *GenericPullRequestResult) Validate() error { 58 | v := validation.New("GenericPullRequestResult") 59 | // Just validate the "inner" object 60 | v.Append(r.CommitResult.Validate(), r.CommitResult, "CommitResult") 61 | return v.Error() 62 | } 63 | 64 | // PullRequestSpec is the messaging interface between the TransactionStorage, and the 65 | // PullRequestProvider. The PullRequestSpec contains all the needed information for creating 66 | // a Pull Request successfully. 67 | type PullRequestSpec interface { 68 | // PullRequestSpec is a superset of PullRequestResult. 69 | PullRequestResult 70 | 71 | // GetMainBranch returns the main branch of the repository. 72 | // +required 73 | GetMainBranch() string 74 | // GetMergeBranch returns the branch that is pending to be merged into main with this PR. 75 | // +required 76 | GetMergeBranch() string 77 | // GetMergeBranch returns the branch that is pending to be merged into main with this PR. 78 | // +required 79 | GetRepositoryRef() gitprovider.RepositoryRef 80 | } 81 | 82 | // GenericPullRequestSpec implements PullRequestSpec. 83 | type GenericPullRequestSpec struct { 84 | // GenericPullRequestSpec is a superset of PullRequestResult. 85 | PullRequestResult 86 | 87 | // MainBranch returns the main branch of the repository. 88 | // +required 89 | MainBranch string 90 | // MergeBranch returns the branch that is pending to be merged into main with this PR. 91 | // +required 92 | MergeBranch string 93 | // RepositoryRef returns the branch that is pending to be merged into main with this PR. 94 | // +required 95 | RepositoryRef gitprovider.RepositoryRef 96 | } 97 | 98 | func (r *GenericPullRequestSpec) GetMainBranch() string { 99 | return r.MainBranch 100 | } 101 | func (r *GenericPullRequestSpec) GetMergeBranch() string { 102 | return r.MergeBranch 103 | } 104 | func (r *GenericPullRequestSpec) GetRepositoryRef() gitprovider.RepositoryRef { 105 | return r.RepositoryRef 106 | } 107 | func (r *GenericPullRequestSpec) Validate() error { 108 | v := validation.New("GenericPullRequestSpec") 109 | // Just validate the "inner" object 110 | v.Append(r.PullRequestResult.Validate(), r.PullRequestResult, "PullRequestResult") 111 | 112 | if len(r.MainBranch) == 0 { 113 | v.Required("MainBranch") 114 | } 115 | if len(r.MergeBranch) == 0 { 116 | v.Required("MergeBranch") 117 | } 118 | if r.RepositoryRef == nil { 119 | v.Required("RepositoryRef") 120 | } 121 | return v.Error() 122 | } 123 | 124 | // PullRequestProvider is an interface for providers that can create so-called "Pull Requests", 125 | // as popularized by Git. A Pull Request is a formal ask for a branch to be merged into the main one. 126 | // It can be UI-based, as in GitHub and GitLab, or it can be using some other method. 127 | type PullRequestProvider interface { 128 | // CreatePullRequest creates a Pull Request using the given specification. 129 | CreatePullRequest(ctx context.Context, spec PullRequestSpec) error 130 | } 131 | -------------------------------------------------------------------------------- /pkg/serializer/error.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "fmt" 5 | 6 | "k8s.io/apimachinery/pkg/runtime/schema" 7 | ) 8 | 9 | // NewUnrecognizedGroupError returns information about that the encountered group was unknown 10 | func NewUnrecognizedGroupError(gvk schema.GroupVersionKind, err error) *UnrecognizedTypeError { 11 | return &UnrecognizedTypeError{ 12 | message: fmt.Sprintf("for scheme unrecognized API group: %s", gvk.Group), 13 | GVK: gvk, 14 | Cause: UnrecognizedTypeErrorCauseUnknownGroup, 15 | Err: err, 16 | } 17 | } 18 | 19 | // NewUnrecognizedVersionError returns information about that the encountered version (in a known group) was unknown 20 | func NewUnrecognizedVersionError(allGVs []schema.GroupVersion, gvk schema.GroupVersionKind, err error) *UnrecognizedTypeError { 21 | return &UnrecognizedTypeError{ 22 | message: fmt.Sprintf("for scheme unrecognized API version: %s. Registered GroupVersions: %v", gvk.GroupVersion().String(), allGVs), 23 | GVK: gvk, 24 | Cause: UnrecognizedTypeErrorCauseUnknownVersion, 25 | Err: err, 26 | } 27 | } 28 | 29 | // NewUnrecognizedKindError returns information about that the encountered kind (in a known group & version) was unknown 30 | func NewUnrecognizedKindError(gvk schema.GroupVersionKind, err error) *UnrecognizedTypeError { 31 | return &UnrecognizedTypeError{ 32 | message: fmt.Sprintf("for scheme unrecognized kind: %s", gvk.Kind), 33 | GVK: gvk, 34 | Cause: UnrecognizedTypeErrorCauseUnknownKind, 35 | Err: err, 36 | } 37 | } 38 | 39 | // UnrecognizedTypeError describes that no such group, version and/or kind was registered in the scheme 40 | type UnrecognizedTypeError struct { 41 | message string 42 | GVK schema.GroupVersionKind 43 | Cause UnrecognizedTypeErrorCause 44 | Err error 45 | } 46 | 47 | // Error implements the error interface 48 | func (e *UnrecognizedTypeError) Error() string { 49 | return fmt.Sprintf("%s. Cause: %s. gvk: %s. error: %v", e.message, e.Cause, e.GVK, e.Err) 50 | } 51 | 52 | // GroupVersionKind returns the GroupVersionKind for the error 53 | func (e *UnrecognizedTypeError) GroupVersionKind() schema.GroupVersionKind { 54 | return e.GVK 55 | } 56 | 57 | // Unwrap allows the standard library unwrap the underlying error 58 | func (e *UnrecognizedTypeError) Unwrap() error { 59 | return e.Err 60 | } 61 | 62 | // UnrecognizedTypeErrorCause is a typed string, describing the error cause for 63 | type UnrecognizedTypeErrorCause string 64 | 65 | const ( 66 | // UnrecognizedTypeErrorCauseUnknownGroup describes that an unknown API group was encountered 67 | UnrecognizedTypeErrorCauseUnknownGroup UnrecognizedTypeErrorCause = "UnknownGroup" 68 | 69 | // UnrecognizedTypeErrorCauseUnknownVersion describes that an unknown API version for a known group was encountered 70 | UnrecognizedTypeErrorCauseUnknownVersion UnrecognizedTypeErrorCause = "UnknownVersion" 71 | 72 | // UnrecognizedTypeErrorCauseUnknownKind describes that an unknown kind for a known group and version was encountered 73 | UnrecognizedTypeErrorCauseUnknownKind UnrecognizedTypeErrorCause = "UnknownKind" 74 | ) 75 | 76 | // NewCRDConversionError creates a new CRDConversionError error 77 | func NewCRDConversionError(gvk *schema.GroupVersionKind, cause CRDConversionErrorCause, err error) *CRDConversionError { 78 | if gvk == nil { 79 | gvk = &schema.GroupVersionKind{} 80 | } 81 | return &CRDConversionError{*gvk, cause, err} 82 | } 83 | 84 | // CRDConversionError describes an error that occurred when converting CRD types 85 | type CRDConversionError struct { 86 | GVK schema.GroupVersionKind 87 | Cause CRDConversionErrorCause 88 | Err error 89 | } 90 | 91 | // Error implements the error interface 92 | func (e *CRDConversionError) Error() string { 93 | return fmt.Sprintf("object with gvk %s isn't convertible due to cause %q: %v", e.GVK, e.Cause, e.Err) 94 | } 95 | 96 | // GroupVersionKind returns the GroupVersionKind for the error 97 | func (e *CRDConversionError) GroupVersionKind() schema.GroupVersionKind { 98 | return e.GVK 99 | } 100 | 101 | // Unwrap allows the standard library unwrap the underlying error 102 | func (e *CRDConversionError) Unwrap() error { 103 | return e.Err 104 | } 105 | 106 | // CRDConversionErrorCause is a typed string, describing the error cause for 107 | type CRDConversionErrorCause string 108 | 109 | const ( 110 | // CRDConversionErrorCauseConvertTo describes an error that was caused by ConvertTo failing 111 | CRDConversionErrorCauseConvertTo CRDConversionErrorCause = "ConvertTo" 112 | 113 | // CRDConversionErrorCauseConvertTo describes an error that was caused by ConvertFrom failing 114 | CRDConversionErrorCauseConvertFrom CRDConversionErrorCause = "ConvertFrom" 115 | 116 | // CRDConversionErrorCauseConvertTo describes an error that was caused by that the scheme wasn't properly set up 117 | CRDConversionErrorCauseSchemeSetup CRDConversionErrorCause = "SchemeSetup" 118 | 119 | // CRDConversionErrorCauseInvalidArgs describes an error that was caused by that conversion targets weren't Hub and Convertible 120 | CRDConversionErrorCauseInvalidArgs CRDConversionErrorCause = "InvalidArgs" 121 | ) 122 | -------------------------------------------------------------------------------- /cmd/sample-app/client/zz_generated.client_car.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | /* 4 | Note: This file is autogenerated! Do not edit it manually! 5 | Edit client_car_template.go instead, and run 6 | hack/generate-client.sh afterwards. 7 | */ 8 | 9 | package client 10 | 11 | import ( 12 | "fmt" 13 | 14 | api "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 15 | 16 | log "github.com/sirupsen/logrus" 17 | "github.com/weaveworks/libgitops/pkg/runtime" 18 | "github.com/weaveworks/libgitops/pkg/storage" 19 | "github.com/weaveworks/libgitops/pkg/storage/filterer" 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | ) 22 | 23 | // CarClient is an interface for accessing Car-specific API objects 24 | type CarClient interface { 25 | // New returns a new Car 26 | New() *api.Car 27 | // Get returns the Car matching given UID from the storage 28 | Get(runtime.UID) (*api.Car, error) 29 | // Set saves the given Car into persistent storage 30 | Set(*api.Car) error 31 | // Patch performs a strategic merge patch on the object with 32 | // the given UID, using the byte-encoded patch given 33 | Patch(runtime.UID, []byte) error 34 | // Find returns the Car matching the given filter, filters can 35 | // match e.g. the Object's Name, UID or a specific property 36 | Find(filter filterer.BaseFilter) (*api.Car, error) 37 | // FindAll returns multiple Cars matching the given filter, filters can 38 | // match e.g. the Object's Name, UID or a specific property 39 | FindAll(filter filterer.BaseFilter) ([]*api.Car, error) 40 | // Delete deletes the Car with the given UID from the storage 41 | Delete(uid runtime.UID) error 42 | // List returns a list of all Cars available 43 | List() ([]*api.Car, error) 44 | } 45 | 46 | // Cars returns the CarClient for the Client object 47 | func (c *SampleInternalClient) Cars() CarClient { 48 | if c.carClient == nil { 49 | c.carClient = newCarClient(c.storage, c.gv) 50 | } 51 | 52 | return c.carClient 53 | } 54 | 55 | // carClient is a struct implementing the CarClient interface 56 | // It uses a shared storage instance passed from the Client together with its own Filterer 57 | type carClient struct { 58 | storage storage.Storage 59 | filterer *filterer.Filterer 60 | gvk schema.GroupVersionKind 61 | } 62 | 63 | // newCarClient builds the carClient struct using the storage implementation and a new Filterer 64 | func newCarClient(s storage.Storage, gv schema.GroupVersion) CarClient { 65 | return &carClient{ 66 | storage: s, 67 | filterer: filterer.NewFilterer(s), 68 | gvk: gv.WithKind(api.KindCar.Title()), 69 | } 70 | } 71 | 72 | // New returns a new Object of its kind 73 | func (c *carClient) New() *api.Car { 74 | log.Tracef("Client.New; GVK: %v", c.gvk) 75 | obj, err := c.storage.New(c.gvk) 76 | if err != nil { 77 | panic(fmt.Sprintf("Client.New must not return an error: %v", err)) 78 | } 79 | return obj.(*api.Car) 80 | } 81 | 82 | // Find returns a single Car based on the given Filter 83 | func (c *carClient) Find(filter filterer.BaseFilter) (*api.Car, error) { 84 | log.Tracef("Client.Find; GVK: %v", c.gvk) 85 | object, err := c.filterer.Find(c.gvk, filter) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return object.(*api.Car), nil 91 | } 92 | 93 | // FindAll returns multiple Cars based on the given Filter 94 | func (c *carClient) FindAll(filter filterer.BaseFilter) ([]*api.Car, error) { 95 | log.Tracef("Client.FindAll; GVK: %v", c.gvk) 96 | matches, err := c.filterer.FindAll(c.gvk, filter) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | results := make([]*api.Car, 0, len(matches)) 102 | for _, item := range matches { 103 | results = append(results, item.(*api.Car)) 104 | } 105 | 106 | return results, nil 107 | } 108 | 109 | // Get returns the Car matching given UID from the storage 110 | func (c *carClient) Get(uid runtime.UID) (*api.Car, error) { 111 | log.Tracef("Client.Get; UID: %q, GVK: %v", uid, c.gvk) 112 | object, err := c.storage.Get(c.gvk, uid) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | return object.(*api.Car), nil 118 | } 119 | 120 | // Set saves the given Car into the persistent storage 121 | func (c *carClient) Set(car *api.Car) error { 122 | log.Tracef("Client.Set; UID: %q, GVK: %v", car.GetUID(), c.gvk) 123 | return c.storage.Set(c.gvk, car) 124 | } 125 | 126 | // Patch performs a strategic merge patch on the object with 127 | // the given UID, using the byte-encoded patch given 128 | func (c *carClient) Patch(uid runtime.UID, patch []byte) error { 129 | return c.storage.Patch(c.gvk, uid, patch) 130 | } 131 | 132 | // Delete deletes the Car from the storage 133 | func (c *carClient) Delete(uid runtime.UID) error { 134 | log.Tracef("Client.Delete; UID: %q, GVK: %v", uid, c.gvk) 135 | return c.storage.Delete(c.gvk, uid) 136 | } 137 | 138 | // List returns a list of all Cars available 139 | func (c *carClient) List() ([]*api.Car, error) { 140 | log.Tracef("Client.List; GVK: %v", c.gvk) 141 | list, err := c.storage.List(c.gvk) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | results := make([]*api.Car, 0, len(list)) 147 | for _, item := range list { 148 | results = append(results, item.(*api.Car)) 149 | } 150 | 151 | return results, nil 152 | } 153 | -------------------------------------------------------------------------------- /pkg/serializer/comments_test.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "encoding/base64" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/runtime" 11 | runtimetest "k8s.io/apimachinery/pkg/runtime/testing" 12 | "sigs.k8s.io/kustomize/kyaml/yaml" 13 | ) 14 | 15 | const sampleData1 = `# Comment 16 | 17 | kind: Test 18 | spec: 19 | # Head comment 20 | data: 21 | - field # Inline comment 22 | - another 23 | thing: 24 | # Head comment 25 | var: true 26 | ` 27 | 28 | const sampleData2 = `kind: Test 29 | spec: 30 | # Head comment 31 | data: 32 | - field # Inline comment 33 | - another: 34 | subthing: "yes" 35 | thing: 36 | # Head comment 37 | var: true 38 | status: 39 | nested: 40 | fields: 41 | # Just a comment 42 | ` 43 | 44 | type internalSimpleOM struct { 45 | runtimetest.InternalSimple 46 | metav1.ObjectMeta `json:"metadata,omitempty"` 47 | } 48 | 49 | func withComments(data string) *internalSimpleOM { 50 | return &internalSimpleOM{ 51 | ObjectMeta: metav1.ObjectMeta{ 52 | Annotations: map[string]string{preserveCommentsAnnotation: data}, 53 | }, 54 | } 55 | } 56 | 57 | func parseRNode(t *testing.T, source string) *yaml.RNode { 58 | rNode, err := yaml.Parse(source) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | return rNode 64 | } 65 | 66 | func TestGetCommentSource(t *testing.T) { 67 | testCases := []struct { 68 | name string 69 | obj runtime.Object 70 | result string 71 | expectedErr bool 72 | }{ 73 | { 74 | name: "no_ObjectMeta", 75 | obj: &runtimetest.InternalSimple{}, 76 | expectedErr: true, 77 | }, 78 | { 79 | name: "no_comments", 80 | obj: &internalSimpleOM{}, 81 | expectedErr: true, 82 | }, 83 | { 84 | name: "invalid_comments", 85 | obj: withComments("ä"), 86 | expectedErr: true, 87 | }, 88 | { 89 | name: "successful_parsing", 90 | obj: withComments(base64.StdEncoding.EncodeToString([]byte(sampleData1))), 91 | result: sampleData1, 92 | expectedErr: false, 93 | }, 94 | } 95 | 96 | for _, tc := range testCases { 97 | t.Run(tc.name, func(t *testing.T) { 98 | source, actualErr := GetCommentSource(tc.obj) 99 | if (actualErr != nil) != tc.expectedErr { 100 | t.Errorf("expected error %t, but received %t: %v", tc.expectedErr, actualErr != nil, actualErr) 101 | } 102 | 103 | if actualErr != nil { 104 | // Already handled above. 105 | return 106 | } 107 | 108 | str, err := source.String() 109 | require.NoError(t, err) 110 | assert.Equal(t, tc.result, str) 111 | }) 112 | } 113 | } 114 | 115 | func TestSetCommentSource(t *testing.T) { 116 | testCases := []struct { 117 | name string 118 | obj runtime.Object 119 | source *yaml.RNode 120 | result string 121 | expectedErr bool 122 | }{ 123 | { 124 | name: "no_ObjectMeta", 125 | obj: &runtimetest.InternalSimple{}, 126 | source: yaml.NewScalarRNode("test"), 127 | expectedErr: true, 128 | }, 129 | { 130 | name: "nil_source", 131 | obj: &internalSimpleOM{}, 132 | source: nil, 133 | result: "", 134 | expectedErr: false, 135 | }, 136 | { 137 | name: "successful_parsing", 138 | obj: &internalSimpleOM{}, 139 | source: parseRNode(t, sampleData1), 140 | result: sampleData1, 141 | expectedErr: false, 142 | }, 143 | } 144 | 145 | for _, tc := range testCases { 146 | t.Run(tc.name, func(t *testing.T) { 147 | actualErr := SetCommentSource(tc.obj, tc.source) 148 | if (actualErr != nil) != tc.expectedErr { 149 | t.Errorf("expected error %t, but received %t: %v", tc.expectedErr, actualErr != nil, actualErr) 150 | } 151 | 152 | if actualErr != nil { 153 | // Already handled above. 154 | return 155 | } 156 | 157 | meta, ok := toMetaObject(tc.obj) 158 | if !ok { 159 | t.Fatal("cannot extract metav1.ObjectMeta") 160 | } 161 | 162 | annotation, ok := getAnnotation(meta, preserveCommentsAnnotation) 163 | if !ok { 164 | t.Fatal("expected annotation to be set, but it is not") 165 | } 166 | 167 | str, err := base64.StdEncoding.DecodeString(annotation) 168 | require.NoError(t, err) 169 | assert.Equal(t, tc.result, string(str)) 170 | }) 171 | } 172 | } 173 | 174 | func TestCommentSourceSetGet(t *testing.T) { 175 | testCases := []struct { 176 | name string 177 | source string 178 | }{ 179 | { 180 | name: "encode_decode_1", 181 | source: sampleData1, 182 | }, 183 | { 184 | name: "encode_decode_2", 185 | source: sampleData2, 186 | }, 187 | } 188 | 189 | for _, tc := range testCases { 190 | t.Run(tc.name, func(t *testing.T) { 191 | obj := &internalSimpleOM{} 192 | assert.NoError(t, SetCommentSource(obj, parseRNode(t, tc.source))) 193 | 194 | rNode, err := GetCommentSource(obj) 195 | assert.NoError(t, err) 196 | 197 | str, err := rNode.String() 198 | require.NoError(t, err) 199 | assert.Equal(t, tc.source, str) 200 | }) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /pkg/storage/mappedrawstorage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "github.com/weaveworks/libgitops/pkg/serializer" 12 | "github.com/weaveworks/libgitops/pkg/util" 13 | ) 14 | 15 | var ( 16 | // ErrNotTracked is returned when the requested resource wasn't found. 17 | ErrNotTracked = fmt.Errorf("untracked object: %w", ErrNotFound) 18 | ) 19 | 20 | // MappedRawStorage is an interface for RawStorages which store their 21 | // data in a flat/unordered directory format like manifest directories. 22 | type MappedRawStorage interface { 23 | RawStorage 24 | 25 | // AddMapping binds a Key's virtual path to a physical file path 26 | AddMapping(key ObjectKey, path string) 27 | // RemoveMapping removes the physical file 28 | // path mapping matching the given Key 29 | RemoveMapping(key ObjectKey) 30 | 31 | // SetMappings overwrites all known mappings 32 | SetMappings(m map[ObjectKey]string) 33 | } 34 | 35 | func NewGenericMappedRawStorage(dir string) MappedRawStorage { 36 | return &GenericMappedRawStorage{ 37 | dir: dir, 38 | fileMappings: make(map[ObjectKey]string), 39 | mux: &sync.Mutex{}, 40 | } 41 | } 42 | 43 | // GenericMappedRawStorage is the default implementation of a MappedRawStorage, 44 | // it stores files in the given directory via a path translation map. 45 | type GenericMappedRawStorage struct { 46 | dir string 47 | fileMappings map[ObjectKey]string 48 | mux *sync.Mutex 49 | } 50 | 51 | func (r *GenericMappedRawStorage) realPath(key ObjectKey) (string, error) { 52 | r.mux.Lock() 53 | path, ok := r.fileMappings[key] 54 | r.mux.Unlock() 55 | if !ok { 56 | return "", fmt.Errorf("GenericMappedRawStorage: cannot resolve %q: %w", key, ErrNotTracked) 57 | } 58 | 59 | return path, nil 60 | } 61 | 62 | // If the file doesn't exist, returns ErrNotFound + ErrNotTracked. 63 | func (r *GenericMappedRawStorage) Read(key ObjectKey) ([]byte, error) { 64 | file, err := r.realPath(key) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return ioutil.ReadFile(file) 70 | } 71 | 72 | func (r *GenericMappedRawStorage) Exists(key ObjectKey) bool { 73 | file, err := r.realPath(key) 74 | if err != nil { 75 | return false 76 | } 77 | 78 | return util.FileExists(file) 79 | } 80 | 81 | func (r *GenericMappedRawStorage) Write(key ObjectKey, content []byte) error { 82 | // GenericMappedRawStorage isn't going to generate files itself, 83 | // only write if the file is already known 84 | file, err := r.realPath(key) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | return ioutil.WriteFile(file, content, 0644) 90 | } 91 | 92 | // If the file doesn't exist, returns ErrNotFound + ErrNotTracked. 93 | func (r *GenericMappedRawStorage) Delete(key ObjectKey) (err error) { 94 | file, err := r.realPath(key) 95 | if err != nil { 96 | return 97 | } 98 | 99 | // GenericMappedRawStorage files can be deleted 100 | // externally, check that the file exists first 101 | if util.FileExists(file) { 102 | err = os.Remove(file) 103 | } 104 | 105 | if err == nil { 106 | r.RemoveMapping(key) 107 | } 108 | 109 | return 110 | } 111 | 112 | func (r *GenericMappedRawStorage) List(kind KindKey) ([]ObjectKey, error) { 113 | result := make([]ObjectKey, 0) 114 | 115 | for key := range r.fileMappings { 116 | // Include objects with the same kind and group, ignore version mismatches 117 | if key.EqualsGVK(kind, false) { 118 | result = append(result, key) 119 | } 120 | } 121 | 122 | return result, nil 123 | } 124 | 125 | // This returns the modification time as a UnixNano string. 126 | // If the file doesn't exist, returns ErrNotFound + ErrNotTracked. 127 | func (r *GenericMappedRawStorage) Checksum(key ObjectKey) (string, error) { 128 | path, err := r.realPath(key) 129 | if err != nil { 130 | return "", err 131 | } 132 | 133 | return checksumFromModTime(path) 134 | } 135 | 136 | func (r *GenericMappedRawStorage) ContentType(key ObjectKey) (ct serializer.ContentType) { 137 | if file, err := r.realPath(key); err == nil { 138 | ct = ContentTypes[filepath.Ext(file)] // Retrieve the correct format based on the extension 139 | } 140 | 141 | return 142 | } 143 | 144 | func (r *GenericMappedRawStorage) WatchDir() string { 145 | return r.dir 146 | } 147 | 148 | func (r *GenericMappedRawStorage) GetKey(path string) (ObjectKey, error) { 149 | for key, p := range r.fileMappings { 150 | if p == path { 151 | return key, nil 152 | } 153 | } 154 | 155 | return objectKey{}, fmt.Errorf("no mapping found for path %q", path) 156 | } 157 | 158 | func (r *GenericMappedRawStorage) AddMapping(key ObjectKey, path string) { 159 | log.Debugf("GenericMappedRawStorage: AddMapping: %q -> %q", key, path) 160 | r.mux.Lock() 161 | r.fileMappings[key] = path 162 | r.mux.Unlock() 163 | } 164 | 165 | func (r *GenericMappedRawStorage) RemoveMapping(key ObjectKey) { 166 | log.Debugf("GenericMappedRawStorage: RemoveMapping: %q", key) 167 | r.mux.Lock() 168 | delete(r.fileMappings, key) 169 | r.mux.Unlock() 170 | } 171 | 172 | func (r *GenericMappedRawStorage) SetMappings(m map[ObjectKey]string) { 173 | log.Debugf("GenericMappedRawStorage: SetMappings: %v", m) 174 | r.mux.Lock() 175 | r.fileMappings = m 176 | r.mux.Unlock() 177 | } 178 | -------------------------------------------------------------------------------- /pkg/storage/transaction/git.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/sirupsen/logrus" 9 | "github.com/weaveworks/libgitops/pkg/gitdir" 10 | "github.com/weaveworks/libgitops/pkg/runtime" 11 | "github.com/weaveworks/libgitops/pkg/serializer" 12 | "github.com/weaveworks/libgitops/pkg/storage" 13 | "github.com/weaveworks/libgitops/pkg/util" 14 | "github.com/weaveworks/libgitops/pkg/util/watcher" 15 | ) 16 | 17 | var excludeDirs = []string{".git"} 18 | 19 | func NewGitStorage(gitDir gitdir.GitDirectory, prProvider PullRequestProvider, ser serializer.Serializer) (TransactionStorage, error) { 20 | // Make sure the repo is cloned. If this func has already been called, it will be a no-op. 21 | if err := gitDir.StartCheckoutLoop(); err != nil { 22 | return nil, err 23 | } 24 | 25 | raw := storage.NewGenericMappedRawStorage(gitDir.Dir()) 26 | s := storage.NewGenericStorage(raw, ser, []runtime.IdentifierFactory{runtime.Metav1NameIdentifier}) 27 | 28 | gitStorage := &GitStorage{ 29 | ReadStorage: s, 30 | s: s, 31 | raw: raw, 32 | gitDir: gitDir, 33 | prProvider: prProvider, 34 | } 35 | // Do a first sync now, and then start the background loop 36 | if err := gitStorage.sync(); err != nil { 37 | return nil, err 38 | } 39 | gitStorage.syncLoop() 40 | 41 | return gitStorage, nil 42 | } 43 | 44 | type GitStorage struct { 45 | storage.ReadStorage 46 | 47 | s storage.Storage 48 | raw storage.MappedRawStorage 49 | gitDir gitdir.GitDirectory 50 | prProvider PullRequestProvider 51 | } 52 | 53 | func (s *GitStorage) syncLoop() { 54 | go func() { 55 | for { 56 | if commit, ok := <-s.gitDir.CommitChannel(); ok { 57 | logrus.Debugf("GitStorage: Got info about commit %q, syncing...", commit) 58 | if err := s.sync(); err != nil { 59 | logrus.Errorf("GitStorage: Got sync error: %v", err) 60 | } 61 | } 62 | } 63 | }() 64 | } 65 | 66 | func (s *GitStorage) sync() error { 67 | mappings, err := computeMappings(s.gitDir.Dir(), s.s) 68 | if err != nil { 69 | return err 70 | } 71 | logrus.Debugf("Rewriting the mappings to %v", mappings) 72 | s.raw.SetMappings(mappings) 73 | return nil 74 | } 75 | 76 | func (s *GitStorage) Transaction(ctx context.Context, streamName string, fn TransactionFunc) error { 77 | // Append random bytes to the end of the stream name if it ends with a dash 78 | if strings.HasSuffix(streamName, "-") { 79 | suffix, err := util.RandomSHA(4) 80 | if err != nil { 81 | return err 82 | } 83 | streamName += suffix 84 | } 85 | 86 | // Make sure we have the latest available state 87 | if err := s.gitDir.Pull(ctx); err != nil { 88 | return err 89 | } 90 | // Make sure no other Git ops can take place during the transaction, wait for other ongoing operations. 91 | s.gitDir.Suspend() 92 | defer s.gitDir.Resume() 93 | // Always switch back to the main branch afterwards. 94 | // TODO ordering of the defers, and return deferred error 95 | defer func() { _ = s.gitDir.CheckoutMainBranch() }() 96 | 97 | // Check out a new branch with the given name 98 | if err := s.gitDir.CheckoutNewBranch(streamName); err != nil { 99 | return err 100 | } 101 | // Invoke the transaction 102 | result, err := fn(ctx, s.s) 103 | if err != nil { 104 | return err 105 | } 106 | // Make sure the result is valid 107 | if err := result.Validate(); err != nil { 108 | return fmt.Errorf("transaction result is not valid: %w", err) 109 | } 110 | // Perform the commit 111 | if err := s.gitDir.Commit(ctx, result.GetAuthorName(), result.GetAuthorEmail(), result.GetMessage()); err != nil { 112 | return err 113 | } 114 | // Return if no PR should be made 115 | prResult, ok := result.(PullRequestResult) 116 | if !ok { 117 | return nil 118 | } 119 | // If a PR was asked for, and no provider was given, error out 120 | if s.prProvider == nil { 121 | return ErrNoPullRequestProvider 122 | } 123 | // Create the PR using the provider. 124 | return s.prProvider.CreatePullRequest(ctx, &GenericPullRequestSpec{ 125 | PullRequestResult: prResult, 126 | MainBranch: s.gitDir.MainBranch(), 127 | MergeBranch: streamName, 128 | RepositoryRef: s.gitDir.RepositoryRef(), 129 | }) 130 | } 131 | 132 | func computeMappings(dir string, s storage.Storage) (map[storage.ObjectKey]string, error) { 133 | validExts := make([]string, 0, len(storage.ContentTypes)) 134 | for ext := range storage.ContentTypes { 135 | validExts = append(validExts, ext) 136 | } 137 | 138 | files, err := watcher.WalkDirectoryForFiles(dir, validExts, excludeDirs) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | // TODO: Compute the difference between the earlier state, and implement EventStorage so the user 144 | // can automatically subscribe to changes of objects between versions. 145 | m := map[storage.ObjectKey]string{} 146 | for _, file := range files { 147 | partObjs, err := storage.DecodePartialObjects(serializer.FromFile(file), s.Serializer().Scheme(), false, nil) 148 | if err != nil { 149 | logrus.Errorf("couldn't decode %q into a partial object: %v", file, err) 150 | continue 151 | } 152 | key, err := s.ObjectKeyFor(partObjs[0]) 153 | if err != nil { 154 | logrus.Errorf("couldn't get objectkey for partial object: %v", err) 155 | continue 156 | } 157 | logrus.Debugf("Adding mapping between %s and %q", key, file) 158 | m[key] = file 159 | } 160 | return m, nil 161 | } 162 | -------------------------------------------------------------------------------- /pkg/client/client_resource_template.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | /* 4 | Note: This file is autogenerated! Do not edit it manually! 5 | Edit client_resource_template.go instead, and run 6 | hack/generate-client.sh afterwards. 7 | */ 8 | 9 | package client 10 | 11 | import ( 12 | "fmt" 13 | 14 | api "API_DIR" 15 | 16 | log "github.com/sirupsen/logrus" 17 | "github.com/weaveworks/libgitops/pkg/runtime" 18 | "github.com/weaveworks/libgitops/pkg/storage" 19 | "github.com/weaveworks/libgitops/pkg/storage/filterer" 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | ) 22 | 23 | // ResourceClient is an interface for accessing Resource-specific API objects 24 | type ResourceClient interface { 25 | // New returns a new Resource 26 | New() *api.Resource 27 | // Get returns the Resource matching given UID from the storage 28 | Get(runtime.UID) (*api.Resource, error) 29 | // Set saves the given Resource into persistent storage 30 | Set(*api.Resource) error 31 | // Patch performs a strategic merge patch on the object with 32 | // the given UID, using the byte-encoded patch given 33 | Patch(runtime.UID, []byte) error 34 | // Find returns the Resource matching the given filter, filters can 35 | // match e.g. the Object's Name, UID or a specific property 36 | Find(filter filterer.BaseFilter) (*api.Resource, error) 37 | // FindAll returns multiple Resources matching the given filter, filters can 38 | // match e.g. the Object's Name, UID or a specific property 39 | FindAll(filter filterer.BaseFilter) ([]*api.Resource, error) 40 | // Delete deletes the Resource with the given UID from the storage 41 | Delete(uid runtime.UID) error 42 | // List returns a list of all Resources available 43 | List() ([]*api.Resource, error) 44 | } 45 | 46 | // Resources returns the ResourceClient for the Client object 47 | func (c *Client) Resources() ResourceClient { 48 | if c.resourceClient == nil { 49 | c.resourceClient = newResourceClient(c.storage, c.gv) 50 | } 51 | 52 | return c.resourceClient 53 | } 54 | 55 | // resourceClient is a struct implementing the ResourceClient interface 56 | // It uses a shared storage instance passed from the Client together with its own Filterer 57 | type resourceClient struct { 58 | storage storage.Storage 59 | filterer *filterer.Filterer 60 | gvk schema.GroupVersionKind 61 | } 62 | 63 | // newResourceClient builds the resourceClient struct using the storage implementation and a new Filterer 64 | func newResourceClient(s storage.Storage, gv schema.GroupVersion) ResourceClient { 65 | return &resourceClient{ 66 | storage: s, 67 | filterer: filterer.NewFilterer(s), 68 | gvk: gv.WithKind(api.KindResource.Title()), 69 | } 70 | } 71 | 72 | // New returns a new Object of its kind 73 | func (c *resourceClient) New() *api.Resource { 74 | log.Tracef("Client.New; GVK: %v", c.gvk) 75 | obj, err := c.storage.New(c.gvk) 76 | if err != nil { 77 | panic(fmt.Sprintf("Client.New must not return an error: %v", err)) 78 | } 79 | return obj.(*api.Resource) 80 | } 81 | 82 | // Find returns a single Resource based on the given Filter 83 | func (c *resourceClient) Find(filter filterer.BaseFilter) (*api.Resource, error) { 84 | log.Tracef("Client.Find; GVK: %v", c.gvk) 85 | object, err := c.filterer.Find(c.gvk, filter) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return object.(*api.Resource), nil 91 | } 92 | 93 | // FindAll returns multiple Resources based on the given Filter 94 | func (c *resourceClient) FindAll(filter filterer.BaseFilter) ([]*api.Resource, error) { 95 | log.Tracef("Client.FindAll; GVK: %v", c.gvk) 96 | matches, err := c.filterer.FindAll(c.gvk, filter) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | results := make([]*api.Resource, 0, len(matches)) 102 | for _, item := range matches { 103 | results = append(results, item.(*api.Resource)) 104 | } 105 | 106 | return results, nil 107 | } 108 | 109 | // Get returns the Resource matching given UID from the storage 110 | func (c *resourceClient) Get(uid runtime.UID) (*api.Resource, error) { 111 | log.Tracef("Client.Get; UID: %q, GVK: %v", uid, c.gvk) 112 | object, err := c.storage.Get(c.gvk, uid) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | return object.(*api.Resource), nil 118 | } 119 | 120 | // Set saves the given Resource into the persistent storage 121 | func (c *resourceClient) Set(resource *api.Resource) error { 122 | log.Tracef("Client.Set; UID: %q, GVK: %v", resource.GetUID(), c.gvk) 123 | return c.storage.Set(c.gvk, resource) 124 | } 125 | 126 | // Patch performs a strategic merge patch on the object with 127 | // the given UID, using the byte-encoded patch given 128 | func (c *resourceClient) Patch(uid runtime.UID, patch []byte) error { 129 | return c.storage.Patch(c.gvk, uid, patch) 130 | } 131 | 132 | // Delete deletes the Resource from the storage 133 | func (c *resourceClient) Delete(uid runtime.UID) error { 134 | log.Tracef("Client.Delete; UID: %q, GVK: %v", uid, c.gvk) 135 | return c.storage.Delete(c.gvk, uid) 136 | } 137 | 138 | // List returns a list of all Resources available 139 | func (c *resourceClient) List() ([]*api.Resource, error) { 140 | log.Tracef("Client.List; GVK: %v", c.gvk) 141 | list, err := c.storage.List(c.gvk) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | results := make([]*api.Resource, 0, len(list)) 147 | for _, item := range list { 148 | results = append(results, item.(*api.Resource)) 149 | } 150 | 151 | return results, nil 152 | } 153 | -------------------------------------------------------------------------------- /cmd/sample-app/client/zz_generated.client_motorcycle.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | /* 4 | Note: This file is autogenerated! Do not edit it manually! 5 | Edit client_motorcycle_template.go instead, and run 6 | hack/generate-client.sh afterwards. 7 | */ 8 | 9 | package client 10 | 11 | import ( 12 | "fmt" 13 | 14 | api "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample" 15 | 16 | log "github.com/sirupsen/logrus" 17 | "github.com/weaveworks/libgitops/pkg/runtime" 18 | "github.com/weaveworks/libgitops/pkg/storage" 19 | "github.com/weaveworks/libgitops/pkg/storage/filterer" 20 | "k8s.io/apimachinery/pkg/runtime/schema" 21 | ) 22 | 23 | // MotorcycleClient is an interface for accessing Motorcycle-specific API objects 24 | type MotorcycleClient interface { 25 | // New returns a new Motorcycle 26 | New() *api.Motorcycle 27 | // Get returns the Motorcycle matching given UID from the storage 28 | Get(runtime.UID) (*api.Motorcycle, error) 29 | // Set saves the given Motorcycle into persistent storage 30 | Set(*api.Motorcycle) error 31 | // Patch performs a strategic merge patch on the object with 32 | // the given UID, using the byte-encoded patch given 33 | Patch(runtime.UID, []byte) error 34 | // Find returns the Motorcycle matching the given filter, filters can 35 | // match e.g. the Object's Name, UID or a specific property 36 | Find(filter filterer.BaseFilter) (*api.Motorcycle, error) 37 | // FindAll returns multiple Motorcycles matching the given filter, filters can 38 | // match e.g. the Object's Name, UID or a specific property 39 | FindAll(filter filterer.BaseFilter) ([]*api.Motorcycle, error) 40 | // Delete deletes the Motorcycle with the given UID from the storage 41 | Delete(uid runtime.UID) error 42 | // List returns a list of all Motorcycles available 43 | List() ([]*api.Motorcycle, error) 44 | } 45 | 46 | // Motorcycles returns the MotorcycleClient for the Client object 47 | func (c *SampleInternalClient) Motorcycles() MotorcycleClient { 48 | if c.motorcycleClient == nil { 49 | c.motorcycleClient = newMotorcycleClient(c.storage, c.gv) 50 | } 51 | 52 | return c.motorcycleClient 53 | } 54 | 55 | // motorcycleClient is a struct implementing the MotorcycleClient interface 56 | // It uses a shared storage instance passed from the Client together with its own Filterer 57 | type motorcycleClient struct { 58 | storage storage.Storage 59 | filterer *filterer.Filterer 60 | gvk schema.GroupVersionKind 61 | } 62 | 63 | // newMotorcycleClient builds the motorcycleClient struct using the storage implementation and a new Filterer 64 | func newMotorcycleClient(s storage.Storage, gv schema.GroupVersion) MotorcycleClient { 65 | return &motorcycleClient{ 66 | storage: s, 67 | filterer: filterer.NewFilterer(s), 68 | gvk: gv.WithKind(api.KindMotorcycle.Title()), 69 | } 70 | } 71 | 72 | // New returns a new Object of its kind 73 | func (c *motorcycleClient) New() *api.Motorcycle { 74 | log.Tracef("Client.New; GVK: %v", c.gvk) 75 | obj, err := c.storage.New(c.gvk) 76 | if err != nil { 77 | panic(fmt.Sprintf("Client.New must not return an error: %v", err)) 78 | } 79 | return obj.(*api.Motorcycle) 80 | } 81 | 82 | // Find returns a single Motorcycle based on the given Filter 83 | func (c *motorcycleClient) Find(filter filterer.BaseFilter) (*api.Motorcycle, error) { 84 | log.Tracef("Client.Find; GVK: %v", c.gvk) 85 | object, err := c.filterer.Find(c.gvk, filter) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return object.(*api.Motorcycle), nil 91 | } 92 | 93 | // FindAll returns multiple Motorcycles based on the given Filter 94 | func (c *motorcycleClient) FindAll(filter filterer.BaseFilter) ([]*api.Motorcycle, error) { 95 | log.Tracef("Client.FindAll; GVK: %v", c.gvk) 96 | matches, err := c.filterer.FindAll(c.gvk, filter) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | results := make([]*api.Motorcycle, 0, len(matches)) 102 | for _, item := range matches { 103 | results = append(results, item.(*api.Motorcycle)) 104 | } 105 | 106 | return results, nil 107 | } 108 | 109 | // Get returns the Motorcycle matching given UID from the storage 110 | func (c *motorcycleClient) Get(uid runtime.UID) (*api.Motorcycle, error) { 111 | log.Tracef("Client.Get; UID: %q, GVK: %v", uid, c.gvk) 112 | object, err := c.storage.Get(c.gvk, uid) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | return object.(*api.Motorcycle), nil 118 | } 119 | 120 | // Set saves the given Motorcycle into the persistent storage 121 | func (c *motorcycleClient) Set(motorcycle *api.Motorcycle) error { 122 | log.Tracef("Client.Set; UID: %q, GVK: %v", motorcycle.GetUID(), c.gvk) 123 | return c.storage.Set(c.gvk, motorcycle) 124 | } 125 | 126 | // Patch performs a strategic merge patch on the object with 127 | // the given UID, using the byte-encoded patch given 128 | func (c *motorcycleClient) Patch(uid runtime.UID, patch []byte) error { 129 | return c.storage.Patch(c.gvk, uid, patch) 130 | } 131 | 132 | // Delete deletes the Motorcycle from the storage 133 | func (c *motorcycleClient) Delete(uid runtime.UID) error { 134 | log.Tracef("Client.Delete; UID: %q, GVK: %v", uid, c.gvk) 135 | return c.storage.Delete(c.gvk, uid) 136 | } 137 | 138 | // List returns a list of all Motorcycles available 139 | func (c *motorcycleClient) List() ([]*api.Motorcycle, error) { 140 | log.Tracef("Client.List; GVK: %v", c.gvk) 141 | list, err := c.storage.List(c.gvk) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | results := make([]*api.Motorcycle, 0, len(list)) 147 | for _, item := range list { 148 | results = append(results, item.(*api.Motorcycle)) 149 | } 150 | 151 | return results, nil 152 | } 153 | -------------------------------------------------------------------------------- /pkg/serializer/encode.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/weaveworks/libgitops/pkg/util" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | ) 9 | 10 | type EncodingOptions struct { 11 | // Use pretty printing when writing to the output. (Default: true) 12 | // TODO: Fix that sometimes omitempty fields aren't respected 13 | Pretty *bool 14 | // Whether to preserve YAML comments internally. This only works for objects embedding metav1.ObjectMeta. 15 | // Only applicable to ContentTypeYAML framers. 16 | // Using any other framer will be silently ignored. Usage of this option also requires setting 17 | // the PreserveComments in DecodingOptions, too. (Default: false) 18 | // TODO: Make this a BestEffort & Strict mode 19 | PreserveComments *bool 20 | 21 | // TODO: Maybe consider an option to always convert to the preferred version (not just internal) 22 | } 23 | 24 | type EncodingOptionsFunc func(*EncodingOptions) 25 | 26 | func WithPrettyEncode(pretty bool) EncodingOptionsFunc { 27 | return func(opts *EncodingOptions) { 28 | opts.Pretty = &pretty 29 | } 30 | } 31 | 32 | func WithCommentsEncode(comments bool) EncodingOptionsFunc { 33 | return func(opts *EncodingOptions) { 34 | opts.PreserveComments = &comments 35 | } 36 | } 37 | 38 | func WithEncodingOptions(newOpts EncodingOptions) EncodingOptionsFunc { 39 | return func(opts *EncodingOptions) { 40 | // TODO: Null-check all of these before using them 41 | *opts = newOpts 42 | } 43 | } 44 | 45 | func defaultEncodeOpts() *EncodingOptions { 46 | return &EncodingOptions{ 47 | Pretty: util.BoolPtr(true), 48 | PreserveComments: util.BoolPtr(false), 49 | } 50 | } 51 | 52 | func newEncodeOpts(fns ...EncodingOptionsFunc) *EncodingOptions { 53 | opts := defaultEncodeOpts() 54 | for _, fn := range fns { 55 | fn(opts) 56 | } 57 | return opts 58 | } 59 | 60 | type encoder struct { 61 | *schemeAndCodec 62 | 63 | opts EncodingOptions 64 | } 65 | 66 | func newEncoder(schemeAndCodec *schemeAndCodec, opts EncodingOptions) Encoder { 67 | return &encoder{ 68 | schemeAndCodec, 69 | opts, 70 | } 71 | } 72 | 73 | // Encode encodes the given objects and writes them to the specified FrameWriter. 74 | // The FrameWriter specifies the ContentType. This encoder will automatically convert any 75 | // internal object given to the preferred external groupversion. No conversion will happen 76 | // if the given object is of an external version. 77 | // TODO: This should automatically convert to the preferred version 78 | func (e *encoder) Encode(fw FrameWriter, objs ...runtime.Object) error { 79 | for _, obj := range objs { 80 | // Get the kind for the given object 81 | gvk, err := GVKForObject(e.scheme, obj) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // If the object is internal, convert it to the preferred external one 87 | if gvk.Version == runtime.APIVersionInternal { 88 | gv, err := prioritizedVersionForGroup(e.scheme, gvk.Group) 89 | if err != nil { 90 | return err 91 | } 92 | gvk.Version = gv.Version 93 | } 94 | 95 | // Encode it 96 | if err := e.EncodeForGroupVersion(fw, obj, gvk.GroupVersion()); err != nil { 97 | return err 98 | } 99 | } 100 | return nil 101 | } 102 | 103 | // EncodeForGroupVersion encodes the given object for the specific groupversion. If the object 104 | // is not of that version currently it will try to convert. The output bytes are written to the 105 | // FrameWriter. The FrameWriter specifies the ContentType. 106 | func (e *encoder) EncodeForGroupVersion(fw FrameWriter, obj runtime.Object, gv schema.GroupVersion) error { 107 | // Get the serializer for the media type 108 | serializerInfo, ok := runtime.SerializerInfoForMediaType(e.codecs.SupportedMediaTypes(), string(fw.ContentType())) 109 | if !ok { 110 | return ErrUnsupportedContentType 111 | } 112 | 113 | // Choose the pretty or non-pretty one 114 | encoder := serializerInfo.Serializer 115 | 116 | // Use the pretty serializer if it was asked for and is defined for the content type 117 | if *e.opts.Pretty { 118 | // Apparently not all SerializerInfos have this field defined (e.g. YAML) 119 | // TODO: This could be considered a bug in upstream, create an issue 120 | if serializerInfo.PrettySerializer != nil { 121 | encoder = serializerInfo.PrettySerializer 122 | } else { 123 | logrus.Debugf("PrettySerializer for ContentType %s is nil, falling back to Serializer.", fw.ContentType()) 124 | } 125 | } 126 | 127 | // Get a version-specific encoder for the specified groupversion 128 | versionEncoder := encoderForVersion(e.scheme, encoder, gv) 129 | 130 | // Cast the object to a metav1.Object to get access to annotations 131 | metaobj, ok := toMetaObject(obj) 132 | // For objects without ObjectMeta, the cast will fail. Allow that failure and do "normal" encoding 133 | if !ok { 134 | return versionEncoder.Encode(obj, fw) 135 | } 136 | 137 | // Specialize the encoder for a specific gv and encode the object 138 | return e.encodeWithCommentSupport(versionEncoder, fw, obj, metaobj) 139 | } 140 | 141 | // encoderForVersion is used instead of CodecFactory.EncoderForVersion, as we want to use our own converter 142 | func encoderForVersion(scheme *runtime.Scheme, encoder runtime.Encoder, gv schema.GroupVersion) runtime.Encoder { 143 | return newConversionCodecForScheme( 144 | scheme, 145 | encoder, // what content-type encoder to use 146 | nil, // no decoder 147 | gv, // specify what the target encode groupversion is 148 | nil, // no target decode groupversion 149 | false, // no defaulting 150 | true, // convert if needed before encode 151 | ) 152 | } 153 | -------------------------------------------------------------------------------- /pkg/serializer/frame_reader.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | 10 | "k8s.io/apimachinery/pkg/runtime/serializer/json" 11 | ) 12 | 13 | const ( 14 | defaultBufSize = 64 * 1024 // 64 kB 15 | defaultMaxFrameSize = 16 * 1024 * 1024 // 16 MB 16 | ) 17 | 18 | var ( 19 | // FrameOverflowErr is returned from FrameReader.ReadFrame when one frame exceeds the 20 | // maximum size of 16 MB. 21 | FrameOverflowErr = errors.New("frame was larger than maximum allowed size") 22 | ) 23 | 24 | // ReadCloser in this package is an alias for io.ReadCloser. It helps in Godoc to locate 25 | // helpers in this package which returns writers (i.e. FromFile and FromBytes) 26 | type ReadCloser io.ReadCloser 27 | 28 | // FrameReader is a content-type specific reader of a given ReadCloser. 29 | // The FrameReader reads frames from the underlying ReadCloser and returns them for consumption. 30 | // When io.EOF is reached, the stream is closed automatically. 31 | type FrameReader interface { 32 | ContentTyped 33 | io.Closer 34 | 35 | // ReadFrame reads frames from the underlying ReadCloser and returns them for consumption. 36 | // When io.EOF is reached, the stream is closed automatically. 37 | ReadFrame() ([]byte, error) 38 | } 39 | 40 | // NewFrameReader returns a FrameReader for the given ContentType and data in the 41 | // ReadCloser. The Reader is automatically closed in io.EOF. ReadFrame is called 42 | // once each Decoder.Decode() or Decoder.DecodeInto() call. When Decoder.DecodeAll() is 43 | // called, the FrameReader is read until io.EOF, upon where it is closed. 44 | func NewFrameReader(contentType ContentType, rc ReadCloser) FrameReader { 45 | switch contentType { 46 | case ContentTypeYAML: 47 | return newFrameReader(json.YAMLFramer.NewFrameReader(rc), contentType) 48 | case ContentTypeJSON: 49 | return newFrameReader(json.Framer.NewFrameReader(rc), contentType) 50 | default: 51 | return &errFrameReader{ErrUnsupportedContentType, contentType} 52 | } 53 | } 54 | 55 | // NewYAMLFrameReader returns a FrameReader that supports both YAML and JSON. Frames are separated by "---\n" 56 | // 57 | // This call is the same as NewFrameReader(ContentTypeYAML, rc) 58 | func NewYAMLFrameReader(rc ReadCloser) FrameReader { 59 | return NewFrameReader(ContentTypeYAML, rc) 60 | } 61 | 62 | // NewJSONFrameReader returns a FrameReader that supports both JSON. Objects are read from the stream one-by-one, 63 | // each object making up its own frame. 64 | // 65 | // This call is the same as NewFrameReader(ContentTypeJSON, rc) 66 | func NewJSONFrameReader(rc ReadCloser) FrameReader { 67 | return NewFrameReader(ContentTypeJSON, rc) 68 | } 69 | 70 | // newFrameReader returns a new instance of the frameReader struct 71 | func newFrameReader(rc io.ReadCloser, contentType ContentType) *frameReader { 72 | return &frameReader{ 73 | rc: rc, 74 | bufSize: defaultBufSize, 75 | maxFrameSize: defaultMaxFrameSize, 76 | contentType: contentType, 77 | } 78 | } 79 | 80 | // frameReader is a FrameReader implementation 81 | type frameReader struct { 82 | rc io.ReadCloser 83 | bufSize int 84 | maxFrameSize int 85 | contentType ContentType 86 | 87 | // TODO: Maybe add mutexes for thread-safety (so no two goroutines read at the same time) 88 | } 89 | 90 | // ReadFrame reads one frame from the underlying io.Reader. ReadFrame 91 | // keeps on reading from the Reader in bufSize blocks, until the Reader either 92 | // returns err == nil or EOF. If the Reader reports an ErrShortBuffer error, 93 | // ReadFrame keeps on reading using new calls. ReadFrame might return both data and 94 | // io.EOF. io.EOF will be returned in the final call. 95 | func (rf *frameReader) ReadFrame() (frame []byte, err error) { 96 | // Temporary buffer to parts of a frame into 97 | var buf []byte 98 | // How many bytes were read by the read call 99 | var n int 100 | // Multiplier for bufsize 101 | c := 1 102 | for { 103 | // Allocate a buffer of a multiple of bufSize. 104 | buf = make([]byte, c*rf.bufSize) 105 | // Call the underlying reader. 106 | n, err = rf.rc.Read(buf) 107 | // Append the returned bytes to the b slice returned 108 | // If n is 0, this call is a no-op 109 | frame = append(frame, buf[:n]...) 110 | 111 | // If the frame got bigger than the max allowed size, return and report the error 112 | if len(frame) > rf.maxFrameSize { 113 | err = FrameOverflowErr 114 | return 115 | } 116 | 117 | // Handle different kinds of errors 118 | switch err { 119 | case io.ErrShortBuffer: 120 | // ignore the "buffer too short" error, and just keep on reading, now doubling the buffer 121 | c *= 2 122 | continue 123 | case nil: 124 | // One document is "done reading", we should return it if valid 125 | // Only return non-empty documents, i.e. skip e.g. leading `---` 126 | if len(bytes.TrimSpace(frame)) > 0 { 127 | // valid non-empty document 128 | return 129 | } 130 | // The document was empty, reset the frame (just to be sure) and continue 131 | frame = nil 132 | continue 133 | case io.EOF: 134 | // we reached the end of the file, close the reader and return 135 | rf.rc.Close() 136 | return 137 | default: 138 | // unknown error, return it immediately 139 | // TODO: Maybe return the error here? 140 | return 141 | } 142 | } 143 | } 144 | 145 | // ContentType returns the content type for the given FrameReader 146 | func (rf *frameReader) ContentType() ContentType { 147 | return rf.contentType 148 | } 149 | 150 | // Close implements io.Closer and closes the underlying ReadCloser 151 | func (rf *frameReader) Close() error { 152 | return rf.rc.Close() 153 | } 154 | 155 | // FromFile returns a ReadCloser from the given file, or a ReadCloser which returns 156 | // the given file open error when read. 157 | func FromFile(filePath string) ReadCloser { 158 | f, err := os.Open(filePath) 159 | if err != nil { 160 | return &errReadCloser{err} 161 | } 162 | return f 163 | } 164 | 165 | // FromBytes returns a ReadCloser from the given byte content. 166 | func FromBytes(content []byte) ReadCloser { 167 | return ioutil.NopCloser(bytes.NewReader(content)) 168 | } 169 | -------------------------------------------------------------------------------- /cmd/sample-gitops/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "time" 10 | 11 | "github.com/fluxcd/go-git-providers/github" 12 | "github.com/fluxcd/go-git-providers/gitprovider" 13 | "github.com/labstack/echo" 14 | homedir "github.com/mitchellh/go-homedir" 15 | "github.com/sirupsen/logrus" 16 | "github.com/spf13/pflag" 17 | "github.com/weaveworks/libgitops/cmd/common" 18 | "github.com/weaveworks/libgitops/cmd/sample-app/apis/sample/scheme" 19 | "github.com/weaveworks/libgitops/pkg/gitdir" 20 | "github.com/weaveworks/libgitops/pkg/logs" 21 | "github.com/weaveworks/libgitops/pkg/storage" 22 | "github.com/weaveworks/libgitops/pkg/storage/transaction" 23 | githubpr "github.com/weaveworks/libgitops/pkg/storage/transaction/pullrequest/github" 24 | "github.com/weaveworks/libgitops/pkg/storage/watch" 25 | "github.com/weaveworks/libgitops/pkg/storage/watch/update" 26 | ) 27 | 28 | var ( 29 | identityFlag = pflag.String("identity-file", "", "Path to where the SSH private key is") 30 | authorNameFlag = pflag.String("author-name", defaultAuthorName, "Author name for Git commits") 31 | authorEmailFlag = pflag.String("author-email", defaultAuthorEmail, "Author email for Git commits") 32 | gitURLFlag = pflag.String("git-url", "", "HTTPS Git URL; where the Git repository is, e.g. https://github.com/luxas/ignite-gitops") 33 | prAssigneeFlag = pflag.StringSlice("pr-assignees", nil, "What user logins to assign for the created PR. The user must have pull access to the repo.") 34 | prMilestoneFlag = pflag.String("pr-milestone", "", "What milestone to tag the PR with") 35 | ) 36 | 37 | const ( 38 | sshKnownHostsFile = "~/.ssh/known_hosts" 39 | 40 | defaultAuthorName = "Weave libgitops" 41 | defaultAuthorEmail = "support@weave.works" 42 | ) 43 | 44 | func main() { 45 | // Parse the version flag 46 | common.ParseVersionFlag() 47 | 48 | // Run the application 49 | if err := run(*identityFlag, *gitURLFlag, os.Getenv("GITHUB_TOKEN"), *authorNameFlag, *authorEmailFlag); err != nil { 50 | fmt.Println(err) 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | func expandAndRead(filePath string) ([]byte, error) { 56 | expandedPath, err := homedir.Expand(filePath) 57 | if err != nil { 58 | return nil, err 59 | } 60 | return ioutil.ReadFile(expandedPath) 61 | } 62 | 63 | func run(identityFile, gitURL, ghToken, authorName, authorEmail string) error { 64 | // Validate parameters 65 | if len(identityFile) == 0 { 66 | return fmt.Errorf("--identity-file is required") 67 | } 68 | if len(gitURL) == 0 { 69 | return fmt.Errorf("--git-url is required") 70 | } 71 | if len(ghToken) == 0 { 72 | return fmt.Errorf("--github-token is required") 73 | } 74 | if len(authorName) == 0 { 75 | return fmt.Errorf("--author-name is required") 76 | } 77 | if len(authorEmail) == 0 { 78 | return fmt.Errorf("--author-email is required") 79 | } 80 | 81 | // Read the identity and known_hosts files 82 | identityContent, err := expandAndRead(identityFile) 83 | if err != nil { 84 | return err 85 | } 86 | knownHostsContent, err := expandAndRead(sshKnownHostsFile) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | // Parse the HTTPS clone URL 92 | repoRef, err := gitprovider.ParseOrgRepositoryURL(gitURL) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | // Create a new GitHub client using the given token 98 | ghClient, err := github.NewClient(github.WithOAuth2Token(ghToken)) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | // Authenticate to the GitDirectory using Git SSH 104 | authMethod, err := gitdir.NewSSHAuthMethod(identityContent, knownHostsContent) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | // Construct the GitDirectory implementation which backs the storage 110 | gitDir, err := gitdir.NewGitDirectory(repoRef, gitdir.GitDirectoryOptions{ 111 | Branch: "master", 112 | Interval: 10 * time.Second, 113 | AuthMethod: authMethod, 114 | }) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | // Create a new PR provider for the GitStorage 120 | prProvider, err := githubpr.NewGitHubPRProvider(ghClient) 121 | if err != nil { 122 | return err 123 | } 124 | // Create a new GitStorage using the GitDirectory, PR provider, and Serializer 125 | gitStorage, err := transaction.NewGitStorage(gitDir, prProvider, scheme.Serializer) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | // Set the log level 131 | logs.Logger.SetLevel(logrus.InfoLevel) 132 | 133 | watchStorage, err := watch.NewManifestStorage(gitDir.Dir(), scheme.Serializer) 134 | if err != nil { 135 | return err 136 | } 137 | defer func() { _ = watchStorage.Close() }() 138 | 139 | updates := make(chan update.Update, 4096) 140 | watchStorage.SetUpdateStream(updates) 141 | 142 | go func() { 143 | for upd := range updates { 144 | logrus.Infof("Got %s update for: %v %v", upd.Event, upd.PartialObject.GetObjectKind().GroupVersionKind(), upd.PartialObject.GetObjectMeta()) 145 | } 146 | }() 147 | 148 | e := common.NewEcho() 149 | 150 | e.GET("/git/", func(c echo.Context) error { 151 | objs, err := gitStorage.List(storage.NewKindKey(common.CarGVK)) 152 | if err != nil { 153 | return err 154 | } 155 | return c.JSON(http.StatusOK, objs) 156 | }) 157 | 158 | e.PUT("/git/:name", func(c echo.Context) error { 159 | name := c.Param("name") 160 | if len(name) == 0 { 161 | return echo.NewHTTPError(http.StatusBadRequest, "Please set name") 162 | } 163 | 164 | objKey := common.CarKeyForName(name) 165 | err := gitStorage.Transaction(context.Background(), fmt.Sprintf("%s-update-", name), func(ctx context.Context, s storage.Storage) (transaction.CommitResult, error) { 166 | 167 | // Update the status of the car 168 | if err := common.SetNewCarStatus(s, objKey); err != nil { 169 | return nil, err 170 | } 171 | 172 | return &transaction.GenericPullRequestResult{ 173 | CommitResult: &transaction.GenericCommitResult{ 174 | AuthorName: authorName, 175 | AuthorEmail: authorEmail, 176 | Title: "Update Car speed", 177 | Description: "We really need to sync this state!", 178 | }, 179 | Labels: []string{"user/bot", "actuator/libgitops", "kind/status-update"}, 180 | Assignees: *prAssigneeFlag, 181 | Milestone: *prMilestoneFlag, 182 | }, nil 183 | }) 184 | if err != nil { 185 | return err 186 | } 187 | 188 | return c.String(200, "OK!") 189 | }) 190 | 191 | return common.StartEcho(e) 192 | } 193 | -------------------------------------------------------------------------------- /pkg/storage/sync/storage.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | /* 4 | 5 | TODO: Revisit if we need this file/package in the future. 6 | 7 | import ( 8 | "fmt" 9 | 10 | log "github.com/sirupsen/logrus" 11 | "github.com/weaveworks/libgitops/pkg/runtime" 12 | "github.com/weaveworks/libgitops/pkg/storage" 13 | "github.com/weaveworks/libgitops/pkg/storage/watch" 14 | "github.com/weaveworks/libgitops/pkg/storage/watch/update" 15 | "github.com/weaveworks/libgitops/pkg/util/sync" 16 | ) 17 | 18 | const updateBuffer = 4096 // How many updates to buffer, 4096 should be enough for even a high update frequency 19 | 20 | // SyncStorage is a Storage implementation taking in multiple Storages and 21 | // keeping them in sync. Any write operation executed on the SyncStorage 22 | // is propagated to all of the Storages it manages (including the embedded 23 | // one). For any retrieval or generation operation, the embedded Storage 24 | // will be used (it is treated as read-write). As all other Storages only 25 | // receive write operations, they can be thought of as write-only. 26 | type SyncStorage struct { 27 | storage.Storage 28 | storages []storage.Storage 29 | inboundStream update.UpdateStream 30 | outboundStream update.UpdateStream 31 | monitor *sync.Monitor 32 | } 33 | 34 | // SyncStorage implements update.EventStorage. 35 | var _ update.EventStorage = &SyncStorage{} 36 | 37 | // NewSyncStorage constructs a new SyncStorage 38 | func NewSyncStorage(rwStorage storage.Storage, wStorages ...storage.Storage) storage.Storage { 39 | ss := &SyncStorage{ 40 | Storage: rwStorage, 41 | storages: append(wStorages, rwStorage), 42 | } 43 | 44 | for _, s := range ss.storages { 45 | if watchStorage, ok := s.(watch.WatchStorage); ok { 46 | // Populate eventStream if we found a watchstorage 47 | if ss.inboundStream == nil { 48 | ss.inboundStream = make(update.UpdateStream, updateBuffer) 49 | } 50 | watchStorage.SetUpdateStream(ss.inboundStream) 51 | } 52 | } 53 | 54 | if ss.inboundStream != nil { 55 | ss.monitor = sync.RunMonitor(ss.monitorFunc) 56 | ss.outboundStream = make(update.UpdateStream, updateBuffer) 57 | } 58 | 59 | return ss 60 | } 61 | 62 | // Set is propagated to all Storages 63 | func (ss *SyncStorage) Set(obj runtime.Object) error { 64 | return ss.runAll(func(s storage.Storage) error { 65 | return s.Set(obj) 66 | }) 67 | } 68 | 69 | // Patch is propagated to all Storages 70 | func (ss *SyncStorage) Patch(key storage.ObjectKey, patch []byte) error { 71 | return ss.runAll(func(s storage.Storage) error { 72 | return s.Patch(key, patch) 73 | }) 74 | } 75 | 76 | // Delete is propagated to all Storages 77 | func (ss *SyncStorage) Delete(key storage.ObjectKey) error { 78 | return ss.runAll(func(s storage.Storage) error { 79 | return s.Delete(key) 80 | }) 81 | } 82 | 83 | func (ss *SyncStorage) Close() error { 84 | // Close all WatchStorages 85 | for _, s := range ss.storages { 86 | if watchStorage, ok := s.(watch.WatchStorage); ok { 87 | _ = watchStorage.Close() 88 | } 89 | } 90 | 91 | // Close the event streams if set 92 | if ss.inboundStream != nil { 93 | close(ss.inboundStream) 94 | } 95 | if ss.outboundStream != nil { 96 | close(ss.outboundStream) 97 | } 98 | // Wait for the monitor goroutine 99 | ss.monitor.Wait() 100 | return nil 101 | } 102 | 103 | func (ss *SyncStorage) GetUpdateStream() update.UpdateStream { 104 | return ss.outboundStream 105 | } 106 | 107 | // runAll runs the given function for all Storages in parallel and aggregates all errors 108 | func (ss *SyncStorage) runAll(f func(storage.Storage) error) (err error) { 109 | type result struct { 110 | int 111 | error 112 | } 113 | 114 | errC := make(chan result) 115 | for i, s := range ss.storages { 116 | go func(i int, s storage.Storage) { 117 | errC <- result{i, f(s)} 118 | }(i, s) // NOTE: This requires i and s as arguments, otherwise they will be evaluated for one Storage only 119 | } 120 | 121 | for i := 0; i < len(ss.storages); i++ { 122 | if result := <-errC; result.error != nil { 123 | if err == nil { 124 | err = fmt.Errorf("SyncStorage: Error in Storage %d: %v", result.int, result.error) 125 | } else { 126 | err = fmt.Errorf("%v\n%29s %d: %v", err, "and error in Storage", result.int, result.error) 127 | } 128 | } 129 | } 130 | 131 | return 132 | } 133 | 134 | func (ss *SyncStorage) monitorFunc() { 135 | log.Debug("SyncStorage: Monitoring thread started") 136 | defer log.Debug("SyncStorage: Monitoring thread stopped") 137 | 138 | // TODO: Support detecting changes done when the GitOps daemon isn't running 139 | // This is difficult to do though, as we have don't know which state is the latest 140 | // For now, only update the state on write when the daemon is running 141 | for { 142 | upd, ok := <-ss.inboundStream 143 | if ok { 144 | log.Debugf("SyncStorage: Received update %v %t", upd, ok) 145 | 146 | gvk := upd.PartialObject.GetObjectKind().GroupVersionKind() 147 | uid := upd.PartialObject.GetUID() 148 | key := storage.NewObjectKey(storage.NewKindKey(gvk), runtime.NewIdentifier(string(uid))) 149 | log.Debugf("SyncStorage: Object has gvk=%q and uid=%q", gvk, uid) 150 | 151 | switch upd.Event { 152 | case update.ObjectEventModify, update.ObjectEventCreate: 153 | // First load the Object using the Storage given in the update, 154 | // then set it using the client constructed above 155 | 156 | obj, err := upd.Storage.Get(key) 157 | if err != nil { 158 | log.Errorf("Failed to get Object with UID %q: %v", upd.PartialObject.GetUID(), err) 159 | continue 160 | } 161 | 162 | if err = ss.Set(obj); err != nil { 163 | log.Errorf("Failed to set Object with UID %q: %v", upd.PartialObject.GetUID(), err) 164 | continue 165 | } 166 | case update.ObjectEventDelete: 167 | // For deletion we use the generated "fake" APIType object 168 | if err := ss.Delete(key); err != nil { 169 | log.Errorf("Failed to delete Object with UID %q: %v", upd.PartialObject.GetUID(), err) 170 | continue 171 | } 172 | } 173 | 174 | // Send the update to the listeners unless the channel is full, 175 | // in which case issue a warning. The channel can hold as many 176 | // updates as updateBuffer specifies. 177 | select { 178 | case ss.outboundStream <- upd: 179 | log.Debugf("SyncStorage: Sent update: %v", upd) 180 | default: 181 | log.Warn("SyncStorage: Failed to send update, channel full") 182 | } 183 | } else { 184 | return 185 | } 186 | } 187 | } 188 | */ 189 | -------------------------------------------------------------------------------- /pkg/storage/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | /* 4 | 5 | TODO: Revisit if we need this file/package in the future. 6 | 7 | import ( 8 | log "github.com/sirupsen/logrus" 9 | "github.com/weaveworks/libgitops/pkg/runtime" 10 | "github.com/weaveworks/libgitops/pkg/serializer" 11 | "github.com/weaveworks/libgitops/pkg/storage" 12 | "k8s.io/apimachinery/pkg/runtime/schema" 13 | ) 14 | 15 | // Cache is an intermediate caching layer, which conforms to Storage 16 | // Typically you back the cache with an actual storage 17 | type Cache interface { 18 | storage.Storage 19 | // Flush is used to write the state of the entire cache to storage 20 | // Warning: this is a very expensive operation 21 | Flush() error 22 | } 23 | 24 | type cache struct { 25 | // storage is the backing Storage for the cache 26 | // used to look up non-cached Objects 27 | storage storage.Storage 28 | 29 | // index caches the Objects by GroupVersionKind and UID 30 | // This guarantees uniqueness when looking up a specific Object 31 | index *index 32 | } 33 | 34 | var _ Cache = &cache{} 35 | 36 | func NewCache(backingStorage storage.Storage) Cache { 37 | c := &cache{ 38 | storage: backingStorage, 39 | index: newIndex(backingStorage), 40 | } 41 | 42 | return c 43 | } 44 | 45 | func (s *cache) Serializer() serializer.Serializer { 46 | return s.storage.Serializer() 47 | } 48 | 49 | func (c *cache) New(gvk schema.GroupVersionKind) (runtime.Object, error) { 50 | // Request the storage to create the Object. The 51 | // newly generated Object has not got an UID which 52 | // is required for indexing, so just return it 53 | // without storing it into the cache 54 | return c.storage.New(gvk) 55 | } 56 | 57 | func (c *cache) Get(gvk schema.GroupVersionKind, uid runtime.UID) (obj runtime.Object, err error) { 58 | log.Tracef("cache: Get %s with UID %q", gvk.Kind, uid) 59 | 60 | // If the requested Object resides in the cache, return it 61 | if obj, err = c.index.loadByID(gvk, uid); err != nil || obj != nil { 62 | return 63 | } 64 | 65 | // Request the Object from the storage 66 | obj, err = c.storage.Get(gvk, uid) 67 | 68 | // If no errors occurred, cache it 69 | if err == nil { 70 | err = c.index.store(obj) 71 | } 72 | 73 | return 74 | } 75 | 76 | func (c *cache) GetMeta(gvk schema.GroupVersionKind, uid runtime.UID) (obj runtime.Object, err error) { 77 | log.Tracef("cache: GetMeta %s with UID %q", gvk.Kind, uid) 78 | 79 | obj, err = c.storage.GetMeta(gvk, uid) 80 | 81 | // If no errors occurred while loading, store the Object in the cache 82 | if err == nil { 83 | err = c.index.storeMeta(obj) 84 | } 85 | 86 | return 87 | } 88 | 89 | func (c *cache) Set(gvk schema.GroupVersionKind, obj runtime.Object) error { 90 | log.Tracef("cache: Set %s with UID %q", gvk.Kind, obj.GetUID()) 91 | 92 | // Store the changed Object in the cache 93 | if err := c.index.store(obj); err != nil { 94 | return err 95 | } 96 | 97 | // TODO: For now the cache always flushes, we might add automatic flushing later 98 | return c.storage.Set(gvk, obj) 99 | } 100 | 101 | func (c *cache) Patch(gvk schema.GroupVersionKind, uid runtime.UID, patch []byte) error { 102 | // TODO: For now patches are always flushed, the cache will load the updated Object on-demand on access 103 | return c.storage.Patch(gvk, uid, patch) 104 | } 105 | 106 | func (c *cache) Delete(gvk schema.GroupVersionKind, uid runtime.UID) error { 107 | log.Tracef("cache: Delete %s with UID %q", gvk.Kind, uid) 108 | 109 | // Delete the given Object from the cache and storage 110 | c.index.delete(gvk, uid) 111 | return c.storage.Delete(gvk, uid) 112 | } 113 | 114 | type listFunc func(gvk schema.GroupVersionKind) ([]runtime.Object, error) 115 | type cacheStoreFunc func([]runtime.Object) error 116 | 117 | // list is a common handler for List and ListMeta 118 | func (c *cache) list(gvk schema.GroupVersionKind, slf, clf listFunc, csf cacheStoreFunc) (objs []runtime.Object, err error) { 119 | var storageCount uint64 120 | if storageCount, err = c.storage.Count(gvk); err != nil { 121 | return 122 | } 123 | 124 | if c.index.count(gvk) != storageCount { 125 | log.Tracef("cache: miss when listing: %s", gvk) 126 | // If the cache doesn't track all of the Objects, request them from the storage 127 | if objs, err = slf(gvk); err != nil { 128 | // If no errors occurred, store the Objects in the cache 129 | err = csf(objs) 130 | } 131 | } else { 132 | log.Tracef("cache: hit when listing: %s", gvk) 133 | // If the cache tracks everything, return the cache's contents 134 | objs, err = clf(gvk) 135 | } 136 | 137 | return 138 | } 139 | 140 | func (c *cache) List(gvk schema.GroupVersionKind) ([]runtime.Object, error) { 141 | return c.list(gvk, c.storage.List, c.index.list, c.index.storeAll) 142 | } 143 | 144 | func (c *cache) ListMeta(gvk schema.GroupVersionKind) ([]runtime.Object, error) { 145 | return c.list(gvk, c.storage.ListMeta, c.index.listMeta, c.index.storeAllMeta) 146 | } 147 | 148 | func (c *cache) Count(gvk schema.GroupVersionKind) (uint64, error) { 149 | // The cache is transparent about how many items it has cached 150 | return c.storage.Count(gvk) 151 | } 152 | 153 | func (c *cache) Checksum(gvk schema.GroupVersionKind, uid runtime.UID) (string, error) { 154 | // The cache is transparent about the checksums 155 | return c.storage.Checksum(gvk, uid) 156 | } 157 | 158 | func (c *cache) RawStorage() storage.RawStorage { 159 | return c.storage.RawStorage() 160 | } 161 | 162 | func (c *cache) Close() error { 163 | return c.storage.Close() 164 | } 165 | 166 | func (c *cache) Flush() error { 167 | // Load the entire cache 168 | allObjects, err := c.index.loadAll() 169 | if err != nil { 170 | return err 171 | } 172 | 173 | for _, obj := range allObjects { 174 | // Request the storage to save each Object 175 | if err := c.storage.Set(obj); err != nil { 176 | return err 177 | } 178 | } 179 | 180 | return nil 181 | } 182 | 183 | // PartialObjectFrom is used to create a bound PartialObjectImpl from an Object. 184 | // Note: This might be useful later (maybe here or maybe in pkg/runtime) if re-enable the cache 185 | func PartialObjectFrom(obj Object) (PartialObject, error) { 186 | tm, ok := obj.GetObjectKind().(*metav1.TypeMeta) 187 | if !ok { 188 | return nil, fmt.Errorf("PartialObjectFrom: Cannot cast obj to *metav1.TypeMeta, is %T", obj.GetObjectKind()) 189 | } 190 | om, ok := obj.GetObjectMeta().(*metav1.ObjectMeta) 191 | if !ok { 192 | return nil, fmt.Errorf("PartialObjectFrom: Cannot cast obj to *metav1.ObjectMeta, is %T", obj.GetObjectMeta()) 193 | } 194 | return &PartialObjectImpl{tm, om}, nil 195 | } 196 | 197 | */ 198 | -------------------------------------------------------------------------------- /pkg/storage/rawstorage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/weaveworks/libgitops/pkg/runtime" 13 | "github.com/weaveworks/libgitops/pkg/serializer" 14 | "github.com/weaveworks/libgitops/pkg/util" 15 | "k8s.io/apimachinery/pkg/runtime/schema" 16 | ) 17 | 18 | // RawStorage is a Key-indexed low-level interface to 19 | // store byte-encoded Objects (resources) in non-volatile 20 | // memory. 21 | type RawStorage interface { 22 | // Read returns a resource's content based on key. 23 | // If the resource does not exist, it returns ErrNotFound. 24 | Read(key ObjectKey) ([]byte, error) 25 | // Exists checks if the resource indicated by key exists. 26 | Exists(key ObjectKey) bool 27 | // Write writes the given content to the resource indicated by key. 28 | // Error returns are implementation-specific. 29 | Write(key ObjectKey, content []byte) error 30 | // Delete deletes the resource indicated by key. 31 | // If the resource does not exist, it returns ErrNotFound. 32 | Delete(key ObjectKey) error 33 | // List returns all matching object keys based on the given KindKey. 34 | List(key KindKey) ([]ObjectKey, error) 35 | // Checksum returns a string checksum for the resource indicated by key. 36 | // If the resource does not exist, it returns ErrNotFound. 37 | Checksum(key ObjectKey) (string, error) 38 | // ContentType returns the content type of the contents of the resource indicated by key. 39 | ContentType(key ObjectKey) serializer.ContentType 40 | 41 | // WatchDir returns the path for Watchers to watch changes in. 42 | WatchDir() string 43 | // GetKey retrieves the Key containing the virtual path based 44 | // on the given physical file path returned by a Watcher. 45 | GetKey(path string) (ObjectKey, error) 46 | } 47 | 48 | func NewGenericRawStorage(dir string, gv schema.GroupVersion, ct serializer.ContentType) RawStorage { 49 | ext := extForContentType(ct) 50 | if ext == "" { 51 | panic("Invalid content type") 52 | } 53 | return &GenericRawStorage{ 54 | dir: dir, 55 | gv: gv, 56 | ct: ct, 57 | ext: ext, 58 | } 59 | } 60 | 61 | // GenericRawStorage is a rawstorage which stores objects as JSON files on disk, 62 | // in the form: ///metadata.json. 63 | // The GenericRawStorage only supports one GroupVersion at a time, and will error if given 64 | // any other resources 65 | type GenericRawStorage struct { 66 | dir string 67 | gv schema.GroupVersion 68 | ct serializer.ContentType 69 | ext string 70 | } 71 | 72 | func (r *GenericRawStorage) keyPath(key ObjectKey) string { 73 | return path.Join(r.dir, key.GetKind(), key.GetIdentifier(), fmt.Sprintf("metadata%s", r.ext)) 74 | } 75 | 76 | func (r *GenericRawStorage) kindKeyPath(kindKey KindKey) string { 77 | return path.Join(r.dir, kindKey.GetKind()) 78 | } 79 | 80 | func (r *GenericRawStorage) validateGroupVersion(kind KindKey) error { 81 | if r.gv.Group == kind.GetGroup() && r.gv.Version == kind.GetVersion() { 82 | return nil 83 | } 84 | 85 | return fmt.Errorf("GroupVersion %s/%s not supported by this GenericRawStorage", kind.GetGroup(), kind.GetVersion()) 86 | } 87 | 88 | func (r *GenericRawStorage) Read(key ObjectKey) ([]byte, error) { 89 | // Validate GroupVersion first 90 | if err := r.validateGroupVersion(key); err != nil { 91 | return nil, err 92 | } 93 | 94 | // Check if the resource indicated by key exists 95 | if !r.Exists(key) { 96 | return nil, ErrNotFound 97 | } 98 | 99 | return ioutil.ReadFile(r.keyPath(key)) 100 | } 101 | 102 | func (r *GenericRawStorage) Exists(key ObjectKey) bool { 103 | // Validate GroupVersion first 104 | if err := r.validateGroupVersion(key); err != nil { 105 | return false 106 | } 107 | 108 | return util.FileExists(r.keyPath(key)) 109 | } 110 | 111 | func (r *GenericRawStorage) Write(key ObjectKey, content []byte) error { 112 | // Validate GroupVersion first 113 | if err := r.validateGroupVersion(key); err != nil { 114 | return err 115 | } 116 | 117 | file := r.keyPath(key) 118 | 119 | // Create the underlying directories if they do not exist already 120 | if !r.Exists(key) { 121 | if err := os.MkdirAll(path.Dir(file), 0755); err != nil { 122 | return err 123 | } 124 | } 125 | 126 | return ioutil.WriteFile(file, content, 0644) 127 | } 128 | 129 | func (r *GenericRawStorage) Delete(key ObjectKey) error { 130 | // Validate GroupVersion first 131 | if err := r.validateGroupVersion(key); err != nil { 132 | return err 133 | } 134 | 135 | // Check if the resource indicated by key exists 136 | if !r.Exists(key) { 137 | return ErrNotFound 138 | } 139 | 140 | return os.RemoveAll(path.Dir(r.keyPath(key))) 141 | } 142 | 143 | func (r *GenericRawStorage) List(kind KindKey) ([]ObjectKey, error) { 144 | // Validate GroupVersion first 145 | if err := r.validateGroupVersion(kind); err != nil { 146 | return nil, err 147 | } 148 | 149 | entries, err := ioutil.ReadDir(r.kindKeyPath(kind)) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | result := make([]ObjectKey, 0, len(entries)) 155 | for _, entry := range entries { 156 | result = append(result, NewObjectKey(kind, runtime.NewIdentifier(entry.Name()))) 157 | } 158 | 159 | return result, nil 160 | } 161 | 162 | // This returns the modification time as a UnixNano string 163 | // If the file doesn't exist, return ErrNotFound 164 | func (r *GenericRawStorage) Checksum(key ObjectKey) (string, error) { 165 | // Validate GroupVersion first 166 | if err := r.validateGroupVersion(key); err != nil { 167 | return "", err 168 | } 169 | 170 | // Check if the resource indicated by key exists 171 | if !r.Exists(key) { 172 | return "", ErrNotFound 173 | } 174 | 175 | return checksumFromModTime(r.keyPath(key)) 176 | } 177 | 178 | func (r *GenericRawStorage) ContentType(_ ObjectKey) serializer.ContentType { 179 | return r.ct 180 | } 181 | 182 | func (r *GenericRawStorage) WatchDir() string { 183 | return r.dir 184 | } 185 | 186 | func (r *GenericRawStorage) GetKey(p string) (ObjectKey, error) { 187 | splitDir := strings.Split(filepath.Clean(r.dir), string(os.PathSeparator)) 188 | splitPath := strings.Split(filepath.Clean(p), string(os.PathSeparator)) 189 | 190 | if len(splitPath) < len(splitDir)+2 { 191 | return nil, fmt.Errorf("path not long enough: %s", p) 192 | } 193 | 194 | for i := 0; i < len(splitDir); i++ { 195 | if splitDir[i] != splitPath[i] { 196 | return nil, fmt.Errorf("path has wrong base: %s", p) 197 | } 198 | } 199 | kind := splitPath[len(splitDir)] 200 | uid := splitPath[len(splitDir)+1] 201 | gvk := schema.GroupVersionKind{ 202 | Group: r.gv.Group, 203 | Version: r.gv.Version, 204 | Kind: kind, 205 | } 206 | 207 | return NewObjectKey(NewKindKey(gvk), runtime.NewIdentifier(uid)), nil 208 | } 209 | 210 | func checksumFromModTime(path string) (string, error) { 211 | fi, err := os.Stat(path) 212 | if err != nil { 213 | return "", err 214 | } 215 | 216 | return strconv.FormatInt(fi.ModTime().UnixNano(), 10), nil 217 | } 218 | -------------------------------------------------------------------------------- /pkg/serializer/comments/comments_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Kubernetes Authors. 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // This package provides a means to copy over comments between 5 | // two kyaml/yaml.RNode trees. This code is derived from 6 | // the sigs.k8s.io/kustomize/kyaml/comments package, at revision 7 | // 600d4f2c0bf174abd76d03e49939ee0c34b2a019. 8 | // 9 | // It has been slightly modified and adapted to not lose any 10 | // comment from the old tree, although the node the comment is 11 | // attached to doesn't exist in the new tree. To solve this, 12 | // this package moves any such comments to the beginning of the 13 | // file. 14 | // This file is a temporary means as long as we're waiting for 15 | // these code changes to get upstreamed to its origin, the kustomize repo. 16 | // https://pkg.go.dev/sigs.k8s.io/kustomize/kyaml/comments?tab=doc#CopyComments 17 | 18 | package comments 19 | 20 | import ( 21 | "strings" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/assert" 25 | "sigs.k8s.io/kustomize/kyaml/yaml" 26 | ) 27 | 28 | func TestCopyComments(t *testing.T) { 29 | testCases := []struct { 30 | name string 31 | from string 32 | to string 33 | expected string 34 | }{ 35 | { 36 | name: "copy_comments", 37 | from: ` 38 | # A 39 | # 40 | # B 41 | 42 | # C 43 | apiVersion: apps/v1 44 | kind: Deployment 45 | spec: # comment 1 46 | # comment 2 47 | replicas: 3 # comment 3 48 | # comment 4 49 | `, 50 | to: ` 51 | apiVersion: apps/v1 52 | kind: Deployment 53 | spec: 54 | replicas: 4 55 | `, 56 | expected: ` 57 | # A 58 | # 59 | # B 60 | 61 | # C 62 | apiVersion: apps/v1 63 | kind: Deployment 64 | spec: # comment 1 65 | # comment 2 66 | replicas: 4 # comment 3 67 | # comment 4 68 | `, 69 | }, { 70 | name: "associative_list", 71 | from: ` 72 | apiVersion: apps/v1 73 | kind: Deployment 74 | spec: 75 | template: 76 | spec: 77 | containers: 78 | - name: foo 79 | image: bar # comment 1 80 | `, 81 | to: ` 82 | apiVersion: apps/v1 83 | kind: Deployment 84 | spec: 85 | template: 86 | spec: 87 | containers: 88 | - name: foo 89 | image: bar 90 | `, 91 | expected: ` 92 | apiVersion: apps/v1 93 | kind: Deployment 94 | spec: 95 | template: 96 | spec: 97 | containers: 98 | - name: foo 99 | image: bar # comment 1 100 | `, 101 | }, { 102 | name: "keep_comments", 103 | from: ` 104 | # A 105 | # 106 | # B 107 | 108 | # C 109 | apiVersion: apps/v1 110 | kind: Deployment 111 | spec: # comment 1 112 | # comment 2 113 | replicas: 3 # comment 3 114 | # comment 4 115 | `, 116 | to: ` 117 | apiVersion: apps/v1 118 | kind: Deployment 119 | spec: 120 | replicas: 4 # comment 5 121 | `, 122 | expected: ` 123 | # A 124 | # 125 | # B 126 | 127 | # C 128 | apiVersion: apps/v1 129 | kind: Deployment 130 | spec: # comment 1 131 | # comment 2 132 | replicas: 4 # comment 5 133 | # comment 4 134 | `, 135 | }, { 136 | name: "copy_item_comments", 137 | from: ` 138 | apiVersion: apps/v1 139 | kind: Deployment 140 | items: 141 | - a # comment 142 | `, 143 | to: ` 144 | apiVersion: apps/v1 145 | kind: Deployment 146 | items: 147 | - a 148 | `, 149 | expected: ` 150 | apiVersion: apps/v1 151 | kind: Deployment 152 | items: 153 | - a # comment 154 | `, 155 | }, { 156 | name: "copy_item_comments_2", 157 | from: ` 158 | apiVersion: apps/v1 159 | kind: Deployment 160 | items: 161 | # comment 162 | - a 163 | `, 164 | to: ` 165 | apiVersion: apps/v1 166 | kind: Deployment 167 | items: 168 | - a 169 | `, 170 | expected: ` 171 | apiVersion: apps/v1 172 | kind: Deployment 173 | items: 174 | # comment 175 | - a 176 | `, 177 | }, { 178 | name: "copy_item_comments_middle", 179 | from: ` 180 | apiVersion: apps/v1 181 | kind: Deployment 182 | items: 183 | - a 184 | - b # comment 185 | - c 186 | `, 187 | to: ` 188 | apiVersion: apps/v1 189 | kind: Deployment 190 | items: 191 | - d 192 | - b 193 | - e 194 | `, 195 | expected: ` 196 | apiVersion: apps/v1 197 | kind: Deployment 198 | items: 199 | - d 200 | - b # comment 201 | - e 202 | `, 203 | }, { 204 | name: "copy_item_comments_moved", 205 | from: ` 206 | apiVersion: apps/v1 207 | kind: Deployment 208 | items: 209 | - a 210 | - b # comment 211 | - c 212 | `, 213 | to: ` 214 | apiVersion: apps/v1 215 | kind: Deployment 216 | items: 217 | - a 218 | - c 219 | - b 220 | `, 221 | expected: ` 222 | apiVersion: apps/v1 223 | kind: Deployment 224 | items: 225 | - a 226 | - c 227 | - b 228 | `, 229 | }, { 230 | name: "copy_item_comments_no_match", 231 | from: ` 232 | apiVersion: apps/v1 233 | kind: Deployment 234 | items: 235 | - a # comment 236 | `, 237 | to: ` 238 | apiVersion: apps/v1 239 | kind: Deployment 240 | items: 241 | - b 242 | `, 243 | expected: ` 244 | apiVersion: apps/v1 245 | kind: Deployment 246 | items: 247 | - b 248 | `, 249 | }, { 250 | name: "copy_item_comments_add", 251 | from: ` 252 | apiVersion: apps/v1 253 | kind: Deployment 254 | items: 255 | - a # comment 256 | `, 257 | to: ` 258 | apiVersion: apps/v1 259 | kind: Deployment 260 | items: 261 | - a 262 | - b 263 | `, 264 | expected: ` 265 | apiVersion: apps/v1 266 | kind: Deployment 267 | items: 268 | - a # comment 269 | - b 270 | `, 271 | }, { 272 | name: "copy_item_comments_remove", 273 | from: ` 274 | apiVersion: apps/v1 275 | kind: Deployment 276 | items: 277 | - a # comment 278 | - b 279 | `, 280 | to: ` 281 | apiVersion: apps/v1 282 | kind: Deployment 283 | items: 284 | - a 285 | `, 286 | expected: ` 287 | apiVersion: apps/v1 288 | kind: Deployment 289 | items: 290 | - a # comment 291 | `, 292 | }, { 293 | name: "copy_comments_folded_style", 294 | from: ` 295 | apiVersion: v1 296 | kind: ConfigMap 297 | data: 298 | somekey: "012345678901234567890123456789012345678901234567890123456789012345678901234" # x 299 | `, 300 | to: ` 301 | apiVersion: v1 302 | kind: ConfigMap 303 | data: 304 | somekey: >- 305 | 012345678901234567890123456789012345678901234567890123456789012345678901234 306 | `, 307 | expected: ` 308 | apiVersion: v1 309 | kind: ConfigMap 310 | data: 311 | somekey: "012345678901234567890123456789012345678901234567890123456789012345678901234" # x 312 | `, 313 | }, { 314 | name: "copy_comments_move_to_top", 315 | from: ` 316 | # Top comment 317 | 318 | apiVersion: v1 319 | kind: ConfigMap # Foo 320 | # Bar 321 | data: 322 | # Baz 323 | somekey: "012345678901234567890123456789012345678901234567890123456789012345678901234" # x 324 | `, 325 | to: ` 326 | apiVersion: v1 327 | `, 328 | expected: ` 329 | # Top comment 330 | # Comments lost during file manipulation: 331 | # Field "data": "Bar" 332 | # Field "somekey": "Baz" 333 | # Field "somekey": "x" 334 | # Field "kind": "Foo" 335 | 336 | apiVersion: v1 337 | `, 338 | }, 339 | } 340 | 341 | for i := range testCases { 342 | tc := testCases[i] 343 | t.Run(tc.name, func(t *testing.T) { 344 | from, err := yaml.Parse(tc.from) 345 | if !assert.NoError(t, err) { 346 | t.FailNow() 347 | } 348 | 349 | to, err := yaml.Parse(tc.to) 350 | if !assert.NoError(t, err) { 351 | t.FailNow() 352 | } 353 | 354 | err = CopyComments(from, to, true) 355 | if !assert.NoError(t, err) { 356 | t.FailNow() 357 | } 358 | 359 | actual, err := to.String() 360 | if !assert.NoError(t, err) { 361 | t.FailNow() 362 | } 363 | 364 | if !assert.Equal(t, strings.TrimSpace(tc.expected), strings.TrimSpace(actual)) { 365 | t.FailNow() 366 | } 367 | }) 368 | } 369 | } 370 | -------------------------------------------------------------------------------- /pkg/serializer/comments.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/weaveworks/libgitops/pkg/serializer/comments" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/apimachinery/pkg/runtime" 13 | "sigs.k8s.io/kustomize/kyaml/yaml" 14 | ) 15 | 16 | const preserveCommentsAnnotation = "serializer.libgitops.weave.works/original-data" 17 | 18 | var ( 19 | // TODO: Investigate if we can just depend on `metav1.Object` interface compliance instead of needing to explicitly 20 | // embed the `metav1.ObjectMeta` struct. 21 | ErrNoObjectMeta = errors.New("the given object cannot store comments, it is not metav1.ObjectMeta compliant") 22 | ErrNoStoredComments = errors.New("the given object does not have stored comments") 23 | ) 24 | 25 | // tryToPreserveComments tries to save the original file data (base64-encoded) into an annotation. 26 | // This original file data can be used at encoding-time to preserve comments 27 | func (d *decoder) tryToPreserveComments(doc []byte, obj runtime.Object, ct ContentType) { 28 | // If the user opted into preserving comments and the format is YAML, proceed 29 | // If they didn't, return directly 30 | if !(*d.opts.PreserveComments && ct == ContentTypeYAML) { 31 | return 32 | } 33 | 34 | // Preserve the original file content in the annotation (this requires embedding ObjectMeta). 35 | if !setCommentSourceBytes(obj, doc) { 36 | // If the object doesn't have ObjectMeta embedded, just do nothing. 37 | logrus.Debugf("Couldn't convert object with GVK %q to metav1.Object, although opts.PreserveComments is enabled", obj.GetObjectKind().GroupVersionKind()) 38 | } 39 | } 40 | 41 | // tryToPreserveComments tries to locate the possibly-saved original file data in the object's annotation 42 | func (e *encoder) encodeWithCommentSupport(versionEncoder runtime.Encoder, fw FrameWriter, obj runtime.Object, metaObj metav1.Object) error { 43 | // If the user did not opt into preserving comments, just sanitize ObjectMeta temporarily and and return 44 | if !*e.opts.PreserveComments { 45 | // Normal encoding without the annotation (so it doesn't leak by accident) 46 | return noAnnotationWrapper(metaObj, e.normalEncodeFunc(versionEncoder, fw, obj)) 47 | } 48 | 49 | // The user requested to preserve comments, but content type is not YAML, so log, sanitize and return 50 | if fw.ContentType() != ContentTypeYAML { 51 | logrus.Debugf("Asked to preserve comments, but ContentType is not YAML, so ignoring") 52 | 53 | // Normal encoding without the annotation (so it doesn't leak by accident) 54 | return noAnnotationWrapper(metaObj, e.normalEncodeFunc(versionEncoder, fw, obj)) 55 | } 56 | 57 | priorNode, err := getCommentSourceMeta(metaObj) 58 | if errors.Is(err, ErrNoStoredComments) { 59 | // No need to delete the annotation as we know it doesn't exist, just do a normal encode 60 | return e.normalEncodeFunc(versionEncoder, fw, obj)() 61 | } else if err != nil { 62 | return err 63 | } 64 | 65 | // Encode the new object into a temporary buffer, it should not be written as the "final result" to the FrameWriter 66 | buf := new(bytes.Buffer) 67 | if err := noAnnotationWrapper(metaObj, e.normalEncodeFunc(versionEncoder, NewYAMLFrameWriter(buf), obj)); err != nil { 68 | // fatal error 69 | return err 70 | } 71 | 72 | // Parse the new, upgraded, encoded YAML into *yaml.RNode for addition 73 | // of comments from prevNode 74 | afterNode, err := yaml.Parse(buf.String()) 75 | if err != nil { 76 | // fatal error 77 | return err 78 | } 79 | 80 | // Copy over comments from the old to the new schema 81 | if err := comments.CopyComments(priorNode, afterNode, true); err != nil { 82 | // fatal error 83 | return err 84 | } 85 | 86 | // Print the new schema with the old comments kept to the FrameWriter 87 | _, err = fmt.Fprint(fw, afterNode.MustString()) 88 | // we're done, exit the encode function 89 | return err 90 | } 91 | 92 | func (e *encoder) normalEncodeFunc(versionEncoder runtime.Encoder, fw FrameWriter, obj runtime.Object) func() error { 93 | return func() error { 94 | return versionEncoder.Encode(obj, fw) 95 | } 96 | } 97 | 98 | // noAnnotationWrapper temporarily removes the preserveComments annotation before and after running the function 99 | // one example of this function is e.normalEncodeFunc 100 | func noAnnotationWrapper(metaobj metav1.Object, fn func() error) error { 101 | // If the annotation exists, delete it and defer add it back. 102 | if val, ok := getAnnotation(metaobj, preserveCommentsAnnotation); ok { 103 | defer setAnnotation(metaobj, preserveCommentsAnnotation, val) 104 | deleteAnnotation(metaobj, preserveCommentsAnnotation) 105 | } 106 | // If the annotation isn't present, just run the function 107 | return fn() 108 | } 109 | 110 | // GetCommentSource retrieves the YAML tree used as the source for transferring comments for the given runtime.Object. 111 | // This may be used externally to implement e.g. re-parenting of the comment source tree when moving structs around. 112 | func GetCommentSource(obj runtime.Object) (*yaml.RNode, error) { 113 | // Cast the object to a metav1.Object to get access to annotations. 114 | // If this fails, the given object does not support storing comments. 115 | metaObj, ok := toMetaObject(obj) 116 | if !ok { 117 | return nil, ErrNoObjectMeta 118 | } 119 | 120 | // Use getCommentSourceMeta to retrieve the comments from the metav1.Object. 121 | return getCommentSourceMeta(metaObj) 122 | } 123 | 124 | // getCommentSourceMeta retrieves the YAML tree used as the source for transferring comments for the given metav1.Object. 125 | func getCommentSourceMeta(metaObj metav1.Object) (*yaml.RNode, error) { 126 | // Fetch the source string for the comments. If this fails, the given object does not have any stored comments. 127 | sourceStr, ok := getAnnotation(metaObj, preserveCommentsAnnotation) 128 | if !ok { 129 | return nil, ErrNoStoredComments 130 | } 131 | 132 | // Decode the base64-encoded comment source string. 133 | sourceBytes, err := base64.StdEncoding.DecodeString(sourceStr) 134 | if err != nil { 135 | return nil, err 136 | } 137 | 138 | // Parse the decoded source data into a *yaml.RNode and return it. 139 | return yaml.Parse(string(sourceBytes)) 140 | } 141 | 142 | // SetCommentSource sets the given YAML tree as the source for transferring comments for the given runtime.Object. 143 | // This may be used externally to implement e.g. re-parenting of the comment source tree when moving structs around. 144 | func SetCommentSource(obj runtime.Object, source *yaml.RNode) error { 145 | // Convert the given tree into a string. This also handles the source == nil case. 146 | str, err := source.String() 147 | if err != nil { 148 | return err 149 | } 150 | 151 | // Convert the string to bytes and pass it to setCommentSourceBytes to be applied. 152 | if !setCommentSourceBytes(obj, []byte(str)) { 153 | // If this fails, the passed object is not metav1.ObjectMeta compliant. 154 | return ErrNoObjectMeta 155 | } 156 | 157 | return nil 158 | } 159 | 160 | // SetCommentSource sets the given bytes as the source for transferring comments for the given runtime.Object. 161 | func setCommentSourceBytes(obj runtime.Object, source []byte) bool { 162 | // Cast the object to a metav1.Object to get access to annotations. 163 | // If this fails, the given object does not support storing comments. 164 | metaObj, ok := toMetaObject(obj) 165 | if !ok { 166 | return false 167 | } 168 | 169 | // base64-encode the comments string. 170 | encodedStr := base64.StdEncoding.EncodeToString(source) 171 | 172 | // Set the value of the comments annotation to the encoded string. 173 | setAnnotation(metaObj, preserveCommentsAnnotation, encodedStr) 174 | return true 175 | } 176 | --------------------------------------------------------------------------------