├── .gitattributes ├── .goreleaser.yml ├── .gitignore ├── manipmongo ├── backoff.go ├── doc.go ├── retryinfo.go ├── sharder.go └── internal │ └── mock_attribute_specifiable.go ├── maniphttp ├── internal │ ├── syscall │ │ ├── syscall_nonlinux.go │ │ └── syscall_linux.go │ └── compiler │ │ ├── filter.go │ │ └── filter_test.go ├── doc.go ├── backoff.go ├── retryinfo.go ├── subscriber_test.go ├── subscriber.go ├── manipulator_unix_test.go └── utils.go ├── manipcli ├── options.go ├── constants.go ├── get_test.go ├── get.go ├── count_test.go ├── delete.go ├── create_test.go ├── count.go ├── cli_test.go ├── delete_many.go ├── list.go ├── listen.go ├── delete_test.go ├── list_test.go ├── update_test.go ├── update.go ├── listen_test.go └── create.go ├── internal ├── objectid │ ├── objectid.go │ └── objectid_test.go ├── backoff │ ├── backoff.go │ └── backoff_test.go ├── idempotency │ └── idempotency.go ├── snip │ ├── snip.go │ └── snip_test.go ├── tracing │ └── tracing.go └── push │ └── utils.go ├── Makefile ├── manipvortex ├── doc.go ├── transaction.go ├── commitlog.go ├── utils.go ├── reconciler_test.go ├── commitlog_test.go ├── processors.go ├── subscriber.go ├── reconciler.go ├── prefetcher_test.go ├── subscriber_test.go ├── README.md ├── options.go └── options_test.go ├── manipmemory ├── doc.go ├── options_test.go ├── options.go ├── schema.go ├── utils_test.go └── utils.go ├── .github └── workflows │ └── build-go.yaml ├── transaction.go ├── retry.go ├── transaction_test.go ├── retry_test.go ├── maniptest ├── doc.go ├── tokenmanager_test.go ├── tokenmanager.go ├── subscriber_test.go └── subscriber.go ├── README.md ├── doc.go ├── go.mod ├── filter.go ├── manipulate.go ├── errors_test.go └── iter.go /.gitattributes: -------------------------------------------------------------------------------- 1 | go.mod filter=remod 2 | go.sum filter=remod 3 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | snapshot: 2 | name_template: "{{ .Tag }}-next" 3 | changelog: 4 | sort: asc 5 | filters: 6 | exclude: 7 | - '^docs:' 8 | - '^test:' 9 | - '^examples:' 10 | builds: 11 | - skip: true 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .tags 3 | 4 | */debug 5 | codegen 6 | *.pyc 7 | *.egg-info 8 | vendor/ 9 | *.lock 10 | testresults.xml 11 | profile.out 12 | unit_coverage.out 13 | cov.report 14 | artifacts 15 | remod.dev 16 | .remod 17 | .idea 18 | dist 19 | /coverage.xml 20 | -------------------------------------------------------------------------------- /manipmongo/backoff.go: -------------------------------------------------------------------------------- 1 | package manipmongo 2 | 3 | import "time" 4 | 5 | var ( 6 | defaultBackoffCurve = []time.Duration{ 7 | 0, 8 | 50 * time.Millisecond, 9 | 100 * time.Millisecond, 10 | 300 * time.Millisecond, 11 | 1 * time.Second, 12 | 5 * time.Second, 13 | 10 * time.Second, 14 | } 15 | ) 16 | -------------------------------------------------------------------------------- /maniphttp/internal/syscall/syscall_nonlinux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | package syscall 5 | 6 | import ( 7 | "syscall" 8 | "time" 9 | ) 10 | 11 | // package to set to the low-level/OS settings 12 | 13 | // MakeDialerControlFunc creates a custom control for the dailer 14 | func MakeDialerControlFunc(d time.Duration) func(string, string, syscall.RawConn) error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /manipcli/options.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | // generateCommandConfig hods the configuration to generate commands. 4 | type cmdConfig struct { 5 | argumentsPrefix string 6 | } 7 | 8 | // Option represents an option can for the generate command. 9 | type cmdOption func(*cmdConfig) 10 | 11 | // optionArgumentsPrefix sets the argument prefixes. 12 | func optionArgumentsPrefix(prefix string) cmdOption { 13 | return func(g *cmdConfig) { 14 | 15 | g.argumentsPrefix = prefix 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /internal/objectid/objectid.go: -------------------------------------------------------------------------------- 1 | package objectid 2 | 3 | import ( 4 | "go.mongodb.org/mongo-driver/bson/primitive" 5 | ) 6 | 7 | // Parse parses the given string and returns 8 | // a primitive.ObjectID and true if the given value is valid, 9 | // otherwise it will return an empty primitive.ObjectID and false. 10 | func Parse(s string) (primitive.ObjectID, bool) { 11 | 12 | if len(s) != 24 { 13 | return primitive.NilObjectID, false 14 | } 15 | 16 | objectID, err := primitive.ObjectIDFromHex(s) 17 | if err != nil { 18 | return primitive.NilObjectID, false 19 | } 20 | return objectID, true 21 | } 22 | -------------------------------------------------------------------------------- /internal/backoff/backoff.go: -------------------------------------------------------------------------------- 1 | package backoff 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // NextWithCurve computes the next backoff time for a given try number, 8 | // optional (non zero) hard deadline using the given backoffs curve. 9 | func NextWithCurve(try int, deadline time.Time, curve []time.Duration) time.Duration { 10 | 11 | if len(curve) == 0 { 12 | return 0 13 | } 14 | 15 | var wait time.Duration 16 | if try >= len(curve) { 17 | wait = curve[len(curve)-1] 18 | } else { 19 | wait = curve[try] 20 | } 21 | 22 | if deadline.IsZero() { 23 | return wait 24 | } 25 | 26 | now := time.Now().Round(time.Second) 27 | if now.Add(wait).After(deadline) && deadline.Sub(now) > 0 { 28 | return deadline.Sub(now) 29 | } 30 | 31 | return wait 32 | } 33 | -------------------------------------------------------------------------------- /manipmongo/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package manipmongo provides a MongoDB backed TransactionalManipulator. 13 | package manipmongo // import "go.aporeto.io/manipulate/manipmongo" 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MAKEFLAGS += --warn-undefined-variables 2 | SHELL := /bin/bash -o pipefail 3 | 4 | export GO111MODULE = on 5 | 6 | default: lint test 7 | 8 | lint: 9 | # --enable=unparam 10 | golangci-lint run \ 11 | --disable-all \ 12 | --exclude-use-default=false \ 13 | --exclude=package-comments \ 14 | --exclude=unused-parameter \ 15 | --enable=errcheck \ 16 | --enable=goimports \ 17 | --enable=ineffassign \ 18 | --enable=revive \ 19 | --enable=unused \ 20 | --enable=staticcheck \ 21 | --enable=unconvert \ 22 | --enable=misspell \ 23 | --enable=prealloc \ 24 | --enable=nakedret \ 25 | --enable=typecheck \ 26 | ./... 27 | 28 | test: 29 | go test ./... -race -cover -covermode=atomic -coverprofile=unit_coverage.out 30 | 31 | sec: 32 | gosec -quiet ./... 33 | -------------------------------------------------------------------------------- /manipvortex/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package manipvortex contains a Manipulator that can be used 13 | // as cache in front of another Manipulator. 14 | package manipvortex // import "go.aporeto.io/manipulate/manipvortex" 15 | -------------------------------------------------------------------------------- /maniphttp/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package maniphttp provides a ReST backed Manipulator. 13 | // 14 | // This is meant to be use to communicate with Bahamut based 15 | // API servers. 16 | package maniphttp // import "go.aporeto.io/manipulate/maniphttp" 17 | -------------------------------------------------------------------------------- /internal/idempotency/idempotency.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package idempotency 13 | 14 | // Keyer is the interface of an object 15 | // that can set an Idempotency Key. 16 | type Keyer interface { 17 | SetIdempotencyKey(string) 18 | IdempotencyKey() string 19 | } 20 | -------------------------------------------------------------------------------- /manipmemory/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package manipmemory provides a go-memdb backed Manipulator. 13 | // 14 | // This is meant to be use to communicate with Bahamut based 15 | // API servers. 16 | package manipmemory // import "go.aporeto.io/manipulate/manipmemory" 17 | -------------------------------------------------------------------------------- /.github/workflows/build-go.yaml: -------------------------------------------------------------------------------- 1 | name: build-go 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | 8 | defaults: 9 | run: 10 | shell: bash 11 | 12 | env: 13 | GO111MODULE: on 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | go: 22 | - "1.21" 23 | - "1.22" 24 | 25 | steps: 26 | - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 27 | 28 | - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4 29 | with: 30 | go-version: ${{ matrix.go }} 31 | cache: true 32 | 33 | - name: setup 34 | run: | 35 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 36 | 37 | - name: build 38 | run: | 39 | make 40 | -------------------------------------------------------------------------------- /maniphttp/internal/syscall/syscall_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package syscall 5 | 6 | // package to set to the low-level/OS settings 7 | 8 | import ( 9 | "os" 10 | "syscall" 11 | "time" 12 | 13 | "golang.org/x/sys/unix" 14 | ) 15 | 16 | // MakeDialerControlFunc creates a custom control for the dailer 17 | func MakeDialerControlFunc(t time.Duration) func(string, string, syscall.RawConn) error { 18 | // return if the tcpUserTimeout is not set. 19 | if t == 0 { 20 | return nil 21 | } 22 | 23 | return func(network, address string, c syscall.RawConn) error { 24 | var sysErr error 25 | err := c.Control(func(fd uintptr) { 26 | sysErr = syscall.SetsockoptInt(int(fd), syscall.SOL_TCP, unix.TCP_USER_TIMEOUT, 27 | int(t.Milliseconds())) 28 | }) 29 | if sysErr != nil { 30 | return os.NewSyscallError("setsockopt", sysErr) 31 | } 32 | return err 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /maniphttp/internal/compiler/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package compiler 13 | 14 | import ( 15 | "net/url" 16 | 17 | "go.aporeto.io/elemental" 18 | ) 19 | 20 | // CompileFilter compiles the given filter into a http query filter. 21 | func CompileFilter(f *elemental.Filter) (url.Values, error) { 22 | return url.Values{"q": []string{f.String()}}, nil 23 | } 24 | -------------------------------------------------------------------------------- /maniphttp/backoff.go: -------------------------------------------------------------------------------- 1 | package maniphttp 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | var ( 9 | defaultBackoffCurve = []time.Duration{ 10 | 0, 11 | 1 * time.Second, 12 | 5 * time.Second, 13 | 10 * time.Second, 14 | 20 * time.Second, 15 | 30 * time.Second, 16 | 60 * time.Second, 17 | } 18 | 19 | strongBackoffCurve = []time.Duration{ 20 | time.Duration(1500+rand.Intn(1000)) * time.Millisecond, // t in (1.5, 2.5) 21 | time.Duration(3000+rand.Intn(1000)) * time.Millisecond, // t in (3,4) 22 | time.Duration(7000+rand.Intn(2000)) * time.Millisecond, // t in (7,9) 23 | time.Duration(14000+rand.Intn(2000)) * time.Millisecond, // t in (14,16) 24 | time.Duration(30000+rand.Intn(2000)) * time.Millisecond, // t in (30,32) 25 | time.Duration(62000+rand.Intn(2000)) * time.Millisecond, // t in (62, 64) 26 | } 27 | 28 | testingBackoffCurve = []time.Duration{ 29 | 0, 30 | 1 * time.Millisecond, 31 | 10 * time.Millisecond, 32 | } 33 | ) 34 | -------------------------------------------------------------------------------- /transaction.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import "github.com/gofrs/uuid" 15 | 16 | // TransactionID is the type used to define a transcation ID of a store 17 | type TransactionID string 18 | 19 | // NewTransactionID returns a new transaction ID. 20 | func NewTransactionID() TransactionID { 21 | 22 | return TransactionID(uuid.Must(uuid.NewV4()).String()) 23 | } 24 | -------------------------------------------------------------------------------- /internal/snip/snip.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package snip 13 | 14 | import ( 15 | "fmt" 16 | "strings" 17 | ) 18 | 19 | // Snip snips the given token from the given error. 20 | func Snip(err error, token string) error { 21 | 22 | if len(token) == 0 || err == nil { 23 | return err 24 | } 25 | 26 | return fmt.Errorf("%s", 27 | strings.Replace( 28 | err.Error(), 29 | token, 30 | "[snip]", 31 | -1), 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /manipvortex/transaction.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "time" 16 | 17 | "go.aporeto.io/elemental" 18 | "go.aporeto.io/manipulate" 19 | ) 20 | 21 | // Transaction is the event that captures the transaction for later processing. It is 22 | // also the structure stored in the transaction logs. 23 | type Transaction struct { 24 | Date time.Time 25 | mctx manipulate.Context 26 | Object elemental.Identifiable 27 | Method elemental.Operation 28 | Deadline time.Time 29 | } 30 | -------------------------------------------------------------------------------- /retry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | ) 18 | 19 | // Retry only calls manipulateFunc for backward compatibility. 20 | // 21 | // Deprecated: manipulate.Retry is deprecated. Retry mechanism is now part of Manipulator implementations. You can safely remove this wrapper. 22 | func Retry(ctx context.Context, manipulateFunc func() error, onRetryFunc func(int, error) error) error { 23 | fmt.Println("DEPRECATED: manipulate.Retry is deprecated. Retry mechanism is now part of Manipulator implementations. You can safely remove this wrapper.") 24 | return manipulateFunc() 25 | } 26 | -------------------------------------------------------------------------------- /transaction_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "testing" 16 | 17 | // nolint:revive // Allow dot imports for readability in tests 18 | . "github.com/smartystreets/goconvey/convey" 19 | ) 20 | 21 | func TestTransaction_NewTransactionID(t *testing.T) { 22 | 23 | Convey("Given I create a NewTransactionID", t, func() { 24 | 25 | tid := NewTransactionID() 26 | 27 | Convey("Then it should have the correct type", func() { 28 | So(tid, ShouldHaveSameTypeAs, TransactionID("ttt")) 29 | }) 30 | 31 | Convey("Then it's len you be correct", func() { 32 | So(len(tid), ShouldEqual, 36) 33 | }) 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /maniphttp/retryinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniphttp 13 | 14 | import ( 15 | "go.aporeto.io/manipulate" 16 | ) 17 | 18 | // A RetryInfo contains information about a retry, 19 | type RetryInfo struct { 20 | URL string 21 | Method string 22 | 23 | err error 24 | try int 25 | mctx manipulate.Context 26 | } 27 | 28 | // Try returns the try number. 29 | func (i RetryInfo) Try() int { 30 | return i.try 31 | } 32 | 33 | // Err returns the error that caused the retry. 34 | func (i RetryInfo) Err() error { 35 | return i.err 36 | } 37 | 38 | // Context returns the manipulate.Context used. 39 | func (i RetryInfo) Context() manipulate.Context { 40 | return i.mctx 41 | } 42 | -------------------------------------------------------------------------------- /manipmemory/options_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmemory 13 | 14 | import ( 15 | "testing" 16 | 17 | // nolint:revive // Allow dot imports for readability in tests 18 | . "github.com/smartystreets/goconvey/convey" 19 | ) 20 | 21 | func Test_newConfig(t *testing.T) { 22 | 23 | Convey("Given call newConfig", t, func() { 24 | 25 | c := newConfig() 26 | 27 | Convey("Then I should get the default config", func() { 28 | So(c.noCopy, ShouldBeFalse) 29 | }) 30 | }) 31 | } 32 | 33 | func Test_Options(t *testing.T) { 34 | 35 | Convey("Calling OptionCredentials should work", t, func() { 36 | c := newConfig() 37 | OptionNoCopy(true)(c) 38 | So(c.noCopy, ShouldBeTrue) 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /manipmemory/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmemory 13 | 14 | // An Option represents a maniphttp.Manipulator option. 15 | type Option func(*config) 16 | 17 | type config struct { 18 | noCopy bool 19 | } 20 | 21 | func newConfig() *config { 22 | return &config{} 23 | } 24 | 25 | // OptionNoCopy tells the manipulator to store the data 26 | // as is without copying it. This is faster, but unsafe 27 | // as pointers are stored as is, allowing random 28 | // modifications. If you use this option, you must 29 | // make sure you are not modifying the object your store 30 | // or retrieve. 31 | func OptionNoCopy(noCopy bool) Option { 32 | return func(c *config) { 33 | c.noCopy = noCopy 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /retry_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "context" 16 | "testing" 17 | 18 | // nolint:revive // Allow dot imports for readability in tests 19 | . "github.com/smartystreets/goconvey/convey" 20 | ) 21 | 22 | func TestManipulate_Retry(t *testing.T) { 23 | 24 | Convey("Given I have a context and a manipulate function that returns no error", t, func() { 25 | 26 | ctx, cancel := context.WithCancel(context.Background()) 27 | defer cancel() 28 | 29 | m := func() error { 30 | return nil 31 | } 32 | 33 | Convey("When I call Retry", func() { 34 | 35 | err := Retry(ctx, m, nil) 36 | 37 | Convey("Then err should be nil", func() { 38 | So(err, ShouldBeNil) 39 | }) 40 | }) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /manipmongo/retryinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmongo 13 | 14 | import ( 15 | "go.aporeto.io/elemental" 16 | "go.aporeto.io/manipulate" 17 | ) 18 | 19 | // A RetryInfo contains information about a retry, 20 | type RetryInfo struct { 21 | Operation elemental.Operation 22 | Identity elemental.Identity 23 | 24 | err error 25 | try int 26 | mctx manipulate.Context 27 | 28 | defaultRetryFunc manipulate.RetryFunc 29 | } 30 | 31 | // Try returns the try number. 32 | func (i RetryInfo) Try() int { 33 | return i.try 34 | } 35 | 36 | // Err returns the error that caused the retry. 37 | func (i RetryInfo) Err() error { 38 | return i.err 39 | } 40 | 41 | // Context returns the manipulate.Context used. 42 | func (i RetryInfo) Context() manipulate.Context { 43 | return i.mctx 44 | } 45 | -------------------------------------------------------------------------------- /maniptest/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package maniptest contains a Mockable TransactionalManipulator. 13 | // It implements all method of the TransactionalManipulator but do nothing. 14 | // 15 | // Methods can be mocked by using one of the MockXX method. 16 | // 17 | // For example: 18 | // 19 | // m := maniptest.NewTestManipulator() 20 | // m.MockCreate(t, func(context *manipulate.Context, objects ...elemental.Identifiable) error { 21 | // return elemental.NewError("title", "description", "subject", 43) 22 | // }) 23 | // 24 | // The next calls to the Create method will use the given method, in the context of the given *testing.T. 25 | // If you need to reset the mocked method in the context of the same test, simply do: 26 | // 27 | // m.MockCreate(t, nil) 28 | package maniptest // import "go.aporeto.io/manipulate/maniptest" 29 | -------------------------------------------------------------------------------- /manipcli/constants.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | // Format Types 4 | const ( 5 | formatTypeColumn = "column" 6 | formatTypeHash = "hash" 7 | formatTypeCount = "count" 8 | formatTypeArray = "array" 9 | ) 10 | 11 | // Output Flags 12 | const ( 13 | flagOutputTable = "table" 14 | flagOutputJSON = "json" 15 | flagOutputNone = "none" 16 | flagOutputDefault = "default" 17 | flagOutputYAML = "yaml" 18 | flagOutputTemplate = "template" 19 | ) 20 | 21 | // General Flags 22 | const ( 23 | flagTrackingID = "tracking-id" 24 | flagOutput = "output" 25 | flagAPI = "api" 26 | flagAPISkipVerify = "api-skip-verify" 27 | flagNamespace = "namespace" 28 | flagToken = "token" 29 | flagCACertPath = "api-cacert" 30 | flagEncoding = "encoding" 31 | flagRecursive = "recursive" 32 | flagParameters = "param" 33 | flagParent = "parent" 34 | ) 35 | 36 | // Create/update FLags 37 | const ( 38 | flagInteractive = "interactive" 39 | flagInputValues = "input-values" 40 | flagInputData = "input-data" 41 | flagEditor = "editor" 42 | flagInputFile = "input-file" 43 | flagInputURL = "input-url" 44 | flagInputSet = "input-set" 45 | flagPrint = "print" 46 | flagRender = "render" 47 | ) 48 | 49 | // List Flags 50 | const ( 51 | flagPageSize = "page-size" 52 | flagPage = "page" 53 | flagFilter = "filter" 54 | flagOrder = "order" 55 | ) 56 | 57 | // Delete Flags 58 | const ( 59 | flagForce = "force" 60 | flagConfirm = "confirm" 61 | ) 62 | -------------------------------------------------------------------------------- /maniphttp/internal/compiler/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package compiler 13 | 14 | import ( 15 | "testing" 16 | 17 | // nolint:revive // Allow dot imports for readability in tests 18 | . "github.com/smartystreets/goconvey/convey" 19 | "go.aporeto.io/elemental" 20 | ) 21 | 22 | func TestFilter_CompileFilter(t *testing.T) { 23 | 24 | Convey("Given I create a new Filter", t, func() { 25 | 26 | f := elemental.NewFilterComposer(). 27 | WithKey("name").Equals("thename"). 28 | WithKey("ID").Equals("xxx"). 29 | WithKey("associatedTags").Contains("yy=zz"). 30 | Done() 31 | 32 | Convey("When I call CompileFilter on it", func() { 33 | 34 | v, err := CompileFilter(f) 35 | 36 | Convey("Then err should be nil", func() { 37 | So(err, ShouldBeNil) 38 | }) 39 | 40 | Convey("Then the filter should be correct", func() { 41 | So(v.Get("q"), ShouldEqual, `name == "thename" and ID == "xxx" and associatedTags contains ["yy=zz"]`) 42 | }) 43 | }) 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /manipvortex/commitlog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "encoding/json" 17 | "os" 18 | ) 19 | 20 | // newLogWriter creates a new log for transactions. At this point there is no 21 | // file rotation. If it runs out of storage it will die. 22 | func newLogWriter(ctx context.Context, filename string, size int) (chan *Transaction, error) { 23 | 24 | f, err := os.Create(filename) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | events := make(chan *Transaction, size) 30 | 31 | go func(f *os.File) { 32 | // #nosec G307 33 | defer f.Close() // nolint errcheck 34 | 35 | for { 36 | 37 | select { 38 | 39 | case d := <-events: 40 | data, err := json.Marshal(d) 41 | if err != nil { 42 | continue 43 | } 44 | 45 | _, err = f.Write(data) 46 | if err != nil { 47 | return 48 | } 49 | 50 | case <-ctx.Done(): 51 | close(events) 52 | return 53 | } 54 | } 55 | }(f) 56 | 57 | return events, nil 58 | 59 | } 60 | -------------------------------------------------------------------------------- /manipmemory/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmemory 13 | 14 | import ( 15 | "go.aporeto.io/elemental" 16 | ) 17 | 18 | // IndexType is the data type of the index. 19 | type IndexType int 20 | 21 | // Values of IndexType. 22 | const ( 23 | IndexTypeString IndexType = iota 24 | IndexTypeSlice 25 | IndexTypeMap 26 | IndexTypeBoolean 27 | IndexTypeStringBased 28 | ) 29 | 30 | // Index configures the attributes that must be indexed. 31 | type Index struct { 32 | 33 | // Name of the index. Must match an attribute of elemental. 34 | Name string 35 | 36 | // Type of the index. 37 | Type IndexType 38 | 39 | // If there is a unique requirement on the index. At least 40 | // one of the indexes must have this set. 41 | Unique bool 42 | 43 | // Attribute is the elemental attribute name. 44 | Attribute string 45 | } 46 | 47 | // IdentitySchema is the configuration of the indexes for the associated identity. 48 | type IdentitySchema struct { 49 | // Identity of the object. 50 | Identity elemental.Identity 51 | 52 | // Indexes of the object 53 | Indexes []*Index 54 | } 55 | -------------------------------------------------------------------------------- /internal/objectid/objectid_test.go: -------------------------------------------------------------------------------- 1 | package objectid 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "go.mongodb.org/mongo-driver/bson/primitive" 8 | ) 9 | 10 | func TestParse(t *testing.T) { 11 | type args struct { 12 | s string 13 | } 14 | tests := []struct { 15 | name string 16 | args args 17 | want func() primitive.ObjectID 18 | want1 bool 19 | }{ 20 | { 21 | "valid", 22 | args{ 23 | "5d66b8f7919e0c446f0b4597", 24 | }, 25 | func() primitive.ObjectID { 26 | id, _ := primitive.ObjectIDFromHex("5d66b8f7919e0c446f0b4597") 27 | return id 28 | }, 29 | true, 30 | }, 31 | { 32 | "not exa", 33 | args{ 34 | "ZZZ6b8f7919e0c446f0b4597", 35 | }, 36 | func() primitive.ObjectID { 37 | return primitive.NilObjectID 38 | }, 39 | false, 40 | }, 41 | { 42 | "too short", 43 | args{ 44 | "5d66b8f7919e0c446f0b459", 45 | }, 46 | func() primitive.ObjectID { 47 | return primitive.NilObjectID 48 | }, 49 | false, 50 | }, 51 | { 52 | "empty", 53 | args{ 54 | "", 55 | }, 56 | func() primitive.ObjectID { 57 | return primitive.NilObjectID 58 | }, 59 | false, 60 | }, 61 | { 62 | "weird stuff", 63 | args{ 64 | "hello world how are you", 65 | }, 66 | func() primitive.ObjectID { 67 | return primitive.NilObjectID 68 | }, 69 | false, 70 | }, 71 | } 72 | for _, tt := range tests { 73 | t.Run(tt.name, func(t *testing.T) { 74 | got, got1 := Parse(tt.args.s) 75 | if !reflect.DeepEqual(got, tt.want()) { 76 | t.Errorf("Parse() got = %v, want %v", got, tt.want()) 77 | } 78 | if got1 != tt.want1 { 79 | t.Errorf("Parse() got1 = %v, want %v", got1, tt.want1) 80 | } 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /internal/tracing/tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package tracing 13 | 14 | import ( 15 | "context" 16 | 17 | opentracing "github.com/opentracing/opentracing-go" 18 | "go.aporeto.io/manipulate" 19 | ) 20 | 21 | // StartTrace starts a new trace from the root span if any. 22 | func StartTrace(mctx manipulate.Context, name string) opentracing.Span { 23 | 24 | if mctx == nil { 25 | sp, _ := opentracing.StartSpanFromContext(context.Background(), name) 26 | return sp 27 | } 28 | 29 | sp, _ := opentracing.StartSpanFromContext(mctx.Context(), name) 30 | 31 | sp.SetTag("manipulate.context.api_version", mctx.Version()) 32 | sp.SetTag("manipulate.context.page", mctx.Page()) 33 | sp.SetTag("manipulate.context.page_size", mctx.PageSize()) 34 | sp.SetTag("manipulate.context.override_protection", mctx.Override()) 35 | sp.SetTag("manipulate.context.recursive", mctx.Recursive()) 36 | 37 | if mctx.Namespace() != "" { 38 | sp.SetTag("manipulate.context.namespace", mctx.Namespace()) 39 | } else { 40 | sp.SetTag("manipulate.context.namespace", "manipulator-default") 41 | } 42 | 43 | if len(mctx.Parameters()) >= 0 { 44 | sp.SetTag("manipulate.context.parameters", mctx.Parameters()) 45 | } 46 | 47 | if mctx.Filter() != nil { 48 | sp.SetTag("manipulate.context.filter", mctx.Filter().String()) 49 | } 50 | 51 | return sp 52 | } 53 | -------------------------------------------------------------------------------- /internal/snip/snip_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package snip 13 | 14 | import ( 15 | "errors" 16 | "testing" 17 | 18 | // nolint:revive // Allow dot imports for readability in tests 19 | . "github.com/smartystreets/goconvey/convey" 20 | ) 21 | 22 | func TestTokenUtils_Snip(t *testing.T) { 23 | 24 | Convey("Given have a token and and error containing the token", t, func() { 25 | 26 | token := "token" 27 | err := errors.New("your token is token") 28 | 29 | Convey("When I call Snip", func() { 30 | 31 | e := Snip(err, token) 32 | 33 | Convey("Then err should have the reference to token snipped", func() { 34 | So(e.Error(), ShouldEqual, "your [snip] is [snip]") 35 | }) 36 | }) 37 | }) 38 | 39 | Convey("Given have a token and and error that doesn't contain the token", t, func() { 40 | 41 | token := "token" 42 | err := errors.New("your secret is secret") 43 | 44 | Convey("When I call Snip", func() { 45 | 46 | e := Snip(err, token) 47 | 48 | Convey("Then err should have the reference to token snipped", func() { 49 | So(e.Error(), ShouldEqual, "your secret is secret") 50 | }) 51 | }) 52 | }) 53 | 54 | Convey("Given I have a token and a nil error", t, func() { 55 | 56 | token := "token" 57 | 58 | Convey("When I call Snip", func() { 59 | 60 | e := Snip(nil, token) 61 | 62 | Convey("Then err should be nil", func() { 63 | So(e, ShouldBeNil) 64 | }) 65 | }) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /manipvortex/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "go.aporeto.io/manipulate" 16 | ) 17 | 18 | func isStrongReadConsistency(mctx manipulate.Context, processor *Processor, defaultConsistency manipulate.ReadConsistency) bool { 19 | 20 | if mctx != nil && mctx.ReadConsistency() != manipulate.ReadConsistencyDefault { 21 | return mctx.ReadConsistency() == manipulate.ReadConsistencyStrong 22 | } 23 | 24 | if processor != nil && processor.ReadConsistency != manipulate.ReadConsistencyDefault { 25 | return processor.ReadConsistency == manipulate.ReadConsistencyStrong 26 | } 27 | 28 | return defaultConsistency == manipulate.ReadConsistencyStrong 29 | } 30 | 31 | func isStrongWriteConsistency(mctx manipulate.Context, processor *Processor, defaultConsistency manipulate.WriteConsistency) bool { 32 | 33 | if mctx != nil && mctx.WriteConsistency() != manipulate.WriteConsistencyDefault { 34 | return mctx.WriteConsistency() == manipulate.WriteConsistencyStrong || mctx.WriteConsistency() == manipulate.WriteConsistencyStrongest 35 | } 36 | 37 | if processor != nil && processor.WriteConsistency != manipulate.WriteConsistencyDefault { 38 | return processor.WriteConsistency == manipulate.WriteConsistencyStrong || processor.WriteConsistency == manipulate.WriteConsistencyStrongest 39 | } 40 | 41 | return defaultConsistency == manipulate.WriteConsistencyStrong || defaultConsistency == manipulate.WriteConsistencyStrongest 42 | } 43 | -------------------------------------------------------------------------------- /manipmemory/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmemory 13 | 14 | import ( 15 | "testing" 16 | 17 | // nolint:revive // Allow dot imports for readability in tests 18 | . "github.com/smartystreets/goconvey/convey" 19 | ) 20 | 21 | func Test_boolIndex(t *testing.T) { 22 | 23 | type testObject struct { 24 | somevalue bool 25 | someothervalue string 26 | } 27 | 28 | Convey("When I call boolindex", t, func() { 29 | 30 | Convey("When I use a good data structure", func() { 31 | t := &testObject{ 32 | somevalue: true, 33 | someothervalue: "somestring", 34 | } 35 | 36 | val, err := boolIndex(t, "somevalue") 37 | So(err, ShouldBeNil) 38 | So(val, ShouldBeTrue) 39 | }) 40 | 41 | Convey("When I use a good data structure with a bad field", func() { 42 | t := &testObject{ 43 | somevalue: true, 44 | someothervalue: "somestring", 45 | } 46 | 47 | _, err := boolIndex(t, "no value") 48 | So(err, ShouldNotBeNil) 49 | }) 50 | 51 | Convey("When I use nil", func() { 52 | t := &testObject{ 53 | somevalue: true, 54 | someothervalue: "somestring", 55 | } 56 | 57 | _, err := boolIndex(t, "no value") 58 | So(err, ShouldNotBeNil) 59 | }) 60 | 61 | Convey("When I use a bad type field", func() { 62 | t := &testObject{ 63 | somevalue: true, 64 | someothervalue: "somestring", 65 | } 66 | 67 | _, err := boolIndex(t, "somestring") 68 | So(err, ShouldNotBeNil) 69 | }) 70 | }) 71 | } 72 | -------------------------------------------------------------------------------- /manipmongo/sharder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmongo 13 | 14 | import ( 15 | "go.aporeto.io/elemental" 16 | "go.aporeto.io/manipulate" 17 | "go.mongodb.org/mongo-driver/bson" 18 | ) 19 | 20 | // A Sharder is the interface of an object that can be use 21 | // to manage sharding of resources. 22 | type Sharder interface { 23 | 24 | // Shard will be call when the shard key needs to be set to 25 | // the given elemental.Identifiable. 26 | Shard(manipulate.TransactionalManipulator, manipulate.Context, elemental.Identifiable) error 27 | 28 | // OnShardedWrite will be called after a successful sharded write 29 | // If it returns an error, this error will be returned to the caller 30 | // of the manipulate Operation, but the object that has been 31 | // created will still be created in database. 32 | OnShardedWrite(manipulate.TransactionalManipulator, manipulate.Context, elemental.Operation, elemental.Identifiable) error 33 | 34 | // FilterOne returns the filter bit as bson.M that must be 35 | // used to perform an efficient localized query for a single object. 36 | // 37 | // You can return nil which will trigger a broadcast. 38 | FilterOne(manipulate.TransactionalManipulator, manipulate.Context, elemental.Identifiable) (bson.D, error) 39 | 40 | // FilterMany returns the filter bit as bson.M that must be 41 | // used to perform an efficient localized query for multiple objects. 42 | // 43 | // You can return nil which will trigger a broadcast. 44 | FilterMany(manipulate.TransactionalManipulator, manipulate.Context, elemental.Identity) (bson.D, error) 45 | } 46 | -------------------------------------------------------------------------------- /internal/push/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package push 13 | 14 | import ( 15 | "fmt" 16 | "io" 17 | "math" 18 | "net/url" 19 | "strings" 20 | "time" 21 | 22 | "go.aporeto.io/elemental" 23 | "go.aporeto.io/manipulate" 24 | ) 25 | 26 | func decodeErrors(r io.Reader, encoding elemental.EncodingType) error { 27 | 28 | es := []elemental.Error{} 29 | 30 | data, err := io.ReadAll(r) 31 | if err != nil { 32 | return manipulate.ErrCannotUnmarshal{Err: fmt.Errorf("%w: %s", err, string(data))} 33 | } 34 | 35 | if err := elemental.Decode(encoding, data, &es); err != nil { 36 | return manipulate.ErrCannotUnmarshal{Err: fmt.Errorf("%w: %s", err, string(data))} 37 | } 38 | 39 | errs := elemental.NewErrors() 40 | for _, e := range es { 41 | errs = append(errs, e) 42 | } 43 | 44 | return errs 45 | } 46 | 47 | func makeURL(u string, namespace string, password string, recursive, supportErrorEvents bool) string { 48 | 49 | u = strings.Replace(u, "https://", "wss://", 1) 50 | 51 | args := []string{ 52 | fmt.Sprintf("namespace=%s", url.QueryEscape(namespace)), 53 | } 54 | 55 | if password != "" { 56 | args = append(args, fmt.Sprintf("token=%s", password)) 57 | } 58 | 59 | if recursive { 60 | args = append(args, "mode=all") 61 | } 62 | 63 | if supportErrorEvents { 64 | args = append(args, "enableErrors=true") 65 | } 66 | 67 | return fmt.Sprintf("%s?%s", u, strings.Join(args, "&")) 68 | } 69 | 70 | const maxBackoff = 8000 71 | 72 | func nextBackoff(try int) time.Duration { 73 | 74 | return time.Duration(math.Min(math.Pow(4.0, float64(try))-1, maxBackoff)) * time.Millisecond 75 | } 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Manipulate 2 | 3 | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/b5011051258243de99974313f4e4a8b6)](https://www.codacy.com/gh/PaloAltoNetworks/manipulate/dashboard?utm_source=github.com&utm_medium=referral&utm_content=PaloAltoNetworks/manipulate&utm_campaign=Badge_Grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/b5011051258243de99974313f4e4a8b6)](https://www.codacy.com/gh/PaloAltoNetworks/manipulate/dashboard?utm_source=github.com&utm_medium=referral&utm_content=PaloAltoNetworks/manipulate&utm_campaign=Badge_Coverage) 4 | 5 | > Note: this readme is a work in progress 6 | 7 | Package manipulate provides everything needed to perform CRUD operations on an 8 | [elemental](https://go.aporeto.io/elemental) based data model. 9 | 10 | The main interface is `Manipulator`. This interface provides various methods for 11 | creation, modification, retrieval and so on. 12 | 13 | A Manipulator works with `elemental.Identifiable`. 14 | 15 | The storage engine used by a Manipulator is abstracted. By default manipulate 16 | provides implementations for Mongo, ReST HTTP 1and a Memory backed datastore. 17 | You can of course implement Your own storage implementation. 18 | 19 | There is also a mocking package called maniptest to make it easy to mock any 20 | manipulator implementation for unit testing. 21 | 22 | Each method of a Manipulator is taking a `manipulate.Context` as argument. The 23 | context is used to pass additional informations like a Filter, or some 24 | Parameters. 25 | 26 | ## Example for creating an object 27 | 28 | ```go 29 | // Create a User from a generated Elemental model. 30 | user := models.NewUser() // always use the initializer to get various default value correctly set. 31 | user.FullName := "Antoine Mercadal" 32 | user.Login := "primalmotion" 33 | 34 | // Create Mongo Manipulator. 35 | m := manipmongo.New("127.0.0.1", "test") 36 | 37 | // Then create the User. 38 | m.Create(nil, user) 39 | ``` 40 | 41 | ## Example for retreving an object 42 | 43 | ```go 44 | // Create a Context with a filter. 45 | ctx := manipulate.NewContextWithFilter(elemental.NewFilterComposer(). 46 | WithKey("login").Equals("primalmotion"). 47 | Done(), 48 | ) 49 | 50 | // Retrieve the users matching the filter. 51 | var users models.UserLists 52 | m.RetrieveMany(ctx, &users) 53 | ``` 54 | -------------------------------------------------------------------------------- /internal/backoff/backoff_test.go: -------------------------------------------------------------------------------- 1 | package backoff 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestNext(t *testing.T) { 9 | 10 | type args struct { 11 | try int 12 | deadline time.Time 13 | curve []time.Duration 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want time.Duration 19 | }{ 20 | { 21 | "empty curve no deadlie", 22 | args{ 23 | 0, 24 | time.Time{}, 25 | nil, 26 | }, 27 | 0 * time.Second, 28 | }, 29 | 30 | { 31 | "try 0 no deadline", 32 | args{ 33 | 0, 34 | time.Time{}, 35 | []time.Duration{ 36 | 1 * time.Second, 37 | 2 * time.Second, 38 | 3 * time.Second, 39 | }, 40 | }, 41 | 1 * time.Second, 42 | }, 43 | { 44 | "try 1 no deadline", 45 | args{ 46 | 1, 47 | time.Time{}, 48 | []time.Duration{ 49 | 1 * time.Second, 50 | 2 * time.Second, 51 | 3 * time.Second, 52 | }, 53 | }, 54 | 2 * time.Second, 55 | }, 56 | { 57 | "try 2 no deadline", 58 | args{ 59 | 2, 60 | time.Time{}, 61 | []time.Duration{ 62 | 1 * time.Second, 63 | 2 * time.Second, 64 | 3 * time.Second, 65 | }, 66 | }, 67 | 3 * time.Second, 68 | }, 69 | { 70 | "try 2+ no deadline", 71 | args{ 72 | 3, 73 | time.Time{}, 74 | []time.Duration{ 75 | 1 * time.Second, 76 | 2 * time.Second, 77 | 3 * time.Second, 78 | }, 79 | }, 80 | 3 * time.Second, 81 | }, 82 | 83 | { 84 | "try before deadline", 85 | args{ 86 | 3, 87 | time.Now().Add(4 * time.Second), 88 | []time.Duration{ 89 | 1 * time.Second, 90 | 2 * time.Second, 91 | 3 * time.Second, 92 | }, 93 | }, 94 | 3 * time.Second, 95 | }, 96 | { 97 | "try after deadline", 98 | args{ 99 | 3, 100 | time.Now().Add(5 * time.Second).Round(time.Second), 101 | []time.Duration{ 102 | 10 * time.Second, 103 | }, 104 | }, 105 | 5 * time.Second, 106 | }, 107 | } 108 | for _, tt := range tests { 109 | t.Run(tt.name, func(t *testing.T) { 110 | if got := NextWithCurve(tt.args.try, tt.args.deadline, tt.args.curve); got != tt.want { 111 | t.Errorf("Next() = %v, want %v", got, tt.want) 112 | } 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | // Package manipulate provides everything needed to perform CRUD operations 13 | // on an https://go.aporeto.io/elemental based data model. 14 | // 15 | // The main interface is Manipulator. This interface provides various 16 | // methods for creation, modification, retrieval and so on. TransactionalManipulator, 17 | // which is an extension of the Manipulator add methods to manage transactions, like 18 | // Commit and Abort. 19 | // 20 | // A Manipulator works with some elemental.Identifiables. 21 | // 22 | // The storage engine used by a Manipulator is abstracted. By default manipulate 23 | // provides implementations for Rest API over HTTP or websocket, Mongo DB, Memory and a mock Manipulator for 24 | // unit testing. You can of course create your own implementation. 25 | // 26 | // Each method of a Manipulator is taking a manipulate.Context as argument. The context is used 27 | // to pass additional informations like a Filter or some Parameters. 28 | // 29 | // Example for creating an object: 30 | // 31 | // // Create a User from a generated Elemental model. 32 | // user := models.NewUser() 33 | // user.FullName, user.Login := "Antoine Mercadal", "primalmotion" 34 | // 35 | // // Create Mongo Manipulator. 36 | // m := manipmongo.NewMongoManipulator([]{"127.0.0.1"}, "test", "db-username", "db-password", "db-authsource", 512) 37 | // 38 | // // Then create the User. 39 | // m.Create(nil, user) 40 | // 41 | // Example for retreving an object: 42 | // 43 | // // Create a Context with a filter. 44 | // ctx := manipulate.NewContextWithFilter( 45 | // manipulate.NewFilterComposer().WithKey("login").Equals("primalmotion"). 46 | // Done()) 47 | // 48 | // // Retrieve the users matching the filter. 49 | // var users models.UserLists 50 | // m.RetrieveMany(ctx, models.UserIdentity, &users) 51 | package manipulate // import "go.aporeto.io/manipulate" 52 | -------------------------------------------------------------------------------- /manipcli/get_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateGetCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a get command", t, func() { 20 | 21 | task1 := testmodel.NewTask() 22 | task1.ID = "617aec75a829de0001da2032" 23 | task1.Name = "task1" 24 | 25 | m := maniptest.NewTestManipulator() 26 | m.MockRetrieve(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 27 | object.SetIdentifier(task1.ID) 28 | object.(*testmodel.Task).Name = task1.Name 29 | return nil 30 | }) 31 | 32 | cmd, err := generateGetCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 33 | return m, nil 34 | }) 35 | 36 | So(err, ShouldEqual, nil) 37 | assertCommandAndSetFlags(cmd) 38 | 39 | Convey("When I call execute with a json output", func() { 40 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 41 | output := bytes.NewBufferString("") 42 | cmd.SetOut(output) 43 | err := cmd.Execute() 44 | 45 | Convey("Then I should get a generated command", func() { 46 | So(err, ShouldEqual, nil) 47 | So(output.String(), ShouldEqual, `{ 48 | "ID": "617aec75a829de0001da2032", 49 | "description": "", 50 | "name": "task1", 51 | "parentID": "", 52 | "parentType": "", 53 | "status": "TODO" 54 | }`) 55 | }) 56 | }) 57 | }) 58 | 59 | Convey("Given I generate a get command that returns an error", t, func() { 60 | 61 | cmd, err := generateGetCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 62 | return nil, fmt.Errorf("boom") 63 | }) 64 | 65 | So(err, ShouldEqual, nil) 66 | assertCommandAndSetFlags(cmd) 67 | 68 | Convey("When I call execute", func() { 69 | 70 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 71 | err := cmd.Execute() 72 | 73 | Convey("Then I should get an error", func() { 74 | So(err, ShouldNotEqual, nil) 75 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 76 | }) 77 | }) 78 | }) 79 | 80 | } 81 | -------------------------------------------------------------------------------- /manipvortex/reconciler_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | "testing" 18 | 19 | // nolint:revive // Allow dot imports for readability in tests 20 | . "github.com/smartystreets/goconvey/convey" 21 | "go.aporeto.io/elemental" 22 | "go.aporeto.io/manipulate" 23 | ) 24 | 25 | func TestTestReconciler(t *testing.T) { 26 | 27 | Convey("Given I create a new TestReconciler", t, func() { 28 | 29 | r := NewTestReconciler() 30 | 31 | Convey("Then it should be initialized", func() { 32 | So(r, ShouldImplement, (*TestReconciler)(nil)) 33 | So(r.(*testReconciler).lock, ShouldNotBeNil) 34 | So(r.(*testReconciler).mocks, ShouldNotBeNil) 35 | }) 36 | 37 | Convey("When I call the Accept method without mock", func() { 38 | 39 | obj, ok, err := r.Reconcile(manipulate.NewContext(context.Background()), elemental.OperationCreate, nil) 40 | 41 | Convey("Then err should be nil", func() { 42 | So(err, ShouldBeNil) 43 | }) 44 | 45 | Convey("Then ok should be true", func() { 46 | So(ok, ShouldBeTrue) 47 | }) 48 | 49 | Convey("The object should be nil", func() { 50 | So(obj, ShouldBeNil) 51 | }) 52 | }) 53 | 54 | Convey("When I call the Accept method with a mock", func() { 55 | 56 | r.MockReconcile(t, func(_ manipulate.Context, _ elemental.Operation, obj elemental.Identifiable) (elemental.Identifiable, bool, error) { 57 | return obj, false, fmt.Errorf("boom") 58 | }) 59 | 60 | obj, ok, err := r.Reconcile(manipulate.NewContext(context.Background()), elemental.OperationCreate, nil) 61 | 62 | Convey("Then err should not be nil", func() { 63 | So(err, ShouldNotBeNil) 64 | So(err.Error(), ShouldEqual, "boom") 65 | }) 66 | 67 | Convey("Then ok should be false", func() { 68 | So(ok, ShouldBeFalse) 69 | }) 70 | 71 | Convey("The object should be nil", func() { 72 | So(obj, ShouldBeNil) 73 | }) 74 | 75 | }) 76 | }) 77 | } 78 | -------------------------------------------------------------------------------- /maniptest/tokenmanager_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniptest 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | "testing" 18 | 19 | // nolint:revive // Allow dot imports for readability in tests 20 | . "github.com/smartystreets/goconvey/convey" 21 | ) 22 | 23 | func TestTestSubscriber_MockIssue(t *testing.T) { 24 | 25 | Convey("Given I have TestTokenManager", t, func() { 26 | 27 | tm := NewTestTokenManager() 28 | 29 | Convey("When I call Issue without mock", func() { 30 | 31 | token, err := tm.Issue(context.Background()) 32 | 33 | Convey("Then err should be nil", func() { 34 | So(err, ShouldBeNil) 35 | }) 36 | 37 | Convey("Then token should be correct", func() { 38 | So(token, ShouldBeEmpty) 39 | }) 40 | }) 41 | 42 | Convey("When I call Issue with a mock", func() { 43 | 44 | tm.MockIssue(t, func(ctx context.Context) (string, error) { 45 | return "token", fmt.Errorf("test") 46 | }) 47 | 48 | token, err := tm.Issue(context.Background()) 49 | 50 | Convey("Then err should be correct", func() { 51 | So(err.Error(), ShouldEqual, "test") 52 | }) 53 | 54 | Convey("Then token should be correct", func() { 55 | So(token, ShouldEqual, "token") 56 | }) 57 | }) 58 | }) 59 | } 60 | 61 | func TestTestSubscriber_MockRun(t *testing.T) { 62 | 63 | Convey("Given I have TestTokenManager", t, func() { 64 | 65 | tm := NewTestTokenManager() 66 | 67 | Convey("When I call Run without mock", func() { 68 | 69 | tm.Run(context.Background(), nil) 70 | 71 | Convey("Then nothing should happen", func() { 72 | }) 73 | }) 74 | 75 | Convey("When I call Issue with a mock", func() { 76 | 77 | tm.MockRun(t, func(ctx context.Context, tokenCh chan string) { 78 | go func() { tokenCh <- "coucou" }() 79 | }) 80 | 81 | ch := make(chan string) 82 | tm.Run(context.Background(), ch) 83 | 84 | Convey("Then the value in the chan should be correct", func() { 85 | So(<-ch, ShouldEqual, "coucou") 86 | }) 87 | }) 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /manipvortex/commitlog_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "encoding/json" 17 | "os" 18 | "testing" 19 | "time" 20 | 21 | // nolint:revive // Allow dot imports for readability in tests 22 | . "github.com/smartystreets/goconvey/convey" 23 | "go.aporeto.io/elemental" 24 | testmodel "go.aporeto.io/elemental/test/model" 25 | "go.aporeto.io/manipulate" 26 | ) 27 | 28 | type testDataType struct { 29 | Date time.Time 30 | Object testmodel.List 31 | Method elemental.Operation 32 | Deadline time.Time 33 | } 34 | 35 | func Test_newLogWriter(t *testing.T) { 36 | 37 | ctx, cancel := context.WithCancel(context.Background()) 38 | defer cancel() 39 | 40 | Convey("When I create a new log writer with a bad file, I should get an error", t, func() { 41 | _, err := newLogWriter(ctx, "", 100) 42 | So(err, ShouldNotBeNil) 43 | }) 44 | 45 | Convey("When I create a new log writer with a good file", t, func() { 46 | c, err := newLogWriter(ctx, "test.log", 100) 47 | defer os.Remove("test.log") // nolint errcheck 48 | 49 | Convey("There should be no error and a valid channel", func() { 50 | So(err, ShouldBeNil) 51 | So(c, ShouldNotBeNil) 52 | So(cap(c), ShouldEqual, 100) 53 | }) 54 | 55 | Convey("When I send an event in the channel, the data should be in the file", func() { 56 | now := time.Now() 57 | 58 | object := &testmodel.List{ 59 | ID: "1", 60 | Name: "Object", 61 | } 62 | 63 | e := &Transaction{ 64 | Date: now, 65 | mctx: manipulate.NewContext(ctx), 66 | Object: object, 67 | Method: elemental.OperationCreate, 68 | Deadline: now.Add(10 * time.Second), 69 | } 70 | c <- e 71 | time.Sleep(500 * time.Millisecond) 72 | 73 | data, err := os.ReadFile("test.log") 74 | So(err, ShouldBeNil) 75 | 76 | model := &testDataType{} 77 | 78 | err = json.Unmarshal(data, &model) 79 | So(err, ShouldBeNil) 80 | So(model.Method, ShouldResemble, e.Method) 81 | So(model.Object.ID, ShouldEqual, "1") 82 | So(model.Object.Name, ShouldEqual, "Object") 83 | }) 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /manipcli/get.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.aporeto.io/elemental" 11 | "go.aporeto.io/manipulate" 12 | ) 13 | 14 | // generateGetCommandForIdentity generates the command to get an object based on its identity. 15 | func generateGetCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 16 | 17 | cmd := &cobra.Command{ 18 | Use: fmt.Sprintf("%s ", identity.Name), 19 | Aliases: []string{identity.Category}, 20 | Short: "Get an existing " + identity.Name, 21 | Args: cobra.ExactArgs(1), 22 | // Aliases: TODO: Missing alias from the spec file -> To be stored in the identity ?, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | 25 | fParam := viper.GetStringSlice("param") 26 | fTrackingID := viper.GetString(flagTrackingID) 27 | fRecursive := viper.GetBool(flagRecursive) 28 | fOutput := viper.GetString(flagOutput) 29 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 30 | fOutputTemplate := viper.GetString(flagOutputTemplate) 31 | 32 | manipulator, err := manipulatorMaker() 33 | if err != nil { 34 | return fmt.Errorf("unable to make manipulator: %w", err) 35 | } 36 | 37 | parameters, err := parametersToURLValues(fParam) 38 | if err != nil { 39 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 40 | } 41 | 42 | options := []manipulate.ContextOption{ 43 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 44 | manipulate.ContextOptionParameters(parameters), 45 | manipulate.ContextOptionFields(fFormatTypeColumn), 46 | manipulate.ContextOptionRecursive(fRecursive), 47 | } 48 | 49 | ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) 50 | defer cancel() 51 | 52 | mctx := manipulate.NewContext(ctx, options...) 53 | 54 | identifiable, err := retrieveObjectByIDOrByName(mctx, manipulator, identity, args[0], modelManager) 55 | if err != nil { 56 | return fmt.Errorf("unable to retrieve %s: %w", identity.Name, err) 57 | } 58 | 59 | outputType := fOutput 60 | if fOutput == flagOutputDefault { 61 | outputType = flagOutputJSON 62 | } 63 | 64 | result, err := formatObjects( 65 | prepareOutputFormat(outputType, formatTypeHash, fFormatTypeColumn, fOutputTemplate), 66 | false, 67 | identifiable, 68 | ) 69 | 70 | if err != nil { 71 | return fmt.Errorf("unable to format output: %w", err) 72 | } 73 | 74 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 75 | return nil 76 | }, 77 | } 78 | 79 | return cmd, nil 80 | } 81 | -------------------------------------------------------------------------------- /manipvortex/processors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "time" 16 | 17 | "go.aporeto.io/elemental" 18 | "go.aporeto.io/manipulate" 19 | ) 20 | 21 | // RetrieveManyHook is the type of a hook for retrieve many 22 | type RetrieveManyHook func(m manipulate.Manipulator, mctx manipulate.Context, dest elemental.Identifiables) (reconcile bool, err error) 23 | 24 | // Processor configures the processing details for a specific identity. 25 | type Processor struct { 26 | 27 | // Identity is the identity of the object that is stored in the DB. 28 | Identity elemental.Identity 29 | 30 | // TODO: Mode is the type of default consistency mode required from the cache. 31 | // This consistency can be overwritten by manipulate options. 32 | WriteConsistency manipulate.WriteConsistency 33 | 34 | // TODO: Mode is the type of default consistency mode required from the cache. 35 | // This consistency can be overwritten by manipulate options. 36 | ReadConsistency manipulate.ReadConsistency 37 | 38 | // QueueingDuration is the maximum time that an object should be 39 | // cached if the backend is not responding. 40 | QueueingDuration time.Duration 41 | 42 | // RetrieveManyHook is a hook function that can be called 43 | // before a RetrieveMany call. It returns an error and a continue 44 | // boolean. If the continue false, we can return without any 45 | // additional calls. 46 | RetrieveManyHook RetrieveManyHook 47 | 48 | // DownstreamReconciler is the custom Reconciler for the processor that 49 | // will be called to reconcile downstream writes. 50 | DownstreamReconciler Reconciler 51 | 52 | // UpstreamReconciler is the custom Reconciler for the processor that 53 | // will be called to reconcile upstream writes. 54 | UpstreamReconciler Reconciler 55 | 56 | // CommitOnEvent with commit the event in the cache even if a client 57 | // is subscribed for this event. If left false, it will only commit 58 | // if no client has subscribed for this event. It will always forward 59 | // the event to the clients. 60 | CommitOnEvent bool 61 | 62 | // LazySync will not sync all data of the identity on startup, but 63 | // will only load data on demand based on the transactions. 64 | LazySync bool 65 | } 66 | -------------------------------------------------------------------------------- /manipcli/count_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateCountCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a count command", t, func() { 20 | 21 | m := maniptest.NewTestManipulator() 22 | m.MockCount(t, func(ctx manipulate.Context, identity elemental.Identity) (int, error) { 23 | return 2, nil 24 | }) 25 | 26 | cmd, err := generateCountCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 27 | return m, nil 28 | }) 29 | 30 | So(err, ShouldEqual, nil) 31 | assertCommandAndSetFlags(cmd) 32 | 33 | Convey("When I call execute without filter", func() { 34 | 35 | output := bytes.NewBufferString("") 36 | cmd.SetOut(output) 37 | err := cmd.Execute() 38 | 39 | Convey("Then I should get a generated command", func() { 40 | So(err, ShouldEqual, nil) 41 | So(output.String(), ShouldEqual, "2") 42 | }) 43 | }) 44 | 45 | Convey("When I call execute with valid filter", func() { 46 | 47 | output := bytes.NewBufferString("") 48 | cmd.SetOut(output) 49 | cmd.Flags().Set("filter", "name == x") // nolint 50 | err := cmd.Execute() 51 | 52 | Convey("Then I should get a generated command", func() { 53 | So(err, ShouldEqual, nil) 54 | So(output.String(), ShouldEqual, "2") 55 | }) 56 | }) 57 | 58 | Convey("When I call execute with invalid filter", func() { 59 | 60 | output := bytes.NewBufferString("") 61 | cmd.SetOut(output) 62 | cmd.Flags().Set("filter", "name...") // nolint 63 | err := cmd.Execute() 64 | 65 | Convey("Then I should get a generated command", func() { 66 | So(err, ShouldNotEqual, nil) 67 | So(err.Error(), ShouldContainSubstring, "unable to parse filter") 68 | }) 69 | }) 70 | }) 71 | 72 | Convey("Given I generate a count command that returns an error", t, func() { 73 | 74 | cmd, err := generateCountCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 75 | return nil, fmt.Errorf("boom") 76 | }) 77 | 78 | So(err, ShouldEqual, nil) 79 | assertCommandAndSetFlags(cmd) 80 | 81 | Convey("When I call execute", func() { 82 | 83 | err := cmd.Execute() 84 | 85 | Convey("Then I should get an error", func() { 86 | So(err, ShouldNotEqual, nil) 87 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 88 | }) 89 | }) 90 | }) 91 | 92 | } 93 | -------------------------------------------------------------------------------- /manipvortex/subscriber.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | "sync" 18 | 19 | "go.aporeto.io/elemental" 20 | "go.aporeto.io/manipulate" 21 | ) 22 | 23 | // vortexSubscriber is the memdb vortex subscriber implementation. 24 | type vortexSubscriber struct { 25 | v *vortexManipulator 26 | subscriberErrorChannel chan error 27 | subscriberEventChannel chan *elemental.Event 28 | subscriberStatusChannel chan manipulate.SubscriberStatus 29 | filter *elemental.PushConfig 30 | 31 | sync.RWMutex 32 | } 33 | 34 | // NewSubscriber creates a new vortex subscriber. 35 | func NewSubscriber(m manipulate.Manipulator, queueSize int) (manipulate.Subscriber, error) { 36 | 37 | v, ok := m.(*vortexManipulator) 38 | if !ok { 39 | return nil, fmt.Errorf("NewSubscriber only works with Vortex manipulator") 40 | } 41 | 42 | if !v.hasBackendSubscriber() { 43 | return nil, fmt.Errorf("vortex has no upstream subscriber: local subscriptions not supported") 44 | } 45 | 46 | s := &vortexSubscriber{ 47 | v: v, 48 | subscriberErrorChannel: make(chan error, queueSize), 49 | subscriberEventChannel: make(chan *elemental.Event, queueSize), 50 | subscriberStatusChannel: make(chan manipulate.SubscriberStatus, queueSize), 51 | } 52 | 53 | v.registerSubscriber(s) 54 | 55 | return s, nil 56 | } 57 | 58 | // Start starts the subscriber. 59 | func (s *vortexSubscriber) Start(ctx context.Context, e *elemental.PushConfig) { 60 | 61 | s.UpdateFilter(e) 62 | } 63 | 64 | // UpdateFilter updates the current push config. 65 | func (s *vortexSubscriber) UpdateFilter(e *elemental.PushConfig) { 66 | 67 | if e == nil { 68 | return 69 | } 70 | 71 | s.Lock() 72 | s.filter = e 73 | s.Unlock() 74 | 75 | s.v.updateFilter() 76 | } 77 | 78 | // Events returns the events channel. 79 | func (s *vortexSubscriber) Events() chan *elemental.Event { return s.subscriberEventChannel } 80 | 81 | // Errors returns the errors channel. 82 | func (s *vortexSubscriber) Errors() chan error { return s.subscriberErrorChannel } 83 | 84 | // Status returns the status channel. 85 | func (s *vortexSubscriber) Status() chan manipulate.SubscriberStatus { 86 | return s.subscriberStatusChannel 87 | } 88 | -------------------------------------------------------------------------------- /manipcli/delete.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.aporeto.io/elemental" 11 | "go.aporeto.io/manipulate" 12 | ) 13 | 14 | // generateDeleteCommandForIdentity generates the command to delete an object based on its identity. 15 | func generateDeleteCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 16 | 17 | cmd := &cobra.Command{ 18 | Use: fmt.Sprintf("%s ", identity.Name), 19 | Aliases: []string{identity.Category}, 20 | Short: "Delete an existing " + identity.Name, 21 | Args: cobra.ExactArgs(1), 22 | // Aliases: TODO: Missing alias from the spec file -> To be stored in the identity ?, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | 25 | fParam := viper.GetStringSlice("param") 26 | fTrackingID := viper.GetString(flagTrackingID) 27 | fOutput := viper.GetString(flagOutput) 28 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 29 | fOutputTemplate := viper.GetString(flagOutputTemplate) 30 | fForce := viper.GetBool(flagForce) 31 | 32 | manipulator, err := manipulatorMaker() 33 | if err != nil { 34 | return fmt.Errorf("unable to make manipulator: %w", err) 35 | } 36 | 37 | parameters, err := parametersToURLValues(fParam) 38 | if err != nil { 39 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 40 | } 41 | 42 | options := []manipulate.ContextOption{ 43 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 44 | manipulate.ContextOptionParameters(parameters), 45 | manipulate.ContextOptionFields(fFormatTypeColumn), 46 | manipulate.ContextOptionOverride(fForce), 47 | } 48 | 49 | ctx, cancel := context.WithTimeout(cmd.Context(), 20*time.Second) 50 | defer cancel() 51 | 52 | mctx := manipulate.NewContext(ctx, options...) 53 | 54 | identifiable, err := retrieveObjectByIDOrByName(mctx, manipulator, identity, args[0], modelManager) 55 | if err != nil { 56 | return fmt.Errorf("unable to retrieve %s: %w", identity.Name, err) 57 | } 58 | 59 | if err := manipulator.Delete(mctx, identifiable); err != nil { 60 | return fmt.Errorf("unable to delete %s: %w", identity.Name, err) 61 | } 62 | 63 | outputType := fOutput 64 | if fOutput == flagOutputDefault { 65 | outputType = flagOutputNone 66 | } 67 | 68 | result, err := formatObjects( 69 | prepareOutputFormat(outputType, formatTypeHash, fFormatTypeColumn, fOutputTemplate), 70 | false, 71 | identifiable, 72 | ) 73 | 74 | if err != nil { 75 | return fmt.Errorf("unable to format output: %w", err) 76 | } 77 | 78 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 79 | return nil 80 | }, 81 | } 82 | 83 | cmd.Flags().BoolP(flagForce, "", false, "Force deletion of protected object") 84 | 85 | return cmd, nil 86 | } 87 | -------------------------------------------------------------------------------- /maniptest/tokenmanager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniptest 13 | 14 | import ( 15 | "context" 16 | "sync" 17 | "testing" 18 | 19 | "go.aporeto.io/manipulate" 20 | ) 21 | 22 | type mockedTokenManagerMethods struct { 23 | issueMock func(context.Context) (string, error) 24 | runMock func(ctx context.Context, tokenCh chan string) 25 | } 26 | 27 | // A TestTokenManager is the interface of mockable test manipulator. 28 | type TestTokenManager interface { 29 | manipulate.TokenManager 30 | MockIssue(t *testing.T, impl func(context.Context) (string, error)) 31 | MockRun(t *testing.T, impl func(ctx context.Context, tokenCh chan string)) 32 | } 33 | 34 | // A testTokenManager is an empty TransactionalManipulator that can be easily mocked. 35 | type testTokenManager struct { 36 | mocks map[*testing.T]*mockedTokenManagerMethods 37 | lock *sync.Mutex 38 | currentTest *testing.T 39 | } 40 | 41 | // NewTestTokenManager returns a new TestTokenManager. 42 | func NewTestTokenManager() TestTokenManager { 43 | return &testTokenManager{ 44 | lock: &sync.Mutex{}, 45 | mocks: map[*testing.T]*mockedTokenManagerMethods{}, 46 | } 47 | } 48 | 49 | func (m *testTokenManager) MockIssue(t *testing.T, impl func(context.Context) (string, error)) { 50 | 51 | m.lock.Lock() 52 | defer m.lock.Unlock() 53 | 54 | m.currentMocks(t).issueMock = impl 55 | } 56 | 57 | func (m *testTokenManager) MockRun(t *testing.T, impl func(ctx context.Context, tokenCh chan string)) { 58 | 59 | m.lock.Lock() 60 | defer m.lock.Unlock() 61 | 62 | m.currentMocks(t).runMock = impl 63 | } 64 | 65 | func (m *testTokenManager) Issue(ctx context.Context) (string, error) { 66 | 67 | m.lock.Lock() 68 | defer m.lock.Unlock() 69 | 70 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.issueMock != nil { 71 | return mock.issueMock(ctx) 72 | } 73 | 74 | return "", nil 75 | } 76 | 77 | func (m *testTokenManager) Run(ctx context.Context, tokenCh chan string) { 78 | 79 | m.lock.Lock() 80 | defer m.lock.Unlock() 81 | 82 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.runMock != nil { 83 | mock.runMock(ctx, tokenCh) 84 | } 85 | } 86 | 87 | func (m *testTokenManager) currentMocks(t *testing.T) *mockedTokenManagerMethods { 88 | 89 | mocks := m.mocks[t] 90 | 91 | if mocks == nil { 92 | mocks = &mockedTokenManagerMethods{} 93 | m.mocks[t] = mocks 94 | } 95 | 96 | m.currentTest = t 97 | return mocks 98 | } 99 | -------------------------------------------------------------------------------- /manipvortex/reconciler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "sync" 16 | "testing" 17 | 18 | "go.aporeto.io/elemental" 19 | "go.aporeto.io/manipulate" 20 | ) 21 | 22 | // An Reconciler can be given to manipvortex to perform 23 | // pre write reconciliation. 24 | type Reconciler interface { 25 | 26 | // Reconcile is called before a write operation to 27 | // to determine if the objects needs reconciliation. If it returns 28 | // false, the objects are ignored. 29 | // If it returns an error, the error will be forwarded to the caller. 30 | // The Reconcile function may modify the objects to perform transformations. 31 | Reconcile(manipulate.Context, elemental.Operation, elemental.Identifiable) (elemental.Identifiable, bool, error) 32 | } 33 | 34 | // A TestReconciler is an Reconciler that can be used for 35 | // testing purposes. 36 | type TestReconciler interface { 37 | Reconciler 38 | MockReconcile(t *testing.T, impl func(manipulate.Context, elemental.Operation, elemental.Identifiable) (elemental.Identifiable, bool, error)) 39 | } 40 | 41 | type mockedReconcilerMethods struct { 42 | reconcileMock func(manipulate.Context, elemental.Operation, elemental.Identifiable) (elemental.Identifiable, bool, error) 43 | } 44 | 45 | type testReconciler struct { 46 | mocks map[*testing.T]*mockedReconcilerMethods 47 | lock *sync.Mutex 48 | currentTest *testing.T 49 | } 50 | 51 | // NewTestReconciler returns a new TestReconciler. 52 | func NewTestReconciler() TestReconciler { 53 | return &testReconciler{ 54 | lock: &sync.Mutex{}, 55 | mocks: map[*testing.T]*mockedReconcilerMethods{}, 56 | } 57 | } 58 | 59 | // MockPrefetch sets the mocked implementation of Reconcile. 60 | func (p *testReconciler) MockReconcile(t *testing.T, impl func(manipulate.Context, elemental.Operation, elemental.Identifiable) (elemental.Identifiable, bool, error)) { 61 | p.currentMocks(t).reconcileMock = impl 62 | } 63 | 64 | func (p *testReconciler) Reconcile(mctx manipulate.Context, op elemental.Operation, i elemental.Identifiable) (elemental.Identifiable, bool, error) { 65 | if mock := p.currentMocks(p.currentTest); mock != nil && mock.reconcileMock != nil { 66 | return mock.reconcileMock(mctx, op, i) 67 | } 68 | 69 | return i, true, nil 70 | } 71 | 72 | func (p *testReconciler) currentMocks(t *testing.T) *mockedReconcilerMethods { 73 | 74 | p.lock.Lock() 75 | defer p.lock.Unlock() 76 | 77 | mocks := p.mocks[t] 78 | 79 | if mocks == nil { 80 | mocks = &mockedReconcilerMethods{} 81 | p.mocks[t] = mocks 82 | } 83 | 84 | p.currentTest = t 85 | return mocks 86 | } 87 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.aporeto.io/manipulate 2 | 3 | go 1.22 4 | 5 | toolchain go1.22.6 6 | 7 | require ( 8 | go.aporeto.io/elemental v1.123.1-0.20240822212917-6f8c7be6698c 9 | go.aporeto.io/wsc v1.51.0 10 | ) 11 | 12 | require ( 13 | github.com/Masterminds/sprig/v3 v3.2.3 14 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 15 | github.com/ghodss/yaml v1.0.0 16 | github.com/gofrs/uuid v4.4.0+incompatible 17 | github.com/golang/mock v1.6.0 18 | github.com/gorilla/websocket v1.5.0 19 | github.com/hashicorp/go-memdb v1.3.4 20 | github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f 21 | github.com/mitchellh/copystructure v1.2.0 22 | github.com/olekukonko/tablewriter v0.0.5 23 | github.com/opentracing/opentracing-go v1.2.0 24 | github.com/smartystreets/goconvey v1.7.2 25 | github.com/spf13/cobra v1.6.1 26 | github.com/spf13/pflag v1.0.5 27 | github.com/spf13/viper v1.15.0 28 | go.mongodb.org/mongo-driver v1.16.0 29 | go.uber.org/zap v1.24.0 30 | golang.org/x/sync v0.7.0 31 | golang.org/x/sys v0.19.0 32 | k8s.io/helm v2.17.0+incompatible 33 | ) 34 | 35 | require ( 36 | github.com/Masterminds/goutils v1.1.1 // indirect 37 | github.com/Masterminds/semver/v3 v3.2.0 // indirect 38 | github.com/fatih/color v1.14.1 // indirect 39 | github.com/fsnotify/fsnotify v1.6.0 // indirect 40 | github.com/golang/snappy v0.0.4 // indirect 41 | github.com/google/uuid v1.3.0 // indirect 42 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect 43 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 44 | github.com/hashicorp/golang-lru v0.5.4 // indirect 45 | github.com/hashicorp/hcl v1.0.0 // indirect 46 | github.com/huandu/xstrings v1.4.0 // indirect 47 | github.com/imdario/mergo v0.3.13 // indirect 48 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 49 | github.com/jtolds/gls v4.20.0+incompatible // indirect 50 | github.com/klauspost/compress v1.13.6 // indirect 51 | github.com/magiconair/properties v1.8.7 // indirect 52 | github.com/mattn/go-colorable v0.1.13 // indirect 53 | github.com/mattn/go-isatty v0.0.17 // indirect 54 | github.com/mattn/go-runewidth v0.0.14 // indirect 55 | github.com/mitchellh/mapstructure v1.5.0 // indirect 56 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 57 | github.com/montanaflynn/stats v0.7.1 // indirect 58 | github.com/pelletier/go-toml/v2 v2.0.6 // indirect 59 | github.com/rivo/uniseg v0.4.3 // indirect 60 | github.com/shopspring/decimal v1.3.1 // indirect 61 | github.com/smartystreets/assertions v1.2.0 // indirect 62 | github.com/spf13/afero v1.9.3 // indirect 63 | github.com/spf13/cast v1.5.0 // indirect 64 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 65 | github.com/subosito/gotenv v1.4.2 // indirect 66 | github.com/ugorji/go/codec v1.2.9 // indirect 67 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 68 | github.com/xdg-go/scram v1.1.2 // indirect 69 | github.com/xdg-go/stringprep v1.0.4 // indirect 70 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 71 | go.uber.org/atomic v1.10.0 // indirect 72 | go.uber.org/multierr v1.9.0 // indirect 73 | golang.org/x/crypto v0.22.0 // indirect 74 | golang.org/x/text v0.14.0 // indirect 75 | gopkg.in/ini.v1 v1.67.0 // indirect 76 | gopkg.in/yaml.v2 v2.4.0 // indirect 77 | gopkg.in/yaml.v3 v3.0.1 // indirect 78 | ) 79 | -------------------------------------------------------------------------------- /manipmongo/internal/mock_attribute_specifiable.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: go.aporeto.io/elemental (interfaces: AttributeSpecifiable) 3 | 4 | // Package internal is a generated GoMock package. 5 | package internal 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | elemental "go.aporeto.io/elemental" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockAttributeSpecifiable is a mock of AttributeSpecifiable interface 14 | type MockAttributeSpecifiable struct { 15 | ctrl *gomock.Controller 16 | recorder *MockAttributeSpecifiableMockRecorder 17 | } 18 | 19 | // MockAttributeSpecifiableMockRecorder is the mock recorder for MockAttributeSpecifiable 20 | type MockAttributeSpecifiableMockRecorder struct { 21 | mock *MockAttributeSpecifiable 22 | } 23 | 24 | // NewMockAttributeSpecifiable creates a new mock instance 25 | func NewMockAttributeSpecifiable(ctrl *gomock.Controller) *MockAttributeSpecifiable { 26 | mock := &MockAttributeSpecifiable{ctrl: ctrl} 27 | mock.recorder = &MockAttributeSpecifiableMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockAttributeSpecifiable) EXPECT() *MockAttributeSpecifiableMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // AttributeSpecifications mocks base method 37 | func (m *MockAttributeSpecifiable) AttributeSpecifications() map[string]elemental.AttributeSpecification { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "AttributeSpecifications") 40 | ret0, _ := ret[0].(map[string]elemental.AttributeSpecification) 41 | return ret0 42 | } 43 | 44 | // AttributeSpecifications indicates an expected call of AttributeSpecifications 45 | func (mr *MockAttributeSpecifiableMockRecorder) AttributeSpecifications() *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttributeSpecifications", reflect.TypeOf((*MockAttributeSpecifiable)(nil).AttributeSpecifications)) 48 | } 49 | 50 | // SpecificationForAttribute mocks base method 51 | func (m *MockAttributeSpecifiable) SpecificationForAttribute(arg0 string) elemental.AttributeSpecification { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "SpecificationForAttribute", arg0) 54 | ret0, _ := ret[0].(elemental.AttributeSpecification) 55 | return ret0 56 | } 57 | 58 | // SpecificationForAttribute indicates an expected call of SpecificationForAttribute 59 | func (mr *MockAttributeSpecifiableMockRecorder) SpecificationForAttribute(arg0 any) *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpecificationForAttribute", reflect.TypeOf((*MockAttributeSpecifiable)(nil).SpecificationForAttribute), arg0) 62 | } 63 | 64 | // ValueForAttribute mocks base method 65 | func (m *MockAttributeSpecifiable) ValueForAttribute(arg0 string) any { 66 | m.ctrl.T.Helper() 67 | ret := m.ctrl.Call(m, "ValueForAttribute", arg0) 68 | ret0, _ := ret[0].(any) 69 | return ret0 70 | } 71 | 72 | // ValueForAttribute indicates an expected call of ValueForAttribute 73 | func (mr *MockAttributeSpecifiableMockRecorder) ValueForAttribute(arg0 any) *gomock.Call { 74 | mr.mock.ctrl.T.Helper() 75 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValueForAttribute", reflect.TypeOf((*MockAttributeSpecifiable)(nil).ValueForAttribute), arg0) 76 | } 77 | -------------------------------------------------------------------------------- /manipcli/create_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateCreateCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a create command", t, func() { 20 | 21 | m := maniptest.NewTestManipulator() 22 | m.MockCreate(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 23 | o := object.(*testmodel.Task) 24 | o.ID = "617aec75a829de0001da2032" 25 | return nil 26 | }) 27 | 28 | cmd, err := generateCreateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 29 | return m, nil 30 | }) 31 | 32 | So(err, ShouldEqual, nil) 33 | assertCommandAndSetFlags(cmd) 34 | 35 | Convey("When I call execute with a json output", func() { 36 | 37 | cmd.Flags().Set("name", "my task") // nolint 38 | cmd.Flags().Set("output", "json") // nolint 39 | output := bytes.NewBufferString("") 40 | cmd.SetOut(output) 41 | err := cmd.Execute() 42 | 43 | Convey("Then I should get a generated command", func() { 44 | So(err, ShouldEqual, nil) 45 | So(output.String(), ShouldEqual, `{ 46 | "ID": "617aec75a829de0001da2032", 47 | "description": "", 48 | "name": "my task", 49 | "parentID": "", 50 | "parentType": "", 51 | "status": "TODO" 52 | }`) 53 | }) 54 | }) 55 | }) 56 | 57 | Convey("Given I generate a create command that returns an error", t, func() { 58 | 59 | cmd, err := generateCreateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 60 | return nil, fmt.Errorf("boom") 61 | }) 62 | 63 | So(err, ShouldEqual, nil) 64 | assertCommandAndSetFlags(cmd) 65 | 66 | Convey("When I call execute", func() { 67 | 68 | cmd.Flags().Set("name", "my task") // nolint 69 | err := cmd.Execute() 70 | 71 | Convey("Then I should get an error", func() { 72 | So(err, ShouldNotEqual, nil) 73 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 74 | }) 75 | }) 76 | }) 77 | 78 | Convey("Given a manipulator that fails", t, func() { 79 | 80 | m := maniptest.NewTestManipulator() 81 | m.MockCreate(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 82 | return fmt.Errorf("boom") 83 | }) 84 | 85 | cmd, err := generateCreateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 86 | return m, nil 87 | }) 88 | 89 | So(err, ShouldEqual, nil) 90 | assertCommandAndSetFlags(cmd) 91 | 92 | Convey("When I call execute with a json output", func() { 93 | 94 | cmd.Flags().Set("name", "my task") // nolint 95 | cmd.Flags().Set("output", "json") // nolint 96 | output := bytes.NewBufferString("") 97 | cmd.SetOut(output) 98 | err := cmd.Execute() 99 | 100 | Convey("Then I should get a generated command", func() { 101 | So(err, ShouldNotEqual, nil) 102 | So(err.Error(), ShouldEqual, "unable to create task: boom") 103 | }) 104 | }) 105 | }) 106 | 107 | } 108 | -------------------------------------------------------------------------------- /manipcli/count.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.aporeto.io/elemental" 11 | "go.aporeto.io/manipulate" 12 | ) 13 | 14 | // generateCountCommandForIdentity generates the command to count all objects based on its identity. 15 | func generateCountCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 16 | 17 | cmd := &cobra.Command{ 18 | Use: identity.Name, 19 | Aliases: []string{identity.Category}, 20 | Short: "Count " + identity.Category, 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | 23 | fParam := viper.GetStringSlice("param") 24 | fTrackingID := viper.GetString(flagTrackingID) 25 | fRecursive := viper.GetBool(flagRecursive) 26 | fFilter := viper.GetString(flagFilter) 27 | fOutput := viper.GetString(flagOutput) 28 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 29 | fOutputTemplate := viper.GetString(flagOutputTemplate) 30 | 31 | manipulator, err := manipulatorMaker() 32 | if err != nil { 33 | return fmt.Errorf("unable to make manipulator: %w", err) 34 | } 35 | 36 | parameters, err := parametersToURLValues(fParam) 37 | if err != nil { 38 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 39 | } 40 | 41 | options := []manipulate.ContextOption{ 42 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 43 | manipulate.ContextOptionParameters(parameters), 44 | manipulate.ContextOptionRecursive(fRecursive), 45 | manipulate.ContextOptionReadConsistency(manipulate.ReadConsistencyStrong), 46 | } 47 | 48 | if fFilter != "" { 49 | f, err := elemental.NewFilterFromString(fFilter) 50 | if err != nil { 51 | return fmt.Errorf("unable to parse filter %s: %s", fFilter, err) 52 | } 53 | options = append(options, manipulate.ContextOptionFilter(f)) 54 | } 55 | 56 | if viper.IsSet(flagParent) { 57 | parentName, parentID, err := splitParentInfo(viper.GetString(flagParent)) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | parent := modelManager.IdentifiableFromString(parentName) 63 | if parent == nil { 64 | return fmt.Errorf("unknown identity %s", parentName) 65 | } 66 | parent.SetIdentifier(parentID) 67 | options = append(options, manipulate.ContextOptionParent(parent)) 68 | } 69 | 70 | ctx, cancel := context.WithTimeout(cmd.Context(), 20*time.Second) 71 | defer cancel() 72 | 73 | mctx := manipulate.NewContext(ctx, options...) 74 | num, err := manipulator.Count(mctx, identity) 75 | if err != nil { 76 | return fmt.Errorf("unable to count %s: %w", identity.Category, err) 77 | } 78 | 79 | outputType := fOutput 80 | if fOutput == flagOutputDefault { 81 | outputType = flagOutputNone 82 | } 83 | 84 | result, err := formatMaps( 85 | prepareOutputFormat(outputType, formatTypeCount, fFormatTypeColumn, fOutputTemplate), 86 | false, 87 | []map[string]any{{formatTypeCount: num}}, 88 | ) 89 | 90 | if err != nil { 91 | return fmt.Errorf("unable to format output: %w", err) 92 | } 93 | 94 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 95 | 96 | return nil 97 | 98 | }, 99 | } 100 | 101 | cmd.Flags().BoolP(flagRecursive, "r", false, "List all objects from the current namespace and all child namespaces.") 102 | cmd.Flags().StringP(flagFilter, "f", "", "Query filter.") 103 | cmd.Flags().StringP(flagParent, "", "", "Provide information about parent resource. Format `name/ID`") 104 | 105 | return cmd, nil 106 | } 107 | -------------------------------------------------------------------------------- /maniphttp/subscriber_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniphttp 13 | 14 | import ( 15 | "crypto/tls" 16 | "testing" 17 | 18 | // nolint:revive // Allow dot imports for readability in tests 19 | . "github.com/smartystreets/goconvey/convey" 20 | "go.aporeto.io/manipulate/maniptest" 21 | ) 22 | 23 | func TestNewSubscriberOption(t *testing.T) { 24 | 25 | Convey("Given I have manipulator", t, func() { 26 | 27 | m := &httpManipulator{ 28 | url: "https://toto.com", 29 | namespace: "mns", 30 | tlsConfig: &tls.Config{}, 31 | } 32 | 33 | Convey("When I newSubscribeConfig ", func() { 34 | 35 | cfg := newSubscribeConfig(m) 36 | 37 | Convey("Then cfg should be correct", func() { 38 | So(cfg.recursive, ShouldBeFalse) 39 | So(cfg.endpoint, ShouldEqual, "events") 40 | So(cfg.namespace, ShouldEqual, "mns") 41 | So(cfg.tlsConfig, ShouldEqual, m.tlsConfig) 42 | So(cfg.supportErrorEvents, ShouldBeFalse) 43 | }) 44 | }) 45 | }) 46 | } 47 | 48 | func TestOptions(t *testing.T) { 49 | 50 | m := &httpManipulator{ 51 | namespace: "mns", 52 | tlsConfig: &tls.Config{}, 53 | } 54 | 55 | Convey("SubscriberOptionRecursive should work", t, func() { 56 | cfg := newSubscribeConfig(m) 57 | SubscriberOptionRecursive(true)(&cfg) 58 | So(cfg.recursive, ShouldBeTrue) 59 | }) 60 | 61 | Convey("SubscriberOptionNamespace should work", t, func() { 62 | cfg := newSubscribeConfig(m) 63 | SubscriberOptionNamespace("/toto")(&cfg) 64 | So(cfg.namespace, ShouldEqual, "/toto") 65 | }) 66 | 67 | Convey("SubscriberOptionEndpoint should work", t, func() { 68 | cfg := newSubscribeConfig(m) 69 | SubscriberOptionEndpoint("/labas/")(&cfg) 70 | So(cfg.endpoint, ShouldEqual, "labas") 71 | }) 72 | 73 | Convey("SubscriberSendCredentialsAsCookie should work", t, func() { 74 | cfg := newSubscribeConfig(m) 75 | SubscriberSendCredentialsAsCookie("creds")(&cfg) 76 | So(cfg.credentialCookieKey, ShouldEqual, "creds") 77 | }) 78 | 79 | Convey("SubscriberOptionSupportErrorEvents should work", t, func() { 80 | cfg := newSubscribeConfig(m) 81 | SubscriberOptionSupportErrorEvents()(&cfg) 82 | So(cfg.supportErrorEvents, ShouldBeTrue) 83 | }) 84 | } 85 | 86 | func TestNewSubscriber(t *testing.T) { 87 | 88 | m := &httpManipulator{ 89 | url: "https://toto.com", 90 | namespace: "mns", 91 | tlsConfig: &tls.Config{}, 92 | } 93 | 94 | Convey("Creating a new subscriber should work", t, func() { 95 | 96 | out := NewSubscriber(m, SubscriberOptionEndpoint("/")) 97 | 98 | So(out, ShouldNotBeNil) 99 | }) 100 | 101 | Convey("Creating a new subscriber with a nil option should panic", t, func() { 102 | 103 | So(func() { NewSubscriber(m, nil) }, ShouldPanicWith, "nil passed as subscriber option") 104 | }) 105 | 106 | Convey("Creating a new subscriber with a non http manipulator should panic", t, func() { 107 | 108 | So(func() { NewSubscriber(maniptest.NewTestManipulator(), nil) }, ShouldPanicWith, "You must pass a HTTP manipulator to maniphttp.NewSubscriber or maniphttp.NewSubscriberWithEndpoint") 109 | }) 110 | 111 | } 112 | -------------------------------------------------------------------------------- /maniptest/subscriber_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniptest 13 | 14 | import ( 15 | "context" 16 | "testing" 17 | 18 | // nolint:revive // Allow dot imports for readability in tests 19 | . "github.com/smartystreets/goconvey/convey" 20 | "go.aporeto.io/elemental" 21 | "go.aporeto.io/manipulate" 22 | ) 23 | 24 | func TestTestSubscriber_MockStart(t *testing.T) { 25 | 26 | Convey("Given I have TestSubscriber", t, func() { 27 | 28 | m := NewTestSubscriber() 29 | 30 | Convey("When I call Start with a panic mock", func() { 31 | m.MockStart(t, func(ctx context.Context, filter *elemental.PushConfig) { 32 | panic("test") 33 | }) 34 | 35 | So(func() { m.Start(context.Background(), nil) }, ShouldPanic) 36 | }) 37 | }) 38 | } 39 | 40 | func TestTestSubscriber_MockUpdateFilter(t *testing.T) { 41 | Convey("Given I have TestSubscriber", t, func() { 42 | 43 | m := NewTestSubscriber() 44 | 45 | Convey("When I call UpdateFilter with a panic mock", func() { 46 | m.MockUpdateFilter(t, func(filter *elemental.PushConfig) { 47 | panic("test") 48 | }) 49 | 50 | So(func() { m.UpdateFilter(nil) }, ShouldPanic) 51 | }) 52 | }) 53 | } 54 | 55 | func TestTestSubscriber_Events(t *testing.T) { 56 | 57 | Convey("Given a TestSubscriber", t, func() { 58 | 59 | m := NewTestSubscriber() 60 | 61 | Convey("When I call Events with no mock, it should return nil", func() { 62 | e := m.Events() 63 | So(e, ShouldBeNil) 64 | }) 65 | 66 | Convey("When I call Events with an Event channel mock", func() { 67 | eventChannel := make(chan *elemental.Event) 68 | 69 | m.MockEvents(t, func() chan *elemental.Event { 70 | return eventChannel 71 | }) 72 | 73 | e := m.Events() 74 | So(e, ShouldNotBeNil) 75 | So(e, ShouldResemble, eventChannel) 76 | }) 77 | }) 78 | } 79 | 80 | func TestTestSubscriber_Errors(t *testing.T) { 81 | 82 | Convey("Given a TestSubscriber", t, func() { 83 | 84 | m := NewTestSubscriber() 85 | 86 | Convey("When I call Errors with no mock, it should return nil", func() { 87 | e := m.Errors() 88 | So(e, ShouldBeNil) 89 | }) 90 | 91 | Convey("When I call Events with an error channel mock", func() { 92 | errorChannel := make(chan error) 93 | 94 | m.MockErrors(t, func() chan error { 95 | return errorChannel 96 | }) 97 | 98 | e := m.Errors() 99 | So(e, ShouldNotBeNil) 100 | So(e, ShouldResemble, errorChannel) 101 | }) 102 | }) 103 | } 104 | 105 | func TestTestSubscriber_Status(t *testing.T) { 106 | 107 | Convey("Given a TestSubscriber", t, func() { 108 | 109 | m := NewTestSubscriber() 110 | 111 | Convey("When I call Status with no mock, it should return nil", func() { 112 | e := m.Status() 113 | So(e, ShouldBeNil) 114 | }) 115 | 116 | Convey("When I call Status with an status channel mock", func() { 117 | statusChannel := make(chan manipulate.SubscriberStatus) 118 | 119 | m.MockStatus(t, func() chan manipulate.SubscriberStatus { 120 | return statusChannel 121 | }) 122 | 123 | e := m.Status() 124 | So(e, ShouldNotBeNil) 125 | So(e, ShouldResemble, statusChannel) 126 | }) 127 | 128 | }) 129 | } 130 | -------------------------------------------------------------------------------- /manipvortex/prefetcher_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | "testing" 18 | 19 | "go.aporeto.io/elemental" 20 | testmodel "go.aporeto.io/elemental/test/model" 21 | "go.aporeto.io/manipulate" 22 | 23 | // nolint:revive // Allow dot imports for readability in tests 24 | . "github.com/smartystreets/goconvey/convey" 25 | ) 26 | 27 | func TestTestPrefetcher(t *testing.T) { 28 | 29 | Convey("Given I create a new TestPrefetcher", t, func() { 30 | 31 | p := NewTestPrefetcher() 32 | 33 | Convey("Then it should be initialized", func() { 34 | So(p, ShouldImplement, (*TestPrefetcher)(nil)) 35 | So(p.(*testPrefetcher).lock, ShouldNotBeNil) 36 | So(p.(*testPrefetcher).mocks, ShouldNotBeNil) 37 | }) 38 | 39 | Convey("When I call Prefetch", func() { 40 | 41 | out, err := p.Prefetch(context.Background(), elemental.OperationRetrieve, elemental.EmptyIdentity, nil, nil) 42 | 43 | Convey("Then err should be nil", func() { 44 | So(err, ShouldBeNil) 45 | }) 46 | 47 | Convey("Then out should be nil", func() { 48 | So(out, ShouldBeNil) 49 | }) 50 | }) 51 | 52 | Convey("When I call WarmUp", func() { 53 | 54 | out, err := p.WarmUp(context.Background(), nil, nil, elemental.EmptyIdentity) 55 | 56 | Convey("Then err should be nil", func() { 57 | So(err, ShouldBeNil) 58 | }) 59 | 60 | Convey("Then out should be nil", func() { 61 | So(out, ShouldBeNil) 62 | }) 63 | 64 | }) 65 | 66 | Convey("When I call Flush", func() { 67 | 68 | p.Flush() 69 | 70 | Convey("Then nothing should happen", func() {}) 71 | 72 | }) 73 | 74 | Convey("When I mock and call Prefetch", func() { 75 | 76 | p.MockPrefetch(t, func(context.Context, elemental.Operation, elemental.Identity, manipulate.Manipulator, manipulate.Context) (elemental.Identifiables, error) { 77 | return testmodel.ListsList{}, fmt.Errorf("boom") 78 | }) 79 | 80 | out, err := p.Prefetch(context.Background(), elemental.OperationRetrieve, elemental.EmptyIdentity, nil, nil) 81 | 82 | Convey("Then err should not be nil", func() { 83 | So(err, ShouldNotBeNil) 84 | So(err.Error(), ShouldEqual, "boom") 85 | }) 86 | 87 | Convey("Then out should not be nil", func() { 88 | So(out, ShouldNotBeNil) 89 | }) 90 | }) 91 | 92 | Convey("When I mock and call WarmUp", func() { 93 | 94 | p.MockWarmUp(t, func(context.Context, manipulate.Manipulator, elemental.ModelManager, elemental.Identity) (elemental.Identifiables, error) { 95 | return testmodel.ListsList{}, fmt.Errorf("boom") 96 | }) 97 | 98 | out, err := p.WarmUp(context.Background(), nil, nil, elemental.EmptyIdentity) 99 | 100 | Convey("Then err should not be nil", func() { 101 | So(err, ShouldNotBeNil) 102 | So(err.Error(), ShouldEqual, "boom") 103 | }) 104 | 105 | Convey("Then out should not be nil", func() { 106 | So(out, ShouldNotBeNil) 107 | }) 108 | }) 109 | 110 | Convey("When I mock and call Flush", func() { 111 | 112 | p.MockFlush(t, func() { panic("boom") }) 113 | 114 | Convey("The it should panic", func() { 115 | So(func() { p.Flush() }, ShouldPanicWith, "boom") 116 | }) 117 | 118 | }) 119 | }) 120 | } 121 | -------------------------------------------------------------------------------- /manipvortex/subscriber_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "context" 16 | "testing" 17 | 18 | "go.aporeto.io/manipulate" 19 | 20 | // nolint:revive // Allow dot imports for readability in tests 21 | . "github.com/smartystreets/goconvey/convey" 22 | testmodel "go.aporeto.io/elemental/test/model" 23 | "go.aporeto.io/manipulate/maniptest" 24 | ) 25 | 26 | func Test_NewSubscriber(t *testing.T) { 27 | 28 | ctx, cancel := context.WithCancel(context.Background()) 29 | defer cancel() 30 | 31 | Convey("Given a valid manipulator with a backend", t, func() { 32 | m := maniptest.NewTestManipulator() 33 | s := maniptest.NewTestSubscriber() 34 | d, err := newDatastore() 35 | So(err, ShouldBeNil) 36 | 37 | v, err := New( 38 | ctx, 39 | d, 40 | newIdentityProcessor(manipulate.ReadConsistencyEventual, manipulate.WriteConsistencyStrong), 41 | testmodel.Manager(), 42 | OptionUpstreamManipulator(m), 43 | OptionUpstreamSubscriber(s), 44 | ) 45 | So(err, ShouldBeNil) 46 | 47 | Convey("When I request a new subscriber, it should be valid", func() { 48 | s, err := NewSubscriber(v, 100) 49 | So(err, ShouldBeNil) 50 | So(s, ShouldNotBeNil) 51 | }) 52 | }) 53 | 54 | Convey("Given an invalid manipulator, the method should error", t, func() { 55 | _, err := NewSubscriber(nil, 100) 56 | So(err, ShouldNotBeNil) 57 | }) 58 | 59 | Convey("Given a valid manipulator with no upstream subscriber, it should error", t, func() { 60 | m := maniptest.NewTestManipulator() 61 | d, err := newDatastore() 62 | So(err, ShouldBeNil) 63 | 64 | v, err := New( 65 | ctx, 66 | d, 67 | newIdentityProcessor(manipulate.ReadConsistencyEventual, manipulate.WriteConsistencyStrong), 68 | testmodel.Manager(), 69 | OptionUpstreamManipulator(m), 70 | ) 71 | So(err, ShouldBeNil) 72 | 73 | Convey("When I request a new subscriber, it should be valid", func() { 74 | _, err := NewSubscriber(v, 100) 75 | So(err, ShouldNotBeNil) 76 | }) 77 | }) 78 | } 79 | 80 | func Test_SubscriberMethods(t *testing.T) { 81 | 82 | ctx, cancel := context.WithCancel(context.Background()) 83 | defer cancel() 84 | 85 | Convey("Given a valid manipulator with a backend subscriber", t, func() { 86 | m := maniptest.NewTestManipulator() 87 | us := maniptest.NewTestSubscriber() 88 | d, err := newDatastore() 89 | So(err, ShouldBeNil) 90 | 91 | v, err := New( 92 | ctx, 93 | d, 94 | newIdentityProcessor(manipulate.ReadConsistencyEventual, manipulate.WriteConsistencyStrong), 95 | testmodel.Manager(), 96 | OptionUpstreamManipulator(m), 97 | OptionUpstreamSubscriber(us), 98 | ) 99 | So(err, ShouldBeNil) 100 | So(v, ShouldNotBeNil) 101 | s, err := NewSubscriber(v, 100) 102 | So(err, ShouldBeNil) 103 | 104 | Convey("When I retrieve the events channel, it should not be nil", func() { 105 | ch := s.Events() 106 | So(ch, ShouldNotBeNil) 107 | So(cap(ch), ShouldEqual, 100) 108 | }) 109 | 110 | Convey("When I retrieve the errors channel, it should not be nil", func() { 111 | ch := s.Errors() 112 | So(ch, ShouldNotBeNil) 113 | So(cap(ch), ShouldEqual, 100) 114 | }) 115 | 116 | Convey("When I retrieve the status channel, it should not be nil", func() { 117 | ch := s.Status() 118 | So(ch, ShouldNotBeNil) 119 | So(cap(ch), ShouldEqual, 100) 120 | }) 121 | }) 122 | } 123 | -------------------------------------------------------------------------------- /manipcli/cli_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | // nolint:revive // Allow dot imports for readability in tests 8 | . "github.com/smartystreets/goconvey/convey" 9 | "github.com/spf13/cobra" 10 | "github.com/spf13/viper" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | ) 13 | 14 | func assertIdentityCommand(cmd *cobra.Command, expectedSubCommands []string) { 15 | 16 | subCmd := cmd.Commands() 17 | So(len(subCmd), ShouldEqual, len(expectedSubCommands)) 18 | sort.Slice(subCmd, func(i, j int) bool { 19 | return subCmd[i].Name() < subCmd[j].Name() 20 | }) 21 | So(subCmd[0].Name(), ShouldEqual, expectedSubCommands[0]) 22 | So(subCmd[1].Name(), ShouldEqual, expectedSubCommands[1]) 23 | } 24 | 25 | func assertCommandAndSetFlags(cmd *cobra.Command) { 26 | 27 | So(cmd, ShouldNotEqual, nil) 28 | cmd.Flags().AddFlagSet(ManipulatorFlagSet()) 29 | cmd.Flags().StringP(flagOutput, "o", "default", "Format to used print output. Options are 'table', 'json', 'yaml', 'none', 'template' or 'default'.") 30 | err := viper.BindPFlags(cmd.Flags()) 31 | So(err, ShouldEqual, nil) 32 | } 33 | 34 | func Test_OptionIgnoreIdentities(t *testing.T) { 35 | 36 | Convey("Given a configuration", t, func() { 37 | cfg := &cliConfig{} 38 | 39 | Convey("When I call OptionIgnoreIdentities", func() { 40 | 41 | OptionIgnoreIdentities(testmodel.TaskIdentity)(cfg) 42 | 43 | Convey("Then I should have the configuration set", func() { 44 | So(cfg.ignoredIdentities, ShouldResemble, map[string]struct{}{ 45 | testmodel.TaskIdentity.Name: {}, 46 | }) 47 | }) 48 | }) 49 | }) 50 | } 51 | 52 | func Test_New(t *testing.T) { 53 | 54 | Convey("Given a valid model manager and a manipulator maker", t, func() { 55 | 56 | cmd := &cobra.Command{ 57 | Use: "test", 58 | Short: "this is a test", 59 | PersistentPreRunE: func(cmd *cobra.Command, args []string) error { 60 | return nil 61 | }, 62 | } 63 | cmd.Flags().AddFlagSet(ManipulatorFlagSet()) 64 | err := viper.BindPFlags(cmd.Flags()) 65 | So(err, ShouldEqual, nil) 66 | 67 | cmd.Flags().Set(flagAPI, "https://test.com") // nolint 68 | cmd.Flags().Set(flagNamespace, "/test") // nolint 69 | cmd.Flags().Set(flagEncoding, "msgpack") // nolint 70 | cmd.Flags().Set(flagToken, "token1234") // nolint 71 | cmd.Flags().Set(flagAPISkipVerify, "true") // nolint 72 | 73 | mmanager := testmodel.Manager() 74 | mmaker := ManipulatorMakerFromFlags() 75 | 76 | Convey("When I call New", func() { 77 | 78 | genCmd := New(mmanager, mmaker, OptionIgnoreIdentities(testmodel.UserIdentity)) 79 | 80 | Convey("Then I should get a generated command", func() { 81 | So(genCmd, ShouldNotEqual, nil) 82 | So(genCmd.Use, ShouldEqual, "api [command] [flags]") 83 | So(genCmd.HasSubCommands(), ShouldEqual, true) 84 | 85 | subCommands := genCmd.Commands() 86 | So(len(subCommands), ShouldEqual, 8) 87 | So(subCommands[0].Name(), ShouldEqual, "count") 88 | So(subCommands[1].Name(), ShouldEqual, "create") 89 | So(subCommands[2].Name(), ShouldEqual, "delete") 90 | So(subCommands[3].Name(), ShouldEqual, "delete-many") 91 | So(subCommands[4].Name(), ShouldEqual, "get") 92 | So(subCommands[5].Name(), ShouldEqual, "list") 93 | So(subCommands[6].Name(), ShouldEqual, "listen") 94 | So(subCommands[7].Name(), ShouldEqual, "update") 95 | 96 | expectedSubCommands := []string{"list", "task"} 97 | 98 | assertIdentityCommand(subCommands[0], expectedSubCommands) 99 | assertIdentityCommand(subCommands[1], expectedSubCommands) 100 | assertIdentityCommand(subCommands[2], expectedSubCommands) 101 | assertIdentityCommand(subCommands[3], expectedSubCommands) 102 | assertIdentityCommand(subCommands[4], expectedSubCommands) 103 | assertIdentityCommand(subCommands[5], expectedSubCommands) 104 | assertIdentityCommand(subCommands[7], expectedSubCommands) 105 | 106 | // Listen 107 | listenSubCommands := subCommands[6].Commands() 108 | So(len(listenSubCommands), ShouldEqual, 0) 109 | }) 110 | }) 111 | }) 112 | 113 | } 114 | -------------------------------------------------------------------------------- /manipvortex/README.md: -------------------------------------------------------------------------------- 1 | # Vortex - Local Cache for Elemental Objects 2 | 3 | Vortex is a generic elemental cache, currently backed by MemDB. Other implementations 4 | are possible in the future. 5 | 6 | Vortex requires three components: 7 | 8 | * A manipulator that allows Vortex to synchronize objects in the cache with a remote backend. 9 | 10 | * An optional subscriber that allows vortex to synchronize events from an upstream susbscriber. 11 | 12 | * A local datastore where it will cache objects. The datastore must be properly configured with any indexes as required by your queries. 13 | 14 | The downstream manipulator is optional and vortex can also be used a standalone embedded 15 | database that obays the manipulate interfaces. Note, that vortex supports both the 16 | manipulator and subscriber interfaces and it will propagate any events from the 17 | downstream to upstream callers. 18 | 19 | For every object you must provide a processor configuration that will define any hook configuration. Separate hooks are possible before its transaction: 20 | 21 | * RemoteHook will be applied before the transaction is send upstream. 22 | 23 | * LocalHost will be applied before the transaction is commited locally. 24 | 25 | * RetrieveManyHook is applied before retrieve many operations. 26 | 27 | For every identity, you can configure the cache mode: 28 | 29 | * Write-through indicates that the object will be first commited downstream and only if it succeeds it will be committed localy. 30 | 31 | *Write-back indicates that the object will be stored in a local queue and later it will be send to the downstream database. 32 | 33 | You can take a look at the test files for examples. 34 | 35 | Here is a simple example: 36 | 37 | ```go 38 | // NewMemoryDB will create the DB. 39 | func NewMemoryDB( 40 | ctx context.Context, 41 | b manipulate.TokenManager, 42 | api string, 43 | model elemental.ModelManager, 44 | ) (manipulate.Manipulator, context.CancelFunc, error) { 45 | 46 | connectContext, cancel := context.WithCancel(ctx) 47 | 48 | m, err := maniphttp.New( 49 | connectContext, 50 | api, 51 | maniphttp.OptionNamespace(namespace), 52 | maniphttp.OptionTokenManager(b), 53 | ) 54 | 55 | if err != nil { 56 | cancel() 57 | return nil, nil, err 58 | } 59 | 60 | subscriber := maniphttp.NewSubscriber(m, true) 61 | subscriber.Start(ctx, nil) 62 | 63 | indexConfig := map[string]*config.MemDBIdentity{ 64 | testmodel.ListIdentity.Category: &config.MemDBIdentity{ 65 | Identity: testmodel.ListIdentity, 66 | Indexes: []*config.IndexConfig{ 67 | &config.IndexConfig{ 68 | Name: "id", 69 | Type: config.String, 70 | Unique: true, 71 | Attribute: "ID", 72 | }, 73 | &config.IndexConfig{ 74 | Name: "Name", 75 | Type: config.String, 76 | Attribute: "Name", 77 | , 78 | &config.IndexConfig{ 79 | Name: "Slice", 80 | Type: config.Slice, 81 | Attribute: "Slice", 82 | }, 83 | }, 84 | }, 85 | } 86 | 87 | // Create a data store, register the identities and start it. 88 | datastore, err := memdbvortex.NewDatastore(indexConfig) 89 | if err != nil { 90 | return nil, cancel, fmt.Errorf("failed to create local memory db: %s", err) 91 | } 92 | 93 | // Create the processors and the vortex. 94 | processors := map[string]*config.ProcessorConfiguration{ 95 | testmodel.ListIdentity.Name: &config.ProcessorConfiguration{ 96 | Identity: testmodel.ListIdentity, 97 | CommitOnEvent: true, 98 | }, 99 | 100 | v, err := memdbvortex.NewMemDBVortex( 101 | ctx, 102 | datastore, 103 | processors, 104 | gaia.Manager(), 105 | memdbvortex.OptionBackendManipulator(s), 106 | memdbvortex.OptionBackendSubscriber(subscriber), 107 | ) 108 | 109 | return v, cancel, err 110 | } 111 | 112 | ``` 113 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "fmt" 16 | 17 | "go.aporeto.io/elemental" 18 | ) 19 | 20 | // NewFiltersFromQueryParameters returns the filters matching any `q` parameters. 21 | func NewFiltersFromQueryParameters(parameters elemental.Parameters) (*elemental.Filter, error) { 22 | 23 | values := parameters.Get("q").StringValues() 24 | filters := make([]*elemental.Filter, len(values)) 25 | 26 | for i, query := range values { 27 | 28 | f, err := elemental.NewFilterFromString(query) 29 | if err != nil { 30 | return nil, fmt.Errorf("unable to parse filter in query parameter: %w", err) 31 | } 32 | 33 | filters[i] = f.Done() 34 | } 35 | 36 | switch len(filters) { 37 | case 0: 38 | return nil, nil 39 | case 1: 40 | return filters[0], nil 41 | default: 42 | return elemental.NewFilterComposer().Or(filters...).Done(), nil 43 | } 44 | } 45 | 46 | // NewNamespaceFilter returns a manipulate filter used to create the namespace filter. 47 | func NewNamespaceFilter(namespace string, recursive bool) *elemental.Filter { 48 | 49 | return NewNamespaceFilterWithCustomProperty("namespace", namespace, recursive) 50 | } 51 | 52 | // NewNamespaceFilterWithCustomProperty allows to create a namespace filter based on a property that 53 | // is different from `namespace`. 54 | func NewNamespaceFilterWithCustomProperty(propertyName string, namespace string, recursive bool) *elemental.Filter { 55 | 56 | if namespace == "" { 57 | namespace = "/" 58 | } 59 | 60 | if !recursive { 61 | return elemental.NewFilterComposer().WithKey(propertyName).Equals(namespace).Done() 62 | } 63 | 64 | if namespace == "/" { 65 | return elemental.NewFilterComposer().WithKey(propertyName).Matches("^/").Done() 66 | } 67 | 68 | return elemental.NewFilterComposer().Or( 69 | elemental.NewFilterComposer(). 70 | WithKey(propertyName).Equals(namespace). 71 | Done(), 72 | elemental.NewFilterComposer(). 73 | WithKey(propertyName).Matches("^"+namespace+"/"). 74 | Done(), 75 | ).Done() 76 | } 77 | 78 | // NewPropagationFilter returns additional namespace filter matching objects that are in 79 | // the namespace ancestors chain and propagate down. 80 | func NewPropagationFilter(namespace string) *elemental.Filter { 81 | 82 | return NewPropagationFilterWithCustomProperty("propagate", "namespace", namespace, nil) 83 | } 84 | 85 | // NewPropagationFilterWithCustomProperty returns additional namespace filter matching objects that are in 86 | // the namespace ancestors chain and propagate down. The two first properties allows to 87 | // define the property name to use for propation and namespace. 88 | // You can also set an additional filter that will be be AND'ed to each subfilters, allowing 89 | // to create filters like `(namespace == '/parent' and propagate == true and customProp == 'x')` 90 | func NewPropagationFilterWithCustomProperty( 91 | propagationPropName string, 92 | namespacePropName string, 93 | namespace string, 94 | addititionalFiltering *elemental.Filter, 95 | ) *elemental.Filter { 96 | 97 | filters := []*elemental.Filter{} 98 | 99 | for _, pns := range elemental.NamespaceAncestorsNames(namespace) { 100 | f := NewNamespaceFilterWithCustomProperty(namespacePropName, pns, false). 101 | WithKey(propagationPropName).Equals(true). 102 | Done() 103 | 104 | if addititionalFiltering != nil { 105 | f.And(addititionalFiltering) 106 | } 107 | 108 | filters = append(filters, f) 109 | } 110 | 111 | switch len(filters) { 112 | 113 | case 0: 114 | return nil 115 | case 1: 116 | return filters[0] 117 | default: 118 | return elemental.NewFilterComposer().Or(filters...).Done() 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /manipcli/delete_many.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.aporeto.io/elemental" 11 | "go.aporeto.io/manipulate" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | // generateDeleteManyCommandForIdentity generates the command to delete many objects based on its identity. 16 | func generateDeleteManyCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 17 | 18 | cmd := &cobra.Command{ 19 | Use: fmt.Sprintf("%s", identity.Name), 20 | Aliases: []string{identity.Category}, 21 | Short: "Delete multiple " + identity.Category, 22 | // Aliases: TODO: Missing alias from the spec file -> To be stored in the identity ?, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | 25 | fParam := viper.GetStringSlice("param") 26 | fTrackingID := viper.GetString(flagTrackingID) 27 | fConfirm := viper.GetBool(flagConfirm) 28 | fFilter := viper.GetString(flagFilter) 29 | fOutput := viper.GetString(flagOutput) 30 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 31 | fOutputTemplate := viper.GetString(flagOutputTemplate) 32 | fNamespace := viper.GetString(flagNamespace) 33 | 34 | manipulator, err := manipulatorMaker() 35 | if err != nil { 36 | return fmt.Errorf("unable to make manipulator: %w", err) 37 | } 38 | 39 | parameters, err := parametersToURLValues(fParam) 40 | if err != nil { 41 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 42 | } 43 | 44 | options := []manipulate.ContextOption{ 45 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 46 | manipulate.ContextOptionParameters(parameters), 47 | manipulate.ContextOptionFields(fFormatTypeColumn), 48 | manipulate.ContextOptionOverride(fConfirm), 49 | } 50 | 51 | if fFilter != "" { 52 | f, err := elemental.NewFilterFromString(fFilter) 53 | if err != nil { 54 | return fmt.Errorf("unable to parse filter %s: %s", fFilter, err) 55 | } 56 | options = append(options, manipulate.ContextOptionFilter(f)) 57 | } 58 | 59 | ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) 60 | defer cancel() 61 | 62 | mctx := manipulate.NewContext(ctx, options...) 63 | 64 | identifiables := modelManager.Identifiables(identity) 65 | if err := manipulator.RetrieveMany(mctx, identifiables); err != nil { 66 | return fmt.Errorf("unable to retrieve %s: %w", identity.Category, err) 67 | } 68 | 69 | objects := identifiables.List() 70 | 71 | if !fConfirm { 72 | for _, item := range objects { 73 | zap.L().Debug(fmt.Sprintf("- %s with ID=%s will be removed", identity.Name, item.Identifier())) 74 | } 75 | return fmt.Errorf("you are about to delete %d %s. If you are sure, please use --%s option to delete %v", len(objects), identity.Category, flagConfirm, fConfirm) 76 | } 77 | 78 | var deleted elemental.IdentifiablesList 79 | 80 | errs := elemental.NewErrors() 81 | for _, o := range objects { 82 | 83 | nsable, ok := o.(elemental.Namespaceable) 84 | if ok { 85 | mctx = mctx.Derive(manipulate.ContextOptionNamespace(nsable.GetNamespace())) 86 | } else { 87 | mctx = mctx.Derive(manipulate.ContextOptionNamespace(fNamespace)) 88 | } 89 | 90 | if err := manipulator.Delete(mctx, o); err != nil { 91 | errs = errs.Append(err) 92 | continue 93 | } 94 | 95 | deleted = append(deleted, o) 96 | } 97 | 98 | if len(errs) > 0 { 99 | return fmt.Errorf("some %s were not deleted: %w", identity.Category, errs) 100 | } 101 | 102 | outputType := fOutput 103 | if fOutput == flagOutputDefault { 104 | outputType = flagOutputNone 105 | } 106 | 107 | result, err := formatObjects( 108 | prepareOutputFormat(outputType, formatTypeArray, fFormatTypeColumn, fOutputTemplate), 109 | true, 110 | deleted..., 111 | ) 112 | 113 | if err != nil { 114 | return fmt.Errorf("unable to format output: %w", err) 115 | } 116 | 117 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 118 | return nil 119 | }, 120 | } 121 | 122 | cmd.Flags().StringP(flagFilter, "f", "", "Query filter.") 123 | cmd.Flags().BoolP(flagConfirm, "", false, "Confirm deletion of multiple objects") 124 | 125 | return cmd, nil 126 | } 127 | -------------------------------------------------------------------------------- /maniphttp/subscriber.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniphttp 13 | 14 | import ( 15 | "crypto/tls" 16 | "fmt" 17 | "net/http" 18 | "strings" 19 | 20 | "go.aporeto.io/manipulate" 21 | "go.aporeto.io/manipulate/internal/push" 22 | ) 23 | 24 | type subscribeConfig struct { 25 | namespace string 26 | credentialCookieKey string 27 | endpoint string 28 | supportErrorEvents bool 29 | recursive bool 30 | tlsConfig *tls.Config 31 | } 32 | 33 | func newSubscribeConfig(m *httpManipulator) subscribeConfig { 34 | return subscribeConfig{ 35 | endpoint: "events", 36 | namespace: m.namespace, 37 | tlsConfig: m.tlsConfig, 38 | } 39 | } 40 | 41 | // SubscriberOption represents option to NewSubscriber. 42 | type SubscriberOption func(*subscribeConfig) 43 | 44 | // SubscriberOptionRecursive makes the subscriber to listen 45 | // to events in current namespace and all children. 46 | func SubscriberOptionRecursive(recursive bool) SubscriberOption { 47 | return func(cfg *subscribeConfig) { 48 | cfg.recursive = recursive 49 | } 50 | } 51 | 52 | // SubscriberOptionNamespace sets the namespace from where the subscription 53 | // should start. 54 | // By default it is the same as the manipulator. 55 | func SubscriberOptionNamespace(namespace string) SubscriberOption { 56 | return func(cfg *subscribeConfig) { 57 | cfg.namespace = namespace 58 | } 59 | } 60 | 61 | // SubscriberOptionEndpoint sets the endpint to connect to. 62 | // By default it is /events. 63 | func SubscriberOptionEndpoint(endpoint string) SubscriberOption { 64 | return func(cfg *subscribeConfig) { 65 | cfg.endpoint = strings.TrimRight(strings.TrimLeft(endpoint, "/"), "/") 66 | } 67 | } 68 | 69 | // SubscriberSendCredentialsAsCookie makes the subscriber send the 70 | // crendentials as cookie using the provided key 71 | func SubscriberSendCredentialsAsCookie(key string) SubscriberOption { 72 | return func(cfg *subscribeConfig) { 73 | cfg.credentialCookieKey = key 74 | } 75 | } 76 | 77 | // SubscriberOptionSupportErrorEvents will result in connecting to the socket server by declaring that you are capable of 78 | // handling error events. 79 | func SubscriberOptionSupportErrorEvents() SubscriberOption { 80 | return func(cfg *subscribeConfig) { 81 | cfg.supportErrorEvents = true 82 | } 83 | } 84 | 85 | // NewSubscriber returns a new subscription. 86 | func NewSubscriber(manipulator manipulate.Manipulator, options ...SubscriberOption) manipulate.Subscriber { 87 | 88 | m, ok := manipulator.(*httpManipulator) 89 | if !ok { 90 | panic("You must pass a HTTP manipulator to maniphttp.NewSubscriber or maniphttp.NewSubscriberWithEndpoint") 91 | } 92 | 93 | cfg := newSubscribeConfig(m) 94 | for _, opt := range options { 95 | if opt == nil { 96 | panic("nil passed as subscriber option") 97 | } 98 | opt(&cfg) 99 | } 100 | 101 | if cfg.tlsConfig != nil { 102 | cfg.tlsConfig = cfg.tlsConfig.Clone() 103 | cfg.tlsConfig.NextProtos = nil 104 | } 105 | 106 | return push.NewSubscriber( 107 | fmt.Sprintf("%s/%s", m.url, cfg.endpoint), 108 | cfg.namespace, 109 | m.currentPassword(), 110 | m.registerRenewNotifier, 111 | m.unregisterRenewNotifier, 112 | cfg.tlsConfig, 113 | http.Header{ 114 | "Content-Type": []string{string(m.encoding)}, 115 | "Accept": []string{string(m.encoding)}, 116 | }, 117 | cfg.supportErrorEvents, 118 | cfg.recursive, 119 | cfg.credentialCookieKey, 120 | ) 121 | } 122 | 123 | // NewSubscriberWithEndpoint returns a new subscription connecting to specific endpoint. 124 | func NewSubscriberWithEndpoint(manipulator manipulate.Manipulator, endpoint string, recursive bool) manipulate.Subscriber { 125 | 126 | return NewSubscriber(manipulator, SubscriberOptionRecursive(recursive), SubscriberOptionEndpoint(endpoint)) 127 | } 128 | -------------------------------------------------------------------------------- /manipcli/list.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.aporeto.io/elemental" 11 | "go.aporeto.io/manipulate" 12 | ) 13 | 14 | // generateListCommandForIdentity generates the command to list all objects based on its identity. 15 | func generateListCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 16 | 17 | cmd := &cobra.Command{ 18 | Use: identity.Name, 19 | Aliases: []string{identity.Category}, 20 | Short: "List all " + identity.Category, 21 | // Aliases: TODO: Missing alias from the spec file -> To be stored in the identity ?, 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | 24 | fParam := viper.GetStringSlice("param") 25 | fTrackingID := viper.GetString(flagTrackingID) 26 | fRecursive := viper.GetBool(flagRecursive) 27 | fPage := viper.GetInt(flagPage) 28 | fPageSize := viper.GetInt(flagPageSize) 29 | fOrder := viper.GetStringSlice(flagOrder) 30 | fFilter := viper.GetString(flagFilter) 31 | fOutput := viper.GetString(flagOutput) 32 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 33 | fOutputTemplate := viper.GetString(flagOutputTemplate) 34 | 35 | var dest elemental.Identifiables 36 | if len(fFormatTypeColumn) == 0 { 37 | dest = modelManager.Identifiables(identity) 38 | } else { 39 | dest = modelManager.SparseIdentifiables(identity) 40 | } 41 | if dest == nil { 42 | return fmt.Errorf("unable to list %s. unknown identity", identity.Category) 43 | } 44 | 45 | manipulator, err := manipulatorMaker() 46 | if err != nil { 47 | return fmt.Errorf("unable to make manipulator: %w", err) 48 | } 49 | 50 | parameters, err := parametersToURLValues(fParam) 51 | if err != nil { 52 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 53 | } 54 | 55 | options := []manipulate.ContextOption{ 56 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 57 | manipulate.ContextOptionParameters(parameters), 58 | manipulate.ContextOptionFields(fFormatTypeColumn), 59 | manipulate.ContextOptionRecursive(fRecursive), 60 | manipulate.ContextOptionPage(fPage, fPageSize), 61 | manipulate.ContextOptionOrder(fOrder...), 62 | } 63 | 64 | if fFilter != "" { 65 | f, err := elemental.NewFilterFromString(fFilter) 66 | if err != nil { 67 | return fmt.Errorf("unable to parse filter %s: %s", fFilter, err) 68 | } 69 | options = append(options, manipulate.ContextOptionFilter(f)) 70 | } 71 | 72 | if viper.IsSet(flagParent) { 73 | parentName, parentID, err := splitParentInfo(viper.GetString(flagParent)) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | parent := modelManager.IdentifiableFromString(parentName) 79 | if parent == nil { 80 | return fmt.Errorf("unknown identity %s", parentName) 81 | } 82 | parent.SetIdentifier(parentID) 83 | options = append(options, manipulate.ContextOptionParent(parent)) 84 | } 85 | 86 | ctx, cancel := context.WithTimeout(cmd.Context(), 20*time.Second) 87 | defer cancel() 88 | 89 | mctx := manipulate.NewContext(ctx, options...) 90 | if err := manipulator.RetrieveMany(mctx, dest); err != nil { 91 | return fmt.Errorf("unable to retrieve all %s: %w", identity.Category, err) 92 | } 93 | 94 | outputType := fOutput 95 | if fOutput == flagOutputDefault { 96 | outputType = flagOutputJSON 97 | } 98 | 99 | result, err := formatObjects( 100 | prepareOutputFormat(outputType, formatTypeArray, fFormatTypeColumn, fOutputTemplate), 101 | true, 102 | dest.List()..., 103 | ) 104 | 105 | if err != nil { 106 | return fmt.Errorf("unable to format output: %w", err) 107 | } 108 | 109 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 110 | return nil 111 | }, 112 | } 113 | 114 | cmd.Flags().BoolP(flagRecursive, "r", false, "List all objects from the current namespace and all child namespaces.") 115 | cmd.Flags().IntP(flagPageSize, "S", 0, "Page size to retrieve.") 116 | cmd.Flags().IntP(flagPage, "P", 0, "Page number to retrieve.") 117 | cmd.Flags().StringP(flagFilter, "f", "", "Query filter.") 118 | cmd.Flags().StringSliceP(flagOrder, "O", nil, "Ordering of the result.") 119 | cmd.Flags().StringP(flagParent, "", "", "Provide information about parent resource. Format `name/ID`") 120 | 121 | return cmd, nil 122 | } 123 | -------------------------------------------------------------------------------- /manipcli/listen.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "sync" 9 | 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | "go.aporeto.io/elemental" 13 | "go.aporeto.io/manipulate" 14 | "go.aporeto.io/manipulate/maniphttp" 15 | "go.uber.org/zap" 16 | ) 17 | 18 | // generateListenCommandForIdentity generates the command to listen for events based on its identity. 19 | func generateListenCommand(modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker) (*cobra.Command, error) { 20 | 21 | cmd := &cobra.Command{ 22 | Use: "listen", 23 | Short: "Listen for events", 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | 26 | fRecursive := viper.GetBool(flagRecursive) 27 | fOutput := viper.GetString(flagOutput) 28 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 29 | fOutputTemplate := viper.GetString(flagOutputTemplate) 30 | 31 | manipulator, err := manipulatorMaker() 32 | if err != nil { 33 | return fmt.Errorf("unable to make manipulator: %w", err) 34 | } 35 | 36 | subscriber := maniphttp.NewSubscriber( 37 | manipulator, 38 | maniphttp.SubscriberOptionRecursive(fRecursive), 39 | maniphttp.SubscriberOptionSupportErrorEvents(), 40 | ) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | var filter *elemental.PushConfig 46 | filterIdentities := viper.GetStringSlice("identity") 47 | 48 | if len(filterIdentities) > 0 { 49 | 50 | filter = elemental.NewPushConfig() 51 | 52 | for _, i := range filterIdentities { 53 | identity := modelManager.IdentityFromAny(i) 54 | if identity.IsEmpty() { 55 | return fmt.Errorf("unknown identity %s", i) 56 | } 57 | filter.FilterIdentity(identity.Name) 58 | } 59 | } 60 | 61 | outputType := fOutput 62 | if fOutput == flagOutputDefault { 63 | outputType = flagOutputJSON 64 | } 65 | outputFormat := prepareOutputFormat(outputType, formatTypeHash, fFormatTypeColumn, fOutputTemplate) 66 | 67 | var once sync.Once 68 | terminated := make(chan struct{}) 69 | 70 | go func() { 71 | 72 | pullErrorIfAny := func() error { 73 | select { 74 | case err := <-subscriber.Errors(): 75 | return err 76 | default: 77 | return nil 78 | } 79 | } 80 | 81 | for { 82 | select { 83 | 84 | case evt := <-subscriber.Events(): 85 | result, err := formatEvents(outputFormat, false, evt) 86 | if err != nil { 87 | zap.L().Error("unable to format event", zap.Error(err)) 88 | } 89 | 90 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 91 | 92 | case st := <-subscriber.Status(): 93 | switch st { 94 | case manipulate.SubscriberStatusInitialConnection: 95 | zap.L().Debug("status update", zap.String("status", "connected")) 96 | case manipulate.SubscriberStatusInitialConnectionFailure: 97 | zap.L().Warn("status update", zap.String("status", "connect failed"), zap.Error(pullErrorIfAny())) 98 | case manipulate.SubscriberStatusDisconnection: 99 | zap.L().Warn("status update", zap.String("status", "disconnected"), zap.Error(pullErrorIfAny())) 100 | case manipulate.SubscriberStatusReconnection: 101 | zap.L().Info("status update", zap.String("status", "reconnected")) 102 | case manipulate.SubscriberStatusReconnectionFailure: 103 | zap.L().Debug("status update", zap.String("status", "reconnection failed"), zap.Error(pullErrorIfAny())) 104 | case manipulate.SubscriberStatusFinalDisconnection: 105 | zap.L().Debug("status update", zap.String("status", "terminated")) 106 | once.Do(func() { close(terminated) }) 107 | } 108 | 109 | case err := <-subscriber.Errors(): 110 | zap.L().Error("Error received", zap.Error(err)) 111 | } 112 | } 113 | }() 114 | 115 | ctx, cancel := context.WithCancel(cmd.Context()) 116 | defer cancel() 117 | subscriber.Start(ctx, filter) 118 | 119 | c := make(chan os.Signal, 1) 120 | signal.Reset(os.Interrupt) 121 | signal.Notify(c, os.Interrupt) 122 | 123 | select { 124 | case <-c: 125 | cancel() 126 | case <-cmd.Context().Done(): 127 | fmt.Println("command context done") 128 | } 129 | 130 | <-terminated 131 | 132 | return nil 133 | }, 134 | } 135 | 136 | cmd.Flags().BoolP(flagRecursive, "r", false, "Listen to all events in the current namespace and all child namespaces.") 137 | cmd.Flags().StringSliceP("identity", "i", []string{}, "Only display events for the given identities.") 138 | 139 | return cmd, nil 140 | } 141 | -------------------------------------------------------------------------------- /manipvortex/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "time" 16 | 17 | "go.aporeto.io/manipulate" 18 | ) 19 | 20 | type config struct { 21 | upstreamManipulator manipulate.Manipulator 22 | upstreamSubscriber manipulate.Subscriber 23 | logfile string 24 | enableLog bool 25 | transactionQueue chan *Transaction 26 | readConsistency manipulate.ReadConsistency 27 | writeConsistency manipulate.WriteConsistency 28 | defaultQueueDuration time.Duration 29 | defaultPageSize int 30 | prefetcher Prefetcher 31 | upstreamReconciler Reconciler 32 | downstreamReconciler Reconciler 33 | disableUpstreamCommit bool 34 | } 35 | 36 | func newConfig() *config { 37 | return &config{ 38 | transactionQueue: make(chan *Transaction, 1000), 39 | readConsistency: manipulate.ReadConsistencyEventual, 40 | writeConsistency: manipulate.WriteConsistencyStrong, 41 | defaultQueueDuration: time.Second, 42 | defaultPageSize: 10000, 43 | } 44 | } 45 | 46 | // Option represents an option can can be passed to NewContext. 47 | type Option func(*config) 48 | 49 | // OptionDefaultConsistency sets the default read and write consistency. 50 | func OptionDefaultConsistency(read manipulate.ReadConsistency, write manipulate.WriteConsistency) Option { 51 | return func(cfg *config) { 52 | if read != manipulate.ReadConsistencyDefault { 53 | cfg.readConsistency = read 54 | } 55 | if write != manipulate.WriteConsistencyDefault { 56 | cfg.writeConsistency = write 57 | } 58 | } 59 | } 60 | 61 | // OptionUpstreamManipulator sets the upstream manipulator. 62 | func OptionUpstreamManipulator(manipulator manipulate.Manipulator) Option { 63 | return func(cfg *config) { 64 | cfg.upstreamManipulator = manipulator 65 | } 66 | } 67 | 68 | // OptionUpstreamSubscriber sets the upstream subscriber. 69 | // Note the given subscriber must NOT be started or the events 70 | // will be received twice, needlessly loading the VortexDB. 71 | func OptionUpstreamSubscriber(s manipulate.Subscriber) Option { 72 | return func(cfg *config) { 73 | cfg.upstreamSubscriber = s 74 | } 75 | } 76 | 77 | // OptionTransactionLog sets the transaction log file. 78 | func OptionTransactionLog(filename string) Option { 79 | return func(cfg *config) { 80 | cfg.logfile = filename 81 | cfg.enableLog = filename != "" 82 | } 83 | } 84 | 85 | // OptionTransactionQueueLength sets the queue length of the 86 | // transaction queue. 87 | func OptionTransactionQueueLength(n int) Option { 88 | return func(cfg *config) { 89 | cfg.transactionQueue = make(chan *Transaction, n) 90 | } 91 | } 92 | 93 | // OptionTransactionQueueDuration sets the default queue transaction 94 | // duration. Once expired, the transaction is discarded. 95 | func OptionTransactionQueueDuration(d time.Duration) Option { 96 | return func(cfg *config) { 97 | cfg.defaultQueueDuration = d 98 | } 99 | } 100 | 101 | // OptionDefaultPageSize is the page size during fetching. 102 | func OptionDefaultPageSize(defaultPageSize int) Option { 103 | return func(cfg *config) { 104 | cfg.defaultPageSize = defaultPageSize 105 | } 106 | } 107 | 108 | // OptionPrefetcher sets the Prefetcher to use. 109 | func OptionPrefetcher(p Prefetcher) Option { 110 | return func(cfg *config) { 111 | cfg.prefetcher = p 112 | } 113 | } 114 | 115 | // OptionDownstreamReconciler sets the global downstream Reconcilers to use. 116 | func OptionDownstreamReconciler(r Reconciler) Option { 117 | return func(cfg *config) { 118 | cfg.downstreamReconciler = r 119 | } 120 | } 121 | 122 | // OptionUpstreamReconciler sets the global upstream Reconcilers to use. 123 | func OptionUpstreamReconciler(r Reconciler) Option { 124 | return func(cfg *config) { 125 | cfg.upstreamReconciler = r 126 | } 127 | } 128 | 129 | // OptionDisableCommitUpstream sets the global upstream Reconcilers to use. 130 | func OptionDisableCommitUpstream(disabled bool) Option { 131 | return func(cfg *config) { 132 | cfg.disableUpstreamCommit = disabled 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /manipvortex/options_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipvortex 13 | 14 | import ( 15 | "testing" 16 | "time" 17 | 18 | "go.aporeto.io/manipulate" 19 | 20 | // nolint:revive // Allow dot imports for readability in tests 21 | . "github.com/smartystreets/goconvey/convey" 22 | "go.aporeto.io/manipulate/maniptest" 23 | ) 24 | 25 | func Test_newOptions(t *testing.T) { 26 | Convey("Given I call newConfig", t, func() { 27 | 28 | cfg := newConfig() 29 | 30 | Convey("Then it should be correct", func() { 31 | So(cfg.enableLog, ShouldBeFalse) 32 | So(cfg.logfile, ShouldBeEmpty) 33 | So(cfg.readConsistency, ShouldEqual, manipulate.ReadConsistencyEventual) 34 | So(cfg.writeConsistency, ShouldEqual, manipulate.WriteConsistencyStrong) 35 | So(cfg.upstreamManipulator, ShouldBeNil) 36 | So(cfg.upstreamSubscriber, ShouldBeNil) 37 | So(cfg.defaultQueueDuration, ShouldEqual, time.Second) 38 | So(cfg.transactionQueue, ShouldHaveSameTypeAs, make(chan *Transaction, 1000)) 39 | So(cfg.defaultPageSize, ShouldEqual, 10000) 40 | So(cfg.upstreamReconciler, ShouldBeNil) 41 | So(cfg.downstreamReconciler, ShouldBeNil) 42 | }) 43 | }) 44 | } 45 | 46 | func Test_Options(t *testing.T) { 47 | Convey("Given a memdb vortex memory", t, func() { 48 | 49 | cfg := newConfig() 50 | 51 | Convey("OptionUpstreamManipulator should work", func() { 52 | m := maniptest.NewTestManipulator() 53 | OptionUpstreamManipulator(m)(cfg) 54 | So(cfg.upstreamManipulator, ShouldResemble, m) 55 | }) 56 | 57 | Convey("OptionUpstreamSubscriber should work", func() { 58 | s := maniptest.NewTestSubscriber() 59 | OptionUpstreamSubscriber(s)(cfg) 60 | So(cfg.upstreamSubscriber, ShouldResemble, s) 61 | }) 62 | 63 | Convey("OptionTransactionLog should work", func() { 64 | OptionTransactionLog("somefile")(cfg) 65 | So(cfg.logfile, ShouldResemble, "somefile") 66 | So(cfg.enableLog, ShouldBeTrue) 67 | }) 68 | 69 | Convey("OptionTransactionQueueLength it should work", func() { 70 | OptionTransactionQueueLength(13)(cfg) 71 | So(cfg.transactionQueue, ShouldNotBeNil) 72 | So(cap(cfg.transactionQueue), ShouldEqual, 13) 73 | }) 74 | 75 | Convey("OptionDefaultConsistency should work", func() { 76 | OptionDefaultConsistency(manipulate.ReadConsistencyStrong, manipulate.WriteConsistencyNone)(cfg) 77 | So(cfg.readConsistency, ShouldEqual, manipulate.ReadConsistencyStrong) 78 | So(cfg.writeConsistency, ShouldEqual, manipulate.WriteConsistencyNone) 79 | }) 80 | 81 | Convey("OptionDefaultConsistency with defaults should work", func() { 82 | OptionDefaultConsistency(manipulate.ReadConsistencyDefault, manipulate.WriteConsistencyDefault)(cfg) 83 | So(cfg.readConsistency, ShouldEqual, manipulate.ReadConsistencyEventual) 84 | So(cfg.writeConsistency, ShouldEqual, manipulate.WriteConsistencyStrong) 85 | }) 86 | 87 | Convey("OptionTransactionQueueDuration with defaults should work", func() { 88 | OptionTransactionQueueDuration(time.Minute)(cfg) 89 | So(cfg.defaultQueueDuration, ShouldEqual, time.Minute) 90 | }) 91 | 92 | Convey("OptionDefaultPageSize with defaults should work", func() { 93 | OptionDefaultPageSize(12)(cfg) 94 | So(cfg.defaultPageSize, ShouldEqual, 12) 95 | }) 96 | 97 | Convey("OptionPrefetcher with defaults should work", func() { 98 | p := NewTestPrefetcher() 99 | OptionPrefetcher(p)(cfg) 100 | So(cfg.prefetcher, ShouldEqual, p) 101 | }) 102 | 103 | Convey("OptionUpstreamReconciler with defaults should work", func() { 104 | r := NewTestReconciler() 105 | OptionUpstreamReconciler(r)(cfg) 106 | So(cfg.upstreamReconciler, ShouldEqual, r) 107 | }) 108 | 109 | Convey("OptionDownstreamReconciler with defaults should work", func() { 110 | r := NewTestReconciler() 111 | OptionDownstreamReconciler(r)(cfg) 112 | So(cfg.downstreamReconciler, ShouldEqual, r) 113 | }) 114 | 115 | Convey("OptionDisableCommitUpstream with defaults should work", func() { 116 | OptionDisableCommitUpstream(true)(cfg) 117 | So(cfg.disableUpstreamCommit, ShouldBeTrue) 118 | }) 119 | }) 120 | } 121 | -------------------------------------------------------------------------------- /maniphttp/manipulator_unix_test.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | // Copyright 2019 Aporeto Inc. 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package maniphttp 16 | 17 | import ( 18 | "context" 19 | "crypto/tls" 20 | "net" 21 | "net/http" 22 | "syscall" 23 | "testing" 24 | "time" 25 | 26 | // nolint:revive // Allow dot imports for readability in tests 27 | . "github.com/smartystreets/goconvey/convey" 28 | internalsyscall "go.aporeto.io/manipulate/maniphttp/internal/syscall" 29 | "golang.org/x/sys/unix" 30 | ) 31 | 32 | func TestHTTP_TCPUserTimeout(t *testing.T) { 33 | Convey("When I create a simple manipulator with custom transport, with TCP option", t, func() { 34 | dialer := (&net.Dialer{ 35 | Timeout: 10 * time.Second, 36 | KeepAlive: 30 * time.Second, 37 | Control: internalsyscall.MakeDialerControlFunc(30 * time.Second), 38 | }).DialContext 39 | 40 | transport := &http.Transport{ 41 | DialContext: dialer, 42 | } 43 | transport.TLSClientConfig = &tls.Config{} 44 | 45 | mm, _ := New( 46 | context.Background(), 47 | "http://url.com/", 48 | OptionHTTPTransport(transport), 49 | OptionTCPUserTimeout(40*time.Second), 50 | ) 51 | 52 | m := mm.(*httpManipulator) 53 | 54 | Convey("Then the tls config is correct", func() { 55 | So(m.tlsConfig, ShouldEqual, transport.TLSClientConfig) 56 | }) 57 | Convey("Then the dialer is correct", func() { 58 | l, err := net.Listen("tcp", ":0") 59 | So(err, ShouldBeNil) 60 | 61 | opt := -1 62 | dctx := m.client.Transport.(*http.Transport).DialContext 63 | So(dctx, ShouldNotBeNil) 64 | conn, err := dctx(context.TODO(), "tcp", l.Addr().String()) 65 | So(err, ShouldBeNil) 66 | 67 | tcpConn, ok := conn.(*net.TCPConn) 68 | So(ok, ShouldBeTrue) 69 | 70 | rawConn, err := tcpConn.SyscallConn() 71 | So(err, ShouldBeNil) 72 | 73 | err = rawConn.Control(func(fd uintptr) { 74 | opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) 75 | }) 76 | So(err, ShouldBeNil) 77 | So(opt, ShouldEqual, 30*time.Second/time.Millisecond) 78 | l.Close() // nolint 79 | }) 80 | }) 81 | Convey("When I create a simple manipulator with default transport, with TCP_USER_TIMEOUT", t, func() { 82 | mm, _ := New( 83 | context.Background(), 84 | "http://url.com/", 85 | OptionTCPUserTimeout(40*time.Second), 86 | ) 87 | 88 | m := mm.(*httpManipulator) 89 | 90 | Convey("Then the dialer is correct", func() { 91 | l, err := net.Listen("tcp", ":0") 92 | So(err, ShouldBeNil) 93 | 94 | opt := -1 95 | dctx := m.client.Transport.(*http.Transport).DialContext 96 | So(dctx, ShouldNotBeNil) 97 | conn, err := dctx(context.TODO(), "tcp", l.Addr().String()) 98 | So(err, ShouldBeNil) 99 | 100 | tcpConn, ok := conn.(*net.TCPConn) 101 | So(ok, ShouldBeTrue) 102 | 103 | rawConn, err := tcpConn.SyscallConn() 104 | So(err, ShouldBeNil) 105 | 106 | err = rawConn.Control(func(fd uintptr) { 107 | opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) 108 | }) 109 | So(err, ShouldBeNil) 110 | So(opt, ShouldEqual, 40*time.Second/time.Millisecond) 111 | 112 | l.Close() // nolint 113 | }) 114 | }) 115 | Convey("When I create a simple manipulator with default transport, without TCP_USER_TIMEOUT", t, func() { 116 | mm, _ := New( 117 | context.Background(), 118 | "http://url.com/", 119 | ) 120 | 121 | m := mm.(*httpManipulator) 122 | 123 | Convey("Then the dialer is correct", func() { 124 | l, err := net.Listen("tcp4", ":0") 125 | So(err, ShouldBeNil) 126 | 127 | opt := -1 128 | dctx := m.client.Transport.(*http.Transport).DialContext 129 | So(dctx, ShouldNotBeNil) 130 | conn, err := dctx(context.TODO(), "tcp4", l.Addr().String()) 131 | So(err, ShouldBeNil) 132 | 133 | tcpConn, ok := conn.(*net.TCPConn) 134 | So(ok, ShouldBeTrue) 135 | 136 | rawConn, err := tcpConn.SyscallConn() 137 | So(err, ShouldBeNil) 138 | 139 | err = rawConn.Control(func(fd uintptr) { 140 | opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) 141 | }) 142 | So(err, ShouldBeNil) 143 | So(opt, ShouldEqual, 0) 144 | 145 | l.Close() // nolint 146 | }) 147 | }) 148 | } 149 | -------------------------------------------------------------------------------- /manipcli/delete_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateDeleteCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a delete command", t, func() { 20 | 21 | task1 := testmodel.NewTask() 22 | task1.ID = "617aec75a829de0001da2032" 23 | task1.Name = "task1" 24 | 25 | m := maniptest.NewTestManipulator() 26 | m.MockRetrieveMany(t, func(mctx manipulate.Context, dest elemental.Identifiables) error { 27 | tasks := testmodel.TasksList{ 28 | task1, 29 | } 30 | *dest.(*testmodel.TasksList) = tasks 31 | return nil 32 | }) 33 | 34 | m.MockDelete(t, func(ctx manipulate.Context, object elemental.Identifiable) error { 35 | return nil 36 | }) 37 | 38 | cmd, err := generateDeleteCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 39 | return m, nil 40 | }) 41 | 42 | So(err, ShouldEqual, nil) 43 | assertCommandAndSetFlags(cmd) 44 | 45 | Convey("When I call execute", func() { 46 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 47 | 48 | output := bytes.NewBufferString("") 49 | cmd.SetOut(output) 50 | err := cmd.Execute() 51 | 52 | Convey("Then I should get a generated command", func() { 53 | So(err, ShouldEqual, nil) 54 | So(output.String(), ShouldEqual, "617aec75a829de0001da2032") 55 | }) 56 | }) 57 | }) 58 | 59 | Convey("Given I generate a delete command that returns an error", t, func() { 60 | 61 | cmd, err := generateDeleteCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 62 | return nil, fmt.Errorf("boom") 63 | }) 64 | 65 | So(err, ShouldEqual, nil) 66 | assertCommandAndSetFlags(cmd) 67 | 68 | Convey("When I call execute", func() { 69 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 70 | err := cmd.Execute() 71 | 72 | Convey("Then I should get an error", func() { 73 | So(err, ShouldNotEqual, nil) 74 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 75 | }) 76 | }) 77 | }) 78 | 79 | Convey("Given I generate a delete command and a manipulator that fails on retrieve", t, func() { 80 | 81 | task1 := testmodel.NewTask() 82 | task1.ID = "617aec75a829de0001da2032" 83 | task1.Name = "task1" 84 | 85 | m := maniptest.NewTestManipulator() 86 | m.MockRetrieve(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 87 | return fmt.Errorf("retrieve boom") 88 | }) 89 | 90 | m.MockDelete(t, func(ctx manipulate.Context, object elemental.Identifiable) error { 91 | return nil 92 | }) 93 | 94 | cmd, err := generateDeleteCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 95 | return m, nil 96 | }) 97 | 98 | So(err, ShouldEqual, nil) 99 | assertCommandAndSetFlags(cmd) 100 | 101 | Convey("When I call execute", func() { 102 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 103 | 104 | output := bytes.NewBufferString("") 105 | cmd.SetOut(output) 106 | err := cmd.Execute() 107 | 108 | Convey("Then I should get a generated command", func() { 109 | So(err, ShouldNotEqual, nil) 110 | So(err.Error(), ShouldEqual, "unable to retrieve task: retrieve boom") 111 | }) 112 | }) 113 | }) 114 | 115 | Convey("Given I generate a delete command and a manipulator that fails on delete", t, func() { 116 | 117 | task1 := testmodel.NewTask() 118 | task1.ID = "617aec75a829de0001da2032" 119 | task1.Name = "task1" 120 | 121 | m := maniptest.NewTestManipulator() 122 | 123 | m.MockDelete(t, func(ctx manipulate.Context, object elemental.Identifiable) error { 124 | return fmt.Errorf("boom") 125 | }) 126 | 127 | cmd, err := generateDeleteCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 128 | return m, nil 129 | }) 130 | 131 | So(err, ShouldEqual, nil) 132 | assertCommandAndSetFlags(cmd) 133 | 134 | Convey("When I call execute", func() { 135 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 136 | 137 | output := bytes.NewBufferString("") 138 | cmd.SetOut(output) 139 | err := cmd.Execute() 140 | 141 | Convey("Then I should get a generated command", func() { 142 | So(err, ShouldNotEqual, nil) 143 | So(err.Error(), ShouldEqual, "unable to delete task: boom") 144 | }) 145 | }) 146 | }) 147 | 148 | } 149 | -------------------------------------------------------------------------------- /maniptest/subscriber.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniptest 13 | 14 | import ( 15 | "context" 16 | "sync" 17 | "testing" 18 | 19 | "go.aporeto.io/elemental" 20 | "go.aporeto.io/manipulate" 21 | ) 22 | 23 | type mockedSubscriberMethods struct { 24 | startMock func(context.Context, *elemental.PushConfig) 25 | updateFilerMock func(*elemental.PushConfig) 26 | eventsMock func() chan *elemental.Event 27 | errorsMock func() chan error 28 | statusMock func() chan manipulate.SubscriberStatus 29 | } 30 | 31 | // A TestSubscriber is the interface of mockable test manipulator. 32 | type TestSubscriber interface { 33 | manipulate.Subscriber 34 | MockStart(t *testing.T, impl func(context.Context, *elemental.PushConfig)) 35 | MockUpdateFilter(t *testing.T, impl func(*elemental.PushConfig)) 36 | MockEvents(t *testing.T, impl func() chan *elemental.Event) 37 | MockErrors(t *testing.T, impl func() chan error) 38 | MockStatus(t *testing.T, impl func() chan manipulate.SubscriberStatus) 39 | } 40 | 41 | // A testSubscriber is an empty TransactionalManipulator that can be easily mocked. 42 | type testSubscriber struct { 43 | mocks map[*testing.T]*mockedSubscriberMethods 44 | lock *sync.Mutex 45 | currentTest *testing.T 46 | } 47 | 48 | // NewTestSubscriber returns a new TestSubscriber. 49 | func NewTestSubscriber() TestSubscriber { 50 | return &testSubscriber{ 51 | lock: &sync.Mutex{}, 52 | mocks: map[*testing.T]*mockedSubscriberMethods{}, 53 | } 54 | } 55 | 56 | func (m *testSubscriber) MockStart(t *testing.T, impl func(context.Context, *elemental.PushConfig)) { 57 | 58 | m.lock.Lock() 59 | defer m.lock.Unlock() 60 | 61 | m.currentMocks(t).startMock = impl 62 | } 63 | 64 | func (m *testSubscriber) MockUpdateFilter(t *testing.T, impl func(*elemental.PushConfig)) { 65 | 66 | m.lock.Lock() 67 | defer m.lock.Unlock() 68 | 69 | m.currentMocks(t).updateFilerMock = impl 70 | } 71 | 72 | func (m *testSubscriber) MockEvents(t *testing.T, impl func() chan *elemental.Event) { 73 | 74 | m.lock.Lock() 75 | defer m.lock.Unlock() 76 | 77 | m.currentMocks(t).eventsMock = impl 78 | } 79 | 80 | func (m *testSubscriber) MockErrors(t *testing.T, impl func() chan error) { 81 | 82 | m.lock.Lock() 83 | defer m.lock.Unlock() 84 | 85 | m.currentMocks(t).errorsMock = impl 86 | } 87 | 88 | func (m *testSubscriber) MockStatus(t *testing.T, impl func() chan manipulate.SubscriberStatus) { 89 | 90 | m.lock.Lock() 91 | defer m.lock.Unlock() 92 | 93 | m.currentMocks(t).statusMock = impl 94 | } 95 | 96 | func (m *testSubscriber) Start(ctx context.Context, filter *elemental.PushConfig) { 97 | 98 | m.lock.Lock() 99 | defer m.lock.Unlock() 100 | 101 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.startMock != nil { 102 | mock.startMock(ctx, filter) 103 | } 104 | 105 | } 106 | 107 | func (m *testSubscriber) UpdateFilter(filter *elemental.PushConfig) { 108 | 109 | m.lock.Lock() 110 | defer m.lock.Unlock() 111 | 112 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.updateFilerMock != nil { 113 | mock.updateFilerMock(filter) 114 | } 115 | 116 | } 117 | 118 | func (m *testSubscriber) Events() chan *elemental.Event { 119 | 120 | m.lock.Lock() 121 | defer m.lock.Unlock() 122 | 123 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.eventsMock != nil { 124 | return mock.eventsMock() 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func (m *testSubscriber) Errors() chan error { 131 | 132 | m.lock.Lock() 133 | defer m.lock.Unlock() 134 | 135 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.errorsMock != nil { 136 | return mock.errorsMock() 137 | } 138 | 139 | return nil 140 | } 141 | 142 | func (m *testSubscriber) Status() chan manipulate.SubscriberStatus { 143 | 144 | m.lock.Lock() 145 | defer m.lock.Unlock() 146 | 147 | if mock := m.currentMocks(m.currentTest); mock != nil && mock.statusMock != nil { 148 | return mock.statusMock() 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func (m *testSubscriber) currentMocks(t *testing.T) *mockedSubscriberMethods { 155 | 156 | mocks := m.mocks[t] 157 | 158 | if mocks == nil { 159 | mocks = &mockedSubscriberMethods{} 160 | m.mocks[t] = mocks 161 | } 162 | 163 | m.currentTest = t 164 | return mocks 165 | } 166 | -------------------------------------------------------------------------------- /manipcli/list_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateListCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a delete-many command", t, func() { 20 | 21 | task1 := testmodel.NewTask() 22 | task1.ID = "617aec75a829de0001da2032" 23 | task1.Name = "task1" 24 | 25 | task2 := testmodel.NewTask() 26 | task2.ID = "111aec75a829de0001da1111" 27 | task2.Name = "task2" 28 | 29 | m := maniptest.NewTestManipulator() 30 | m.MockRetrieveMany(t, func(mctx manipulate.Context, dest elemental.Identifiables) error { 31 | tasks := testmodel.TasksList{ 32 | task1, 33 | task2, 34 | } 35 | *dest.(*testmodel.TasksList) = tasks 36 | return nil 37 | }) 38 | 39 | cmd, err := generateListCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 40 | return m, nil 41 | }) 42 | 43 | So(err, ShouldEqual, nil) 44 | assertCommandAndSetFlags(cmd) 45 | 46 | Convey("When I call execute without filter", func() { 47 | output := bytes.NewBufferString("") 48 | cmd.SetOut(output) 49 | err := cmd.Execute() 50 | 51 | Convey("Then I should get a generated command", func() { 52 | So(err, ShouldEqual, nil) 53 | So(output.String(), ShouldEqual, `[ 54 | { 55 | "ID": "617aec75a829de0001da2032", 56 | "description": "", 57 | "name": "task1", 58 | "parentID": "", 59 | "parentType": "", 60 | "status": "TODO" 61 | }, 62 | { 63 | "ID": "111aec75a829de0001da1111", 64 | "description": "", 65 | "name": "task2", 66 | "parentID": "", 67 | "parentType": "", 68 | "status": "TODO" 69 | } 70 | ]`) 71 | }) 72 | }) 73 | 74 | Convey("When I call execute with a valid filter", func() { 75 | output := bytes.NewBufferString("") 76 | cmd.SetOut(output) 77 | cmd.Flags().Set("filter", "name == x") // nolint 78 | err := cmd.Execute() 79 | 80 | Convey("Then I should get a generated command", func() { 81 | So(err, ShouldEqual, nil) 82 | So(output.String(), ShouldEqual, `[ 83 | { 84 | "ID": "617aec75a829de0001da2032", 85 | "description": "", 86 | "name": "task1", 87 | "parentID": "", 88 | "parentType": "", 89 | "status": "TODO" 90 | }, 91 | { 92 | "ID": "111aec75a829de0001da1111", 93 | "description": "", 94 | "name": "task2", 95 | "parentID": "", 96 | "parentType": "", 97 | "status": "TODO" 98 | } 99 | ]`) 100 | }) 101 | }) 102 | 103 | Convey("When I call execute with an invalid filter", func() { 104 | output := bytes.NewBufferString("") 105 | cmd.SetOut(output) 106 | cmd.Flags().Set("filter", "name...") // nolint 107 | err := cmd.Execute() 108 | 109 | Convey("Then I should get a generated command", func() { 110 | So(err, ShouldNotEqual, nil) 111 | So(err.Error(), ShouldContainSubstring, "unable to parse filter") 112 | }) 113 | }) 114 | 115 | }) 116 | 117 | Convey("Given I generate a delete-many command that returns an error", t, func() { 118 | 119 | cmd, err := generateListCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 120 | return nil, fmt.Errorf("boom") 121 | }) 122 | 123 | So(err, ShouldEqual, nil) 124 | assertCommandAndSetFlags(cmd) 125 | 126 | Convey("When I call execute", func() { 127 | 128 | err := cmd.Execute() 129 | 130 | Convey("Then I should get an error", func() { 131 | So(err, ShouldNotEqual, nil) 132 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 133 | }) 134 | }) 135 | }) 136 | 137 | Convey("Given I generate a delete-many command and a manipulator that fails", t, func() { 138 | 139 | m := maniptest.NewTestManipulator() 140 | m.MockRetrieveMany(t, func(mctx manipulate.Context, dest elemental.Identifiables) error { 141 | return fmt.Errorf("boom") 142 | }) 143 | 144 | cmd, err := generateListCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 145 | return m, nil 146 | }) 147 | 148 | So(err, ShouldEqual, nil) 149 | assertCommandAndSetFlags(cmd) 150 | 151 | Convey("When I call execute without filter", func() { 152 | output := bytes.NewBufferString("") 153 | cmd.SetOut(output) 154 | err := cmd.Execute() 155 | 156 | Convey("Then I should get a generated command", func() { 157 | So(err, ShouldNotEqual, nil) 158 | So(err.Error(), ShouldEqual, "unable to retrieve all tasks: boom") 159 | 160 | }) 161 | }) 162 | }) 163 | 164 | } 165 | -------------------------------------------------------------------------------- /manipmemory/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipmemory 13 | 14 | import ( 15 | "fmt" 16 | "reflect" 17 | "strings" 18 | 19 | memdb "github.com/hashicorp/go-memdb" 20 | "go.aporeto.io/elemental" 21 | ) 22 | 23 | // stringBasedFieldIndex is used to extract a field from an object 24 | // using reflection and builds an index on that field. The Indexer 25 | // takes objects that the underlying is string, even though the original 26 | // type is not string. For example, if you declare a type as 27 | // 28 | // type ABC string 29 | // 30 | // then you should use this indexer. It implements the memdb indexer 31 | // interface. 32 | type stringBasedFieldIndex struct { 33 | Field string 34 | Lowercase bool 35 | } 36 | 37 | // FromObject implements the memdb indexer interface. 38 | func (s *stringBasedFieldIndex) FromObject(obj any) (bool, []byte, error) { 39 | v := reflect.ValueOf(obj) 40 | v = reflect.Indirect(v) // Dereference the pointer if any 41 | 42 | fv := v.FieldByName(s.Field) 43 | if !fv.IsValid() { 44 | return false, nil, 45 | fmt.Errorf("field '%s' for %#v is invalid", s.Field, obj) 46 | } 47 | 48 | val := fv.String() 49 | if val == "" { 50 | return false, nil, nil 51 | } 52 | 53 | if s.Lowercase { 54 | val = strings.ToLower(val) 55 | } 56 | 57 | // Add the null character as a terminator 58 | val += "\x00" 59 | return true, []byte(val), nil 60 | } 61 | 62 | // FromArgs implements the memdb indexer interface. 63 | func (s *stringBasedFieldIndex) FromArgs(args ...any) ([]byte, error) { 64 | if len(args) != 1 { 65 | return nil, fmt.Errorf("must provide only a single argument") 66 | } 67 | 68 | t := reflect.TypeOf(args[0]) 69 | if t.Kind() != reflect.String { 70 | return nil, fmt.Errorf("argument must be a string: %#v", args[0]) 71 | } 72 | arg := reflect.ValueOf(args[0]).String() 73 | 74 | if s.Lowercase { 75 | arg = strings.ToLower(arg) 76 | } 77 | 78 | // Add the null character as a terminator 79 | arg += "\x00" 80 | return []byte(arg), nil 81 | } 82 | 83 | // createSchema creates the memdb schema from the configuration of the identities. 84 | func createSchema(c *IdentitySchema) (*memdb.TableSchema, error) { 85 | 86 | tableSchema := &memdb.TableSchema{ 87 | Name: c.Identity.Category, 88 | Indexes: map[string]*memdb.IndexSchema{}, 89 | } 90 | 91 | for _, index := range c.Indexes { 92 | 93 | var indexConfig memdb.Indexer 94 | 95 | switch index.Type { 96 | 97 | case IndexTypeSlice: 98 | indexConfig = &memdb.StringSliceFieldIndex{Field: index.Attribute} 99 | 100 | case IndexTypeMap: 101 | indexConfig = &memdb.StringMapFieldIndex{Field: index.Attribute} 102 | 103 | case IndexTypeString: 104 | indexConfig = &memdb.StringFieldIndex{Field: index.Attribute} 105 | 106 | case IndexTypeBoolean: 107 | attr := index.Attribute 108 | indexConfig = &memdb.ConditionalIndex{Conditional: func(obj any) (bool, error) { 109 | return boolIndex(obj, attr) 110 | }} 111 | 112 | case IndexTypeStringBased: 113 | indexConfig = &stringBasedFieldIndex{Field: index.Attribute} 114 | 115 | default: // if the caller is a bozo 116 | return nil, fmt.Errorf("invalid index type: %d", index.Type) 117 | } 118 | 119 | tableSchema.Indexes[index.Name] = &memdb.IndexSchema{ 120 | Name: index.Name, 121 | Unique: index.Unique, 122 | Indexer: indexConfig, 123 | AllowMissing: true, 124 | } 125 | } 126 | 127 | return tableSchema, nil 128 | } 129 | 130 | // boolIndex is a conditional indexer for booleans. 131 | func boolIndex(obj any, field string) (bool, error) { 132 | 133 | v := reflect.ValueOf(obj) 134 | v = reflect.Indirect(v) // Dereference the pointer if any 135 | 136 | fv := v.FieldByName(field) 137 | if !fv.IsValid() { 138 | return false, fmt.Errorf("field '%s' for %#v is invalid", field, obj) 139 | } 140 | 141 | return fv.Bool(), nil 142 | } 143 | 144 | func mergeIn(target, source *map[string]elemental.Identifiable) { 145 | for k, v := range *source { 146 | (*target)[k] = v 147 | } 148 | } 149 | 150 | func intersection(target, source *map[string]elemental.Identifiable, queryStart bool) { 151 | 152 | combined := map[string]elemental.Identifiable{} 153 | 154 | for k, v := range *source { 155 | if _, ok := (*target)[k]; ok || queryStart { 156 | combined[k] = v 157 | } 158 | } 159 | 160 | *target = combined 161 | } 162 | -------------------------------------------------------------------------------- /manipcli/update_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | // nolint:revive // Allow dot imports for readability in tests 9 | . "github.com/smartystreets/goconvey/convey" 10 | "go.aporeto.io/elemental" 11 | testmodel "go.aporeto.io/elemental/test/model" 12 | "go.aporeto.io/manipulate" 13 | "go.aporeto.io/manipulate/maniphttp" 14 | "go.aporeto.io/manipulate/maniptest" 15 | ) 16 | 17 | func Test_generateUpdateCommandForIdentity(t *testing.T) { 18 | 19 | Convey("Given I generate a update command", t, func() { 20 | 21 | task1 := testmodel.NewTask() 22 | task1.ID = "617aec75a829de0001da2032" 23 | task1.Name = "task1" 24 | 25 | m := maniptest.NewTestManipulator() 26 | m.MockRetrieve(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 27 | object.SetIdentifier(task1.ID) 28 | object.(*testmodel.Task).Name = task1.Name 29 | return nil 30 | }) 31 | 32 | m.MockUpdate(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 33 | return nil 34 | }) 35 | 36 | cmd, err := generateUpdateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 37 | return m, nil 38 | }) 39 | 40 | So(err, ShouldEqual, nil) 41 | assertCommandAndSetFlags(cmd) 42 | 43 | Convey("When I call execute with a json output", func() { 44 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 45 | output := bytes.NewBufferString("") 46 | cmd.SetOut(output) 47 | err := cmd.Execute() 48 | 49 | Convey("Then I should get a generated command", func() { 50 | So(err, ShouldEqual, nil) 51 | So(output.String(), ShouldEqual, task1.ID) 52 | }) 53 | }) 54 | }) 55 | 56 | Convey("Given a manipulator that returns an error on update", t, func() { 57 | 58 | task1 := testmodel.NewTask() 59 | task1.ID = "617aec75a829de0001da2032" 60 | task1.Name = "task1" 61 | 62 | m := maniptest.NewTestManipulator() 63 | m.MockRetrieve(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 64 | object.SetIdentifier(task1.ID) 65 | object.(*testmodel.Task).Name = task1.Name 66 | return nil 67 | }) 68 | 69 | m.MockUpdate(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 70 | return fmt.Errorf("update boom") 71 | }) 72 | 73 | cmd, err := generateUpdateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 74 | return m, nil 75 | }) 76 | 77 | So(err, ShouldEqual, nil) 78 | assertCommandAndSetFlags(cmd) 79 | 80 | Convey("When I call execute with a json output", func() { 81 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 82 | output := bytes.NewBufferString("") 83 | cmd.SetOut(output) 84 | err := cmd.Execute() 85 | 86 | Convey("Then I should get a generated command", func() { 87 | So(err, ShouldNotEqual, nil) 88 | So(err.Error(), ShouldEqual, "unable to update task: update boom") 89 | }) 90 | }) 91 | }) 92 | 93 | Convey("Given a manipulator that returns an error on retrieve", t, func() { 94 | 95 | task1 := testmodel.NewTask() 96 | task1.ID = "617aec75a829de0001da2032" 97 | task1.Name = "task1" 98 | 99 | m := maniptest.NewTestManipulator() 100 | m.MockRetrieve(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 101 | return fmt.Errorf("retrieve boom") 102 | }) 103 | 104 | m.MockUpdate(t, func(mctx manipulate.Context, object elemental.Identifiable) error { 105 | return nil 106 | }) 107 | 108 | cmd, err := generateUpdateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 109 | return m, nil 110 | }) 111 | 112 | So(err, ShouldEqual, nil) 113 | assertCommandAndSetFlags(cmd) 114 | 115 | Convey("When I call execute with a json output", func() { 116 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 117 | output := bytes.NewBufferString("") 118 | cmd.SetOut(output) 119 | err := cmd.Execute() 120 | 121 | Convey("Then I should get a generated command", func() { 122 | So(err, ShouldNotEqual, nil) 123 | So(err.Error(), ShouldEqual, "unable to retrieve task: retrieve boom") 124 | }) 125 | }) 126 | }) 127 | 128 | Convey("Given I generate a update command that returns an error", t, func() { 129 | 130 | cmd, err := generateUpdateCommandForIdentity(testmodel.TaskIdentity, testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 131 | return nil, fmt.Errorf("boom") 132 | }) 133 | 134 | So(err, ShouldEqual, nil) 135 | assertCommandAndSetFlags(cmd) 136 | 137 | Convey("When I call execute", func() { 138 | 139 | cmd.SetArgs([]string{"617aec75a829de0001da2032"}) 140 | err := cmd.Execute() 141 | 142 | Convey("Then I should get an error", func() { 143 | So(err, ShouldNotEqual, nil) 144 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 145 | }) 146 | }) 147 | }) 148 | 149 | } 150 | -------------------------------------------------------------------------------- /manipulate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "context" 16 | 17 | "go.aporeto.io/elemental" 18 | ) 19 | 20 | // Manipulator is the interface of a storage backend. 21 | type Manipulator interface { 22 | 23 | // RetrieveMany retrieves the a list of objects with the given elemental.Identity and put them in the given dest. 24 | RetrieveMany(mctx Context, dest elemental.Identifiables) error 25 | 26 | // Retrieve retrieves one or multiple elemental.Identifiables. 27 | // In order to be retrievable, the elemental.Identifiable needs to have their Identifier correctly set. 28 | Retrieve(mctx Context, object elemental.Identifiable) error 29 | 30 | // Create creates a the given elemental.Identifiables. 31 | Create(mctx Context, object elemental.Identifiable) error 32 | 33 | // Update updates one or multiple elemental.Identifiables. 34 | // In order to be updatable, the elemental.Identifiable needs to have their Identifier correctly set. 35 | Update(mctx Context, object elemental.Identifiable) error 36 | 37 | // Delete deletes one or multiple elemental.Identifiables. 38 | // In order to be deletable, the elemental.Identifiable needs to have their Identifier correctly set. 39 | Delete(mctx Context, object elemental.Identifiable) error 40 | 41 | // DeleteMany deletes all objects of with the given identity or 42 | // all the ones matching the filter in the given context. 43 | DeleteMany(mctx Context, identity elemental.Identity) error 44 | 45 | // Count returns the number of objects with the given identity. 46 | Count(mctx Context, identity elemental.Identity) (int, error) 47 | } 48 | 49 | // A TransactionalManipulator is a Manipulator that handles transactions. 50 | type TransactionalManipulator interface { 51 | 52 | // Commit commits the given TransactionID. 53 | Commit(id TransactionID) error 54 | 55 | // Abort aborts the give TransactionID. It returns true if 56 | // a transaction has been effectively aborted, otherwise it returns false. 57 | Abort(id TransactionID) bool 58 | 59 | Manipulator 60 | } 61 | 62 | // A FlushableManipulator is a manipulator that can flush its 63 | // content to somewhere, like a file. 64 | type FlushableManipulator interface { 65 | 66 | // Flush flushes and empties the cache. 67 | Flush(ctx context.Context) error 68 | } 69 | 70 | // A BufferedManipulator is a Manipulator with a local cache 71 | type BufferedManipulator interface { 72 | FlushableManipulator 73 | Manipulator 74 | } 75 | 76 | // SubscriberStatus is the type of a subscriber status. 77 | type SubscriberStatus int 78 | 79 | // Various values of SubscriberEvent. 80 | const ( 81 | SubscriberStatusInitialConnection SubscriberStatus = iota + 1 82 | SubscriberStatusInitialConnectionFailure 83 | SubscriberStatusReconnection 84 | SubscriberStatusReconnectionFailure 85 | SubscriberStatusDisconnection 86 | SubscriberStatusFinalDisconnection 87 | SubscriberStatusTokenRenewal 88 | ) 89 | 90 | // A Subscriber is the interface to control a push event subscription. 91 | type Subscriber interface { 92 | 93 | // Start connects to the websocket and starts collecting events 94 | // until the given context is canceled or any non communication error is 95 | // received. The eventual error will be received in the Errors() channel. 96 | // If not nil, the given push config will be applied right away. 97 | Start(context.Context, *elemental.PushConfig) 98 | 99 | // UpdateFilter updates the current push config. 100 | UpdateFilter(*elemental.PushConfig) 101 | 102 | // Events returns the events channel. 103 | Events() chan *elemental.Event 104 | 105 | // Errors returns the errors channel. 106 | Errors() chan error 107 | 108 | // Status returns the status channel. 109 | Status() chan SubscriberStatus 110 | } 111 | 112 | // A TokenManager issues an renew tokens periodically. 113 | type TokenManager interface { 114 | 115 | // Issues isses a new token. 116 | Issue(context.Context) (string, error) 117 | 118 | // Run runs the token renewal job and published the new token in the 119 | // given channel. 120 | Run(ctx context.Context, tokenCh chan string) 121 | } 122 | 123 | // A SelfTokenManager is a TokenManager that can use the manipulator it is used 124 | // with to retrieve a token. Manipulator will call this function passing 125 | // itself before using any other method of the interface. 126 | type SelfTokenManager interface { 127 | 128 | // SetManipulator will be called to inject the manipulator. 129 | SetManipulator(Manipulator) 130 | 131 | TokenManager 132 | } 133 | -------------------------------------------------------------------------------- /maniphttp/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package maniphttp 13 | 14 | import ( 15 | "context" 16 | "crypto/tls" 17 | "crypto/x509" 18 | "fmt" 19 | "io" 20 | "net" 21 | "net/http" 22 | "strconv" 23 | "strings" 24 | "sync" 25 | "time" 26 | 27 | "go.aporeto.io/elemental" 28 | "go.aporeto.io/manipulate" 29 | "go.aporeto.io/manipulate/maniphttp/internal/compiler" 30 | "go.aporeto.io/manipulate/maniphttp/internal/syscall" 31 | ) 32 | 33 | // AddQueryParameters appends each key-value pair from ctx.Parameters 34 | // to a request as query parameters with proper escaping. 35 | func addQueryParameters(req *http.Request, ctx manipulate.Context) error { 36 | 37 | q := req.URL.Query() 38 | 39 | if f := ctx.Filter(); f != nil { 40 | query, err := compiler.CompileFilter(f) 41 | if err != nil { 42 | return err 43 | } 44 | for k, v := range query { 45 | q[k] = v 46 | } 47 | } 48 | 49 | for k, v := range ctx.Parameters() { 50 | q[k] = v 51 | } 52 | 53 | for _, order := range ctx.Order() { 54 | q.Add("order", order) 55 | } 56 | 57 | if p := ctx.Page(); p != 0 { 58 | q.Add("page", strconv.Itoa(p)) 59 | } 60 | 61 | if p := ctx.PageSize(); p > 0 { 62 | q.Add("pagesize", strconv.Itoa(p)) 63 | } 64 | 65 | if p := ctx.After(); p != "" { 66 | q.Add("after", p) 67 | } 68 | 69 | if l := ctx.Limit(); l > 0 { 70 | q.Add("limit", strconv.Itoa(l)) 71 | } 72 | 73 | if ctx.Recursive() { 74 | q.Add("recursive", "true") 75 | } 76 | 77 | if ctx.Override() { 78 | q.Add("override", "true") 79 | } 80 | 81 | if ctx.Propagated() { 82 | q.Add("propagated", "true") 83 | } 84 | 85 | req.URL.RawQuery = q.Encode() 86 | 87 | return nil 88 | } 89 | 90 | func decodeData(r *http.Response, dest any) (err error) { 91 | 92 | if r.Body == nil { 93 | return manipulate.ErrCannotUnmarshal{Err: fmt.Errorf("nil reader")} 94 | } 95 | 96 | var data []byte 97 | if data, err = io.ReadAll(r.Body); err != nil { 98 | return manipulate.ErrCannotUnmarshal{Err: fmt.Errorf("unable to read data: %w", err)} 99 | } 100 | 101 | encoding := elemental.EncodingTypeJSON 102 | if r.Header.Get("Content-Type") != "" { 103 | encoding, _, err = elemental.EncodingFromHeaders(r.Header) 104 | if err != nil { 105 | return elemental.NewErrors(err) 106 | } 107 | } 108 | 109 | if err = elemental.Decode(encoding, data, dest); err != nil { 110 | return manipulate.ErrCannotUnmarshal{Err: fmt.Errorf("%w. original data:\n%s", err, string(data))} 111 | } 112 | 113 | return nil 114 | } 115 | 116 | var systemCertPoolLock sync.Mutex 117 | var systemCertPool *x509.CertPool 118 | 119 | func getDefaultTLSConfig() *tls.Config { 120 | 121 | systemCertPoolLock.Lock() 122 | defer systemCertPoolLock.Unlock() 123 | 124 | if systemCertPool == nil { 125 | var err error 126 | if systemCertPool, err = x509.SystemCertPool(); err != nil { 127 | panic(fmt.Sprintf("Unable to load system root cert pool: %s", err)) 128 | } 129 | } 130 | 131 | return &tls.Config{ 132 | RootCAs: systemCertPool, 133 | ClientSessionCache: tls.NewLRUClientSessionCache(0), 134 | } 135 | } 136 | 137 | func getDefaultHTTPTransport(url string, disableCompression bool, tcpUserTimeout time.Duration) (*http.Transport, string) { 138 | 139 | dialer := (&net.Dialer{ 140 | Timeout: 10 * time.Second, 141 | KeepAlive: 30 * time.Second, 142 | Control: syscall.MakeDialerControlFunc(tcpUserTimeout), 143 | }).DialContext 144 | 145 | outURL := url 146 | isUnix := strings.HasPrefix(url, "unix://") 147 | isUnixTLS := strings.HasPrefix(url, "unixs://") 148 | 149 | if isUnix || isUnixTLS { 150 | 151 | if isUnixTLS { 152 | outURL = "https://localhost" 153 | } else { 154 | outURL = "http://localhost" 155 | } 156 | 157 | dialer = func(context.Context, string, string) (net.Conn, error) { 158 | return net.Dial("unix", strings.TrimPrefix(strings.TrimPrefix(url, "unix://"), "unixs://")) 159 | } 160 | } 161 | 162 | return &http.Transport{ 163 | ForceAttemptHTTP2: true, 164 | Proxy: http.ProxyFromEnvironment, 165 | DialContext: dialer, 166 | MaxConnsPerHost: 32, 167 | MaxIdleConns: 32, 168 | MaxIdleConnsPerHost: 32, 169 | IdleConnTimeout: 90 * time.Second, 170 | TLSHandshakeTimeout: 10 * time.Second, 171 | ExpectContinueTimeout: 1 * time.Second, 172 | DisableCompression: disableCompression, 173 | }, outURL 174 | } 175 | 176 | func getDefaultClient() *http.Client { 177 | return &http.Client{ 178 | Timeout: 0, // we manage timeouts with contexts only. 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /errors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "errors" 16 | "fmt" 17 | "testing" 18 | 19 | // nolint:revive // Allow dot imports for readability in tests 20 | . "github.com/smartystreets/goconvey/convey" 21 | ) 22 | 23 | func genericErrorTest( 24 | t *testing.T, 25 | errorPrefix string, 26 | makeFactory func(error) error, 27 | makeFactoryOld func(string) error, 28 | verifierFunc func(error) bool, 29 | ) { 30 | 31 | Convey("When I create an error", t, func() { 32 | 33 | oerr := fmt.Errorf("this is a an error") 34 | err := makeFactory(oerr) 35 | 36 | Convey("Then it should be correct", func() { 37 | So(err.Error(), ShouldEqual, errorPrefix+"this is a an error") 38 | So(errors.Is(err, oerr), ShouldBeTrue) 39 | So(verifierFunc(err), ShouldBeTrue) 40 | }) 41 | 42 | olderr := makeFactoryOld("this is a an error") 43 | Convey("Then the old error should be correct", func() { 44 | So(olderr.Error(), ShouldEqual, errorPrefix+"this is a an error") 45 | So(verifierFunc(err), ShouldBeTrue) 46 | }) 47 | }) 48 | } 49 | 50 | func TestThing_Function(t *testing.T) { 51 | 52 | genericErrorTest( 53 | t, 54 | "Unable to unmarshal data: ", 55 | func(err error) error { return ErrCannotUnmarshal{Err: err} }, 56 | func(msg string) error { return NewErrCannotUnmarshal(msg) }, 57 | IsCannotUnmarshalError, 58 | ) 59 | 60 | genericErrorTest( 61 | t, 62 | "Unable to marshal data: ", 63 | func(err error) error { return ErrCannotMarshal{Err: err} }, 64 | func(msg string) error { return NewErrCannotMarshal(msg) }, 65 | IsCannotMarshalError, 66 | ) 67 | 68 | genericErrorTest( 69 | t, 70 | "Object not found: ", 71 | func(err error) error { return ErrObjectNotFound{Err: err} }, 72 | func(msg string) error { return NewErrObjectNotFound(msg) }, 73 | IsObjectNotFoundError, 74 | ) 75 | 76 | genericErrorTest( 77 | t, 78 | "Multiple objects found: ", 79 | func(err error) error { return ErrMultipleObjectsFound{Err: err} }, 80 | func(msg string) error { return NewErrMultipleObjectsFound(msg) }, 81 | IsMultipleObjectsFoundError, 82 | ) 83 | 84 | genericErrorTest( 85 | t, 86 | "Unable to build query: ", 87 | func(err error) error { return ErrCannotBuildQuery{Err: err} }, 88 | func(msg string) error { return NewErrCannotBuildQuery(msg) }, 89 | IsCannotBuildQueryError, 90 | ) 91 | 92 | genericErrorTest( 93 | t, 94 | "Unable to execute query: ", 95 | func(err error) error { return ErrCannotExecuteQuery{Err: err} }, 96 | func(msg string) error { return NewErrCannotExecuteQuery(msg) }, 97 | IsCannotExecuteQueryError, 98 | ) 99 | 100 | genericErrorTest( 101 | t, 102 | "Unable to commit transaction: ", 103 | func(err error) error { return ErrCannotCommit{Err: err} }, 104 | func(msg string) error { return NewErrCannotCommit(msg) }, 105 | IsCannotCommitError, 106 | ) 107 | 108 | genericErrorTest( 109 | t, 110 | "Not implemented: ", 111 | func(err error) error { return ErrNotImplemented{Err: err} }, 112 | func(msg string) error { return NewErrNotImplemented(msg) }, 113 | IsNotImplementedError, 114 | ) 115 | 116 | genericErrorTest( 117 | t, 118 | "Cannot communicate: ", 119 | func(err error) error { return ErrCannotCommunicate{Err: err} }, 120 | func(msg string) error { return NewErrCannotCommunicate(msg) }, 121 | IsCannotCommunicateError, 122 | ) 123 | 124 | genericErrorTest( 125 | t, 126 | "Cannot communicate: ", 127 | func(err error) error { return ErrLocked{Err: err} }, 128 | func(msg string) error { return NewErrLocked(msg) }, 129 | IsLockedError, 130 | ) 131 | 132 | genericErrorTest( 133 | t, 134 | "Transaction not found: ", 135 | func(err error) error { return ErrTransactionNotFound{Err: err} }, 136 | func(msg string) error { return NewErrTransactionNotFound(msg) }, 137 | IsTransactionNotFoundError, 138 | ) 139 | 140 | genericErrorTest( 141 | t, 142 | "Constraint violation: ", 143 | func(err error) error { return ErrConstraintViolation{Err: err} }, 144 | func(msg string) error { return NewErrConstraintViolation(msg) }, 145 | IsConstraintViolationError, 146 | ) 147 | 148 | genericErrorTest( 149 | t, 150 | "Disconnected: ", 151 | func(err error) error { return ErrDisconnected{Err: err} }, 152 | func(msg string) error { return NewErrDisconnected(msg) }, 153 | IsDisconnectedError, 154 | ) 155 | 156 | genericErrorTest( 157 | t, 158 | "Too many requests: ", 159 | func(err error) error { return ErrTooManyRequests{Err: err} }, 160 | func(msg string) error { return NewErrTooManyRequests(msg) }, 161 | IsTooManyRequestsError, 162 | ) 163 | 164 | genericErrorTest( 165 | t, 166 | "TLS error: ", 167 | func(err error) error { return ErrTLS{Err: err} }, 168 | func(msg string) error { return NewErrTLS(msg) }, 169 | IsTLSError, 170 | ) 171 | } 172 | -------------------------------------------------------------------------------- /iter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Aporeto Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package manipulate 13 | 14 | import ( 15 | "context" 16 | "fmt" 17 | 18 | "go.aporeto.io/elemental" 19 | ) 20 | 21 | const iterDefaultBlockSize = 1000 22 | 23 | // IterFunc calls RetrieveMany on the given Manipulator, and will retrieve the data by block 24 | // of the given blockSize. 25 | // 26 | // IterFunc will naturally ends and return when there is no more data to pull. 27 | // 28 | // For each retrieved block, the given func will be called with the 29 | // current data block. If the function returns an error, the error is returned to the caller 30 | // of IterFunc and the iteration stops. 31 | // 32 | // The given context will be used if the underlying manipulator honors it. Be careful to NOT pass 33 | // a filter matching objects then updating the objects to not match anynmore. This would shift 34 | // pagination and will produce unexpected results. To do so, prefer using manipulate.IterUntilFunc 35 | // 36 | // The given manipulate.Context will be used to retry any failed batch recovery. 37 | // 38 | // The identifiablesTemplate parameter is must be an empty elemental.Identifiables that will be used to 39 | // hold the data block. It is reset at every iteration. Do not rely on it to be filled 40 | // once IterFunc is complete. 41 | // 42 | // Finally, if the given blockSize is <= 0, then it will use the default that is 1000. 43 | func IterFunc( 44 | ctx context.Context, 45 | manipulator Manipulator, 46 | identifiablesTemplate elemental.Identifiables, 47 | mctx Context, 48 | iteratorFunc func(block elemental.Identifiables) error, 49 | blockSize int, 50 | ) error { 51 | return doIterFunc(ctx, manipulator, identifiablesTemplate, mctx, iteratorFunc, blockSize, false) 52 | } 53 | 54 | // IterUntilFunc works as IterFunc but pagination will not increase. 55 | // It will always retrieve the first page with a size of given blockSize. 56 | // 57 | // The goal of this function is to be used with a filter, then update (or delete) the 58 | // objects that match until no more are matching. 59 | func IterUntilFunc( 60 | ctx context.Context, 61 | manipulator Manipulator, 62 | identifiablesTemplate elemental.Identifiables, 63 | mctx Context, 64 | iteratorFunc func(block elemental.Identifiables) error, 65 | blockSize int, 66 | ) error { 67 | return doIterFunc(ctx, manipulator, identifiablesTemplate, mctx, iteratorFunc, blockSize, true) 68 | } 69 | 70 | // Iter is a helper function for IterFunc. 71 | // 72 | // It will simply iterates on the object with identity of the given elemental.Identifiables. 73 | // Not that this function cannot populate the data in the identifiable parameter. Instead 74 | // It will return the destination. 75 | // 76 | // # Always pass an empty elemental.Identifiables to this function 77 | // 78 | // For more information, please check IterFunc documentation. 79 | // 80 | // Example: 81 | // 82 | // dest, err := Iter(context.Background(), m, mctx, model.ThingsList{}, 100) 83 | func Iter( 84 | ctx context.Context, 85 | m Manipulator, 86 | mctx Context, 87 | identifiablesTemplate elemental.Identifiables, 88 | blockSize int, 89 | ) (elemental.Identifiables, error) { 90 | 91 | if err := IterFunc( 92 | ctx, 93 | m, 94 | identifiablesTemplate, 95 | mctx, 96 | func(block elemental.Identifiables) error { 97 | identifiablesTemplate = identifiablesTemplate.Append(block.List()...) 98 | return nil 99 | }, 100 | blockSize, 101 | ); err != nil { 102 | return nil, err 103 | } 104 | 105 | return identifiablesTemplate, nil 106 | } 107 | 108 | func doIterFunc( 109 | ctx context.Context, 110 | manipulator Manipulator, 111 | identifiablesTemplate elemental.Identifiables, 112 | mctx Context, 113 | iteratorFunc func(block elemental.Identifiables) error, 114 | blockSize int, 115 | disablePageIncrease bool, 116 | ) error { 117 | 118 | if manipulator == nil { 119 | panic("manipulator must not be nil") 120 | } 121 | 122 | if iteratorFunc == nil { 123 | panic("iteratorFunc must not be nil") 124 | } 125 | 126 | if identifiablesTemplate == nil { 127 | panic("identifiablesTemplate must not be nil") 128 | } 129 | 130 | if mctx == nil { 131 | mctx = NewContext(ctx) 132 | } 133 | 134 | if blockSize <= 0 { 135 | blockSize = iterDefaultBlockSize 136 | } 137 | 138 | var iter int 139 | var after string 140 | 141 | for { 142 | iter++ 143 | 144 | objects := identifiablesTemplate.Copy() 145 | 146 | smctx := mctx.Derive(ContextOptionAfter(after, blockSize)) 147 | 148 | if err := manipulator.RetrieveMany(smctx, objects); err != nil { 149 | return fmt.Errorf("unable to retrieve objects for iteration %d: %w", iter, err) 150 | } 151 | 152 | if len(objects.List()) == 0 { 153 | return nil 154 | } 155 | 156 | if err := iteratorFunc(objects); err != nil { 157 | return fmt.Errorf("iter function returned an error on iteration %d: %w", iter, err) 158 | } 159 | 160 | if smctx.Next() == "" { 161 | return nil 162 | } 163 | 164 | if !disablePageIncrease { 165 | after = smctx.Next() 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /manipcli/update.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/ghodss/yaml" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | "go.aporeto.io/elemental" 13 | "go.aporeto.io/manipulate" 14 | ) 15 | 16 | // generateUpdateCommandForIdentity generates the command to update an object based on its identity. 17 | func generateUpdateCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker, options ...cmdOption) (*cobra.Command, error) { 18 | 19 | cfg := &cmdConfig{} 20 | 21 | for _, opt := range options { 22 | opt(cfg) 23 | } 24 | 25 | cmd := &cobra.Command{ 26 | Use: fmt.Sprintf("%s ", identity.Name), 27 | Aliases: []string{identity.Category}, 28 | Short: "Update an existing " + identity.Name, 29 | Args: cobra.ExactArgs(1), 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | 32 | fParam := viper.GetStringSlice("param") 33 | fTrackingID := viper.GetString(flagTrackingID) 34 | fOutput := viper.GetString(flagOutput) 35 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 36 | fOutputTemplate := viper.GetString(flagOutputTemplate) 37 | fForce := viper.GetBool(flagForce) 38 | 39 | manipulator, err := manipulatorMaker() 40 | if err != nil { 41 | return fmt.Errorf("unable to make manipulator: %w", err) 42 | } 43 | 44 | parameters, err := parametersToURLValues(fParam) 45 | if err != nil { 46 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 47 | } 48 | 49 | options := []manipulate.ContextOption{ 50 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 51 | manipulate.ContextOptionParameters(parameters), 52 | manipulate.ContextOptionFields(fFormatTypeColumn), 53 | manipulate.ContextOptionOverride(fForce), 54 | } 55 | 56 | retrieveCtx, retrieveCancel := context.WithTimeout(cmd.Context(), 20*time.Second) 57 | defer retrieveCancel() 58 | 59 | mctx := manipulate.NewContext(retrieveCtx, options...) 60 | 61 | identifiable, err := retrieveObjectByIDOrByName(mctx, manipulator, identity, args[0], modelManager) 62 | if err != nil { 63 | return fmt.Errorf("unable to retrieve %s: %w", identity.Name, err) 64 | } 65 | 66 | if viper.GetBool(flagInteractive) { 67 | 68 | data, err := openInEditor(identifiable, viper.GetString(flagEditor), cmd.Short, true, false, false) 69 | if err != nil { 70 | return fmt.Errorf("unable to open editor %s: %w", viper.GetString(flagEditor), err) 71 | } 72 | 73 | if data == nil { 74 | return fmt.Errorf("empty data") 75 | } 76 | 77 | if err := json.Unmarshal(data, identifiable); err != nil { 78 | return fmt.Errorf("unable to unmarshall: %w", err) 79 | } 80 | 81 | } else if viper.IsSet(flagInputValues) || viper.IsSet(flagInputData) || viper.IsSet(flagInputFile) || viper.IsSet(flagInputURL) { 82 | 83 | data, err := readData(false) 84 | if err != nil { 85 | return fmt.Errorf("unable to read data: %w", err) 86 | } 87 | 88 | if data != nil { 89 | data, err = yaml.YAMLToJSON(data) 90 | if err != nil { 91 | return err 92 | } 93 | } 94 | 95 | if err := json.Unmarshal(data, identifiable); err != nil { 96 | return fmt.Errorf("unable to unmarshall: %w", err) 97 | } 98 | 99 | } else { 100 | 101 | if err := readViperFlags(identifiable, modelManager, cfg.argumentsPrefix); err != nil { 102 | return fmt.Errorf("unable to read flags: %w", err) 103 | } 104 | } 105 | 106 | updateCtx, updateCancel := context.WithTimeout(cmd.Context(), 20*time.Second) 107 | defer updateCancel() 108 | 109 | mctx = manipulate.NewContext(updateCtx, options...) 110 | 111 | if err := manipulator.Update(mctx, identifiable); err != nil { 112 | return fmt.Errorf("unable to update %s: %w", identity.Name, err) 113 | } 114 | 115 | outputType := fOutput 116 | if fOutput == flagOutputDefault { 117 | outputType = flagOutputNone 118 | } 119 | 120 | result, err := formatObjects( 121 | prepareOutputFormat(outputType, formatTypeHash, fFormatTypeColumn, fOutputTemplate), 122 | false, 123 | identifiable, 124 | ) 125 | 126 | if err != nil { 127 | return fmt.Errorf("unable to format output: %w", err) 128 | } 129 | 130 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 131 | return nil 132 | }, 133 | } 134 | 135 | cmd.Flags().BoolP(flagForce, "", false, "Force modification of protected object") 136 | cmd.Flags().String(flagInputValues, "", "Optional path to file containing templating values") 137 | cmd.Flags().StringP(flagInputData, "d", "", "Data of the request body in the JSON format.") 138 | cmd.Flags().StringP(flagInputFile, "f", "", "Optional file to read the data from. Set `-` to read from stdin") 139 | cmd.Flags().StringP(flagInputURL, "u", "", "Optional url where to read the data from. If you don't set it, stdin or --file will used") 140 | cmd.Flags().StringSlice(flagInputSet, nil, "Set a value to in the imported data in case it is a Go template.") 141 | cmd.Flags().Bool(flagPrint, false, "If set will print the raw data. Only works for --file and --url") 142 | cmd.Flags().Bool(flagRender, false, "If set will render and print the data. Only works for --file and --url") 143 | cmd.Flags().BoolP(flagInteractive, "i", false, "Set to create the object in the given --editor.") 144 | cmd.Flags().StringP(flagEditor, "", "", "Choose the editor when using --interactive.") 145 | 146 | identifiable := modelManager.IdentifiableFromString(identity.Name) 147 | if err := setViperFlags(cmd, identifiable, modelManager, cfg.argumentsPrefix); err != nil { 148 | return nil, fmt.Errorf("unable to set flags for %s: %w", identity.Name, err) 149 | } 150 | 151 | return cmd, nil 152 | } 153 | -------------------------------------------------------------------------------- /manipcli/listen_test.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/gorilla/websocket" 13 | // nolint:revive // Allow dot imports for readability in tests 14 | . "github.com/smartystreets/goconvey/convey" 15 | "go.aporeto.io/elemental" 16 | testmodel "go.aporeto.io/elemental/test/model" 17 | "go.aporeto.io/manipulate" 18 | "go.aporeto.io/manipulate/maniphttp" 19 | ) 20 | 21 | func Test_generateListenCommand(t *testing.T) { 22 | 23 | Convey("Given a server that can listen", t, func() { 24 | 25 | event := &elemental.Event{ 26 | Identity: testmodel.TaskIdentity.Name, 27 | Type: elemental.EventCreate, 28 | } 29 | 30 | data, err := elemental.Encode(elemental.EncodingTypeJSON, event) 31 | if err != nil { 32 | fmt.Println("can't encode task", err) 33 | return 34 | } 35 | 36 | ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | 38 | var upgrader = websocket.Upgrader{} 39 | ws, err := upgrader.Upgrade(w, r, nil) 40 | if err != nil { 41 | fmt.Println("cannot upgrade", err) 42 | return 43 | } 44 | 45 | go func() { 46 | defer ws.Close() // nolint 47 | for { 48 | fmt.Println("writeMessage...") 49 | if err := ws.WriteMessage(websocket.TextMessage, data); err != nil { 50 | fmt.Println("cannot write message", err) 51 | return 52 | } 53 | <-time.After(1 * time.Second) 54 | } 55 | }() 56 | 57 | })) 58 | defer ts.Close() 59 | 60 | ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) 61 | defer cancel() 62 | 63 | tlsConfig := ts.Client().Transport.(*http.Transport).TLSClientConfig 64 | m, err := maniphttp.New(ctx, ts.URL, maniphttp.OptionTLSConfig(tlsConfig)) 65 | So(err, ShouldEqual, nil) 66 | 67 | cmd, err := generateListenCommand(testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 68 | return m, nil 69 | }) 70 | 71 | So(err, ShouldEqual, nil) 72 | assertCommandAndSetFlags(cmd) 73 | 74 | Convey("When I call execute", func() { 75 | 76 | cmd.Flags().Set(flagAPI, ts.URL) // nolint 77 | cmd.Flags().Set(flagNamespace, "/test") // nolint 78 | cmd.Flags().Set(flagEncoding, "json") // nolint 79 | cmd.Flags().Set(flagToken, "token1234") // nolint 80 | cmd.Flags().Set(flagAPISkipVerify, "true") // nolint 81 | cmd.Flags().Set("identity", "task") // nolint 82 | 83 | output := bytes.NewBufferString("") 84 | cmd.SetOut(output) 85 | 86 | execContext, execCancel := context.WithTimeout(context.Background(), 200*time.Millisecond) 87 | defer execCancel() 88 | 89 | err := cmd.ExecuteContext(execContext) 90 | 91 | Convey("Then I should get a generated command", func() { 92 | So(err, ShouldEqual, nil) 93 | So(output.String(), ShouldEqual, `{ 94 | "encoding": "", 95 | "entity": null, 96 | "identity": "task", 97 | "timestamp": null, 98 | "type": "create" 99 | }`) 100 | }) 101 | }) 102 | }) 103 | 104 | Convey("Given a server that fails", t, func() { 105 | 106 | ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 107 | 108 | w.WriteHeader(http.StatusInternalServerError) 109 | })) 110 | defer ts.Close() 111 | 112 | cmd, err := generateListenCommand(testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 113 | return nil, fmt.Errorf("boom") 114 | }) 115 | 116 | So(err, ShouldEqual, nil) 117 | assertCommandAndSetFlags(cmd) 118 | 119 | Convey("When I call execute", func() { 120 | 121 | cmd.Flags().Set(flagAPI, ts.URL) // nolint 122 | cmd.Flags().Set(flagNamespace, "/test") // nolint 123 | cmd.Flags().Set(flagEncoding, "json") // nolint 124 | cmd.Flags().Set(flagToken, "token1234") // nolint 125 | cmd.Flags().Set(flagAPISkipVerify, "true") // nolint 126 | cmd.Flags().Set("identity", "task") // nolint 127 | 128 | output := bytes.NewBufferString("") 129 | cmd.SetOut(output) 130 | 131 | execContext, execCancel := context.WithTimeout(context.Background(), 200*time.Millisecond) 132 | defer execCancel() 133 | 134 | err := cmd.ExecuteContext(execContext) 135 | 136 | Convey("Then I should get a generated command", func() { 137 | So(err, ShouldNotEqual, nil) 138 | So(err.Error(), ShouldEqual, "unable to make manipulator: boom") 139 | }) 140 | }) 141 | }) 142 | 143 | Convey("Given a command with wrong identity", t, func() { 144 | 145 | m, err := maniphttp.New(context.Background(), "https://blabla.bla") 146 | So(err, ShouldEqual, nil) 147 | 148 | cmd, err := generateListenCommand(testmodel.Manager(), func(opts ...maniphttp.Option) (manipulate.Manipulator, error) { 149 | return m, nil 150 | }) 151 | 152 | So(err, ShouldEqual, nil) 153 | assertCommandAndSetFlags(cmd) 154 | 155 | Convey("When I call execute", func() { 156 | 157 | cmd.Flags().Set(flagAPI, "https://mytest.chris") // nolint 158 | cmd.Flags().Set(flagNamespace, "/test") // nolint 159 | cmd.Flags().Set(flagEncoding, "json") // nolint 160 | cmd.Flags().Set(flagToken, "token1234") // nolint 161 | cmd.Flags().Set(flagAPISkipVerify, "true") // nolint 162 | cmd.Flags().Set("identity", "bip") // nolint 163 | 164 | output := bytes.NewBufferString("") 165 | cmd.SetOut(output) 166 | 167 | execContext, execCancel := context.WithTimeout(context.Background(), 200*time.Millisecond) 168 | defer execCancel() 169 | 170 | err := cmd.ExecuteContext(execContext) 171 | 172 | Convey("Then I should get a generated command", func() { 173 | So(err, ShouldNotEqual, nil) 174 | So(err.Error(), ShouldEqual, "unknown identity bip") 175 | }) 176 | }) 177 | }) 178 | } 179 | -------------------------------------------------------------------------------- /manipcli/create.go: -------------------------------------------------------------------------------- 1 | package manipcli 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/ghodss/yaml" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | "go.aporeto.io/elemental" 13 | "go.aporeto.io/manipulate" 14 | ) 15 | 16 | // generateCreateCommandForIdentity generates the command to create an object based on its identity. 17 | func generateCreateCommandForIdentity(identity elemental.Identity, modelManager elemental.ModelManager, manipulatorMaker ManipulatorMaker, options ...cmdOption) (*cobra.Command, error) { 18 | 19 | cfg := &cmdConfig{} 20 | 21 | for _, opt := range options { 22 | opt(cfg) 23 | } 24 | 25 | cmd := &cobra.Command{ 26 | Use: identity.Name, 27 | Aliases: []string{identity.Category}, 28 | Short: "Create a new " + identity.Name, 29 | // Aliases: TODO: Missing alias from the spec file -> To be stored in the identity ?, 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | 32 | fParam := viper.GetStringSlice("param") 33 | fTrackingID := viper.GetString(flagTrackingID) 34 | fOutput := viper.GetString(flagOutput) 35 | fFormatTypeColumn := viper.GetStringSlice(formatTypeColumn) 36 | fOutputTemplate := viper.GetString(flagOutputTemplate) 37 | 38 | manipulator, err := manipulatorMaker() 39 | if err != nil { 40 | return fmt.Errorf("unable to make manipulator: %w", err) 41 | } 42 | 43 | parameters, err := parametersToURLValues(fParam) 44 | if err != nil { 45 | return fmt.Errorf("unable to convert parameters to url values: %w", err) 46 | } 47 | 48 | options := []manipulate.ContextOption{ 49 | manipulate.ContextOptionTracking(fTrackingID, "cli"), 50 | manipulate.ContextOptionParameters(parameters), 51 | manipulate.ContextOptionFields(fFormatTypeColumn), 52 | } 53 | 54 | if viper.IsSet(flagParent) { 55 | parentName, parentID, err := splitParentInfo(viper.GetString(flagParent)) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | parent := modelManager.IdentifiableFromString(parentName) 61 | if parent == nil { 62 | return fmt.Errorf("unknown identity %s", parentName) 63 | } 64 | parent.SetIdentifier(parentID) 65 | options = append(options, manipulate.ContextOptionParent(parent)) 66 | } 67 | 68 | identifiable := modelManager.IdentifiableFromString(identity.Name) 69 | 70 | if viper.GetBool(flagInteractive) { 71 | 72 | data, err := openInEditor(identifiable, viper.GetString(flagEditor), cmd.Short, true, false, false) 73 | if err != nil { 74 | return fmt.Errorf("unable to open editor %s: %w", viper.GetString(flagEditor), err) 75 | } 76 | 77 | if data == nil { 78 | return fmt.Errorf("empty data") 79 | } 80 | 81 | if err := json.Unmarshal(data, identifiable); err != nil { 82 | return fmt.Errorf("unable to unmarshall: %w", err) 83 | } 84 | 85 | } else if viper.IsSet(flagInputValues) || viper.IsSet(flagInputData) || viper.IsSet(flagInputFile) || viper.IsSet(flagInputURL) { 86 | 87 | data, err := readData(false) 88 | if err != nil { 89 | return fmt.Errorf("unable to read data: %w", err) 90 | } 91 | 92 | if data != nil { 93 | data, err = yaml.YAMLToJSON(data) 94 | if err != nil { 95 | return err 96 | } 97 | } 98 | 99 | if err := json.Unmarshal(data, identifiable); err != nil { 100 | return fmt.Errorf("unable to unmarshall: %w", err) 101 | } 102 | 103 | } else { 104 | 105 | if err := readViperFlags(identifiable, modelManager, cfg.argumentsPrefix); err != nil { 106 | return fmt.Errorf("unable to read flags: %w", err) 107 | } 108 | 109 | } 110 | 111 | ctx, cancel := context.WithTimeout(cmd.Context(), 20*time.Second) 112 | defer cancel() 113 | 114 | mctx := manipulate.NewContext(ctx, options...) 115 | if err := manipulator.Create(mctx, identifiable); err != nil { 116 | return fmt.Errorf("unable to create %s: %w", identity.Name, err) 117 | } 118 | 119 | outputType := fOutput 120 | if fOutput == flagOutputDefault { 121 | outputType = flagOutputNone 122 | } 123 | 124 | result, err := formatObjects( 125 | prepareOutputFormat(outputType, formatTypeHash, fFormatTypeColumn, fOutputTemplate), 126 | false, 127 | identifiable, 128 | ) 129 | 130 | if err != nil { 131 | return fmt.Errorf("unable to format output: %w", err) 132 | } 133 | 134 | _, _ = fmt.Fprint(cmd.OutOrStdout(), result) 135 | return nil 136 | }, 137 | } 138 | 139 | identifiable := modelManager.IdentifiableFromString(identity.Name) 140 | if err := setViperFlags(cmd, identifiable, modelManager, cfg.argumentsPrefix); err != nil { 141 | return nil, fmt.Errorf("unable to set flags for %s: %w", identity.Name, err) 142 | } 143 | 144 | cmd.Flags().String(flagInputValues, "", "Optional path to file containing templating values") 145 | cmd.Flags().StringP(flagInputData, "d", "", "Data of the request body in the JSON format.") 146 | cmd.Flags().StringP(flagInputFile, "f", "", "Optional file to read the data from. Set `-` to read from stdin") 147 | cmd.Flags().StringP(flagInputURL, "u", "", "Optional url where to read the data from. If you don't set it, stdin or --file will used") 148 | cmd.Flags().StringSlice(flagInputSet, nil, "Set a value to in the imported data in case it is a Go template.") 149 | cmd.Flags().Bool(flagPrint, false, "If set will print the raw data. Only works for --file and --url") 150 | cmd.Flags().Bool(flagRender, false, "If set will render and print the data. Only works for --file and --url") 151 | cmd.Flags().BoolP(flagInteractive, "i", false, "Set to create the object in the given --editor.") 152 | cmd.Flags().StringP(flagEditor, "", "", "Choose the editor when using --interactive.") 153 | cmd.Flags().StringP(flagParent, "", "", "Provide information about parent resource. Format `name/ID`") 154 | 155 | return cmd, nil 156 | } 157 | --------------------------------------------------------------------------------