├── pkg ├── .gitkeep └── generator │ └── generator.go ├── internal ├── .gitkeep ├── store │ ├── store_test.go │ ├── store.go │ ├── azhelpers.go │ ├── paramstore.go │ ├── gcpsecrets.go │ ├── secretsmanager.go │ ├── azkeyvault.go │ ├── azappconf.go │ ├── aztablestorage.go │ ├── paramstore_test.go │ ├── secretsmanager_test.go │ ├── hashivault.go │ ├── azkeyvault_test.go │ ├── azappconf_test.go │ ├── gcpsecrets_test.go │ ├── aztablestorage_test.go │ └── hashivault_test.go ├── testutils │ └── testutils.go ├── log │ ├── log.go │ └── log_test.go ├── cmdutils │ ├── postprocessor_test.go │ ├── postprocessor.go │ ├── cmdutils.go │ └── cmdutils_test.go ├── strategy │ ├── strategy.go │ └── strategy_test.go └── config │ ├── config_test.go │ └── config.go ├── tools └── tools.go ├── cmd ├── configmanager │ ├── testdata │ │ └── input.yml.cm │ ├── retrieve_test.go │ ├── fromfileinput_test.go │ ├── insert.go │ ├── retrieve.go │ ├── configmanager_test.go │ ├── fromfileinput.go │ └── configmanager.go └── main.go ├── .gitignore ├── .github ├── dependabot.yml └── workflows │ ├── release_container.yml │ ├── release.yml │ └── build.yml ├── Dockerfile ├── docs ├── installation.md ├── adding-provider.md ├── commands.md └── examples.md ├── sonar-project.properties ├── LICENSE.md ├── .golangci.yaml ├── eirctl.yaml ├── examples └── examples.go ├── go.mod ├── configmanager.go └── README.md /pkg/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /internal/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /internal/store/store_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | 3 | package tools 4 | -------------------------------------------------------------------------------- /cmd/configmanager/testdata/input.yml.cm: -------------------------------------------------------------------------------- 1 | app: 2 | conf: UNKNOWN://foo/bar 3 | -------------------------------------------------------------------------------- /internal/testutils/testutils.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | const ( 4 | TestPhrase string = "got: %v want: %v\n" 5 | TestPhraseWithContext string = "%s\n got: %v\n\n want: %v\n" 6 | ) 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # mac 2 | .DS_Store 3 | 4 | # IDEs 5 | .vscode 6 | *.env 7 | 8 | # Go 9 | vendor 10 | bin 11 | dist 12 | .deps/ 13 | 14 | # tests 15 | .coverage 16 | 17 | # local testers and 18 | local/ 19 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | cfgmgr "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" 8 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 9 | ) 10 | 11 | func main() { 12 | logger := log.New(os.Stderr) 13 | cmd := cfgmgr.NewRootCmd(logger) 14 | if err := cmd.Execute(context.Background()); err != nil { 15 | os.Exit(1) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'gomod' 4 | directory: '/' 5 | schedule: 6 | interval: weekly 7 | day: monday 8 | timezone: Europe/London 9 | open-pull-requests-limit: 50 10 | assignees: 11 | - dnitsch 12 | - elvenspellmaker 13 | rebase-strategy: disabled 14 | groups: 15 | low-risk: 16 | applies-to: version-updates 17 | update-types: 18 | - "minor" 19 | - "patch" 20 | -------------------------------------------------------------------------------- /cmd/configmanager/retrieve_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import "testing" 4 | 5 | func TestRetrieve_Command(t *testing.T) { 6 | t.Run("should error on unknown token implementation", func(t *testing.T) { 7 | cmdRunTestHelper(t, &cmdTestInput{args: []string{"get", "--token", "UNKNOWN://foo/bar", "--token", "UNKNOWN://foo/bar1"}, errored: false}) 8 | }) 9 | t.Run("should error on missing flag", func(t *testing.T) { 10 | cmdRunTestHelper(t, &cmdTestInput{args: []string{"get"}, errored: true}) 11 | }) 12 | } 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG Version 2 | ARG Revision 3 | 4 | FROM docker.io/golang:1-trixie AS builder 5 | 6 | ARG Version=0.0.1 7 | ARG Revision=beta01 8 | 9 | WORKDIR /app 10 | 11 | COPY ./ /app 12 | RUN CGO_ENABLED=0 go build -mod=readonly -buildvcs=false \ 13 | -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \ 14 | -o bin/configmanager cmd/main.go 15 | 16 | FROM docker.io/alpine:3 17 | 18 | COPY --from=builder /app/bin/configmanager /usr/bin/configmanager 19 | 20 | ENTRYPOINT ["configmanager"] 21 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | Major platform binaries [here](https://github.com/DevLabFoundry/configmanager/v2/releases) 4 | 5 | *nix binary 6 | 7 | ```bash 8 | curl -L https://github.com/DevLabFoundry/configmanager/v2/releases/latest/download/configmanager-linux -o configmanager 9 | ``` 10 | 11 | MacOS binary 12 | 13 | ```bash 14 | curl -L https://github.com/DevLabFoundry/configmanager/v2/releases/latest/download/configmanager-darwin -o configmanager 15 | ``` 16 | 17 | ```bash 18 | chmod +x configmanager 19 | sudo mv configmanager /usr/local/bin 20 | ``` 21 | 22 | Download specific version: 23 | 24 | ```bash 25 | curl -L https://github.com/DevLabFoundry/configmanager/v2/releases/download/v0.5.0/configmanager-`uname -s` -o configmanager 26 | ``` 27 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.projectKey=dnitsch_configmanager 2 | sonar.organization=dnitsch 3 | # This is the name and version displayed in the SonarCloud UI. 4 | sonar.projectName=configmanager 5 | 6 | # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. 7 | sonar.sources=. 8 | sonar.exclusions=**/*_test.go,**/*_generated*.go,**/*_generated/**,**/vendor/**,**/examples/** 9 | sonar.inclusions=**/*.go 10 | 11 | sonar.tests=. 12 | sonar.test.inclusions=**/*_test.go 13 | sonar.test.exclusions=**/*_generated*.go,**/*_generated/**,**/vendor/** 14 | 15 | sonar.sourceEncoding=UTF-8 16 | sonar.qualitygate.wait=true 17 | 18 | # go 19 | sonar.go.coverage.reportPaths=.coverage/out 20 | sonar.go.tests.reportPaths=.coverage/report-junit.xml 21 | -------------------------------------------------------------------------------- /internal/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 7 | ) 8 | 9 | const implementationNetworkErr string = "implementation %s error: %v for token: %s" 10 | 11 | var ( 12 | ErrRetrieveFailed = errors.New("failed to retrieve config item") 13 | ErrClientInitialization = errors.New("failed to initialize the client") 14 | ErrEmptyResponse = errors.New("value retrieved but empty for token") 15 | ErrServiceCallFailed = errors.New("failed to complete the service call") 16 | ) 17 | 18 | // Strategy iface that all store implementations 19 | // must conform to, in order to be be used by the retrieval implementation 20 | // 21 | // Defined on the package for easier re-use across the program 22 | type Strategy interface { 23 | Token() (s string, e error) 24 | SetToken(s *config.ParsedTokenConfig) 25 | } 26 | -------------------------------------------------------------------------------- /cmd/configmanager/fromfileinput_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestFromStr_command(t *testing.T) { 9 | t.Run("should error on unknown token implementation", func(t *testing.T) { 10 | defer os.Remove("testdata/input.yml") 11 | cmdRunTestHelper(t, &cmdTestInput{args: []string{"fromstr", "--input", "testdata/input.yml.cm", "--path", "testdata/input.yml"}, errored: false}) 12 | }) 13 | t.Run("should error on missing flag", func(t *testing.T) { 14 | cmdRunTestHelper(t, &cmdTestInput{args: []string{"fromstr", "--path", "testdata/input.yml"}, errored: true}) 15 | }) 16 | t.Run("should error on missing var when envsubst enabled", func(t *testing.T) { 17 | cmdRunTestHelper(t, &cmdTestInput{args: []string{"fromstr", "--input", "foo AWSPARAMSTR://${NOT_FOUND}", "-s", "://", "--enable-envsubst", "--path", "stdout"}, errored: true}) 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /cmd/configmanager/insert.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | type insertFlags struct { 10 | insertKv map[string]string 11 | } 12 | 13 | func newInsertCmd(rootCmd *Root) { 14 | defaultInsertKv := make(map[string]string) 15 | f := &insertFlags{} 16 | insertCmd := &cobra.Command{ 17 | Use: "insert", 18 | Aliases: []string{"i", "send", "put"}, 19 | Short: `Creates the config item in the designated backing store`, 20 | Long: ``, 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | return fmt.Errorf("not yet implemented") 23 | }, 24 | PreRunE: func(cmd *cobra.Command, args []string) error { 25 | if len(f.insertKv) < 1 { 26 | return fmt.Errorf("must include at least 1 token map") 27 | } 28 | return nil 29 | }, 30 | } 31 | insertCmd.PersistentFlags().StringToStringVarP(&f.insertKv, "config-pair", "", defaultInsertKv, " token=value pair. This can be specified multiple times.") 32 | _ = insertCmd.MarkPersistentFlagRequired("config-pair") 33 | rootCmd.Cmd.AddCommand(insertCmd) 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 DevLabFoundry 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | # Default set of linters. 4 | # The value can be: `standard`, `all`, `none`, or `fast`. 5 | # Default: standard 6 | default: standard 7 | exclusions: 8 | generated: lax 9 | # Log a warning if an exclusion rule is unused. 10 | # Default: false 11 | warn-unused: true 12 | # Predefined exclusion rules. 13 | # Default: [] 14 | presets: 15 | - comments 16 | - std-error-handling 17 | - common-false-positives 18 | - legacy 19 | # Excluding configuration per-path, per-linter, per-text and per-source. 20 | rules: 21 | # Exclude some linters from running on tests files. 22 | - path: _test\.go 23 | linters: 24 | - gocyclo 25 | - errcheck 26 | - dupl 27 | - gosec 28 | - ineffassign 29 | - staticcheck 30 | - unused 31 | - govet 32 | # Exclude some linters from running on examples files. 33 | - path: examples/ 34 | linters: 35 | - gocyclo 36 | - errcheck 37 | - dupl 38 | - gosec 39 | - ineffassign 40 | - staticcheck 41 | - unused 42 | - govet -------------------------------------------------------------------------------- /internal/store/azhelpers.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | /* 9 | Generic Azure Service Init Helpers 10 | */ 11 | // azServiceHelper returns a service URI and the stripped token 12 | type azServiceHelper struct { 13 | serviceUri string 14 | token string 15 | } 16 | 17 | // azServiceFromToken for azure the first part of the token __must__ always be the 18 | // identifier of the service e.g. the account name for tableStore or the Vault name for KVSecret or 19 | // AppConfig instance 20 | // take parameter specifies the number of elements to take from the start only 21 | // 22 | // e.g. a value of 2 for take will take first 2 elements from the slices 23 | // 24 | // For AppConfig or KeyVault we ONLY need the AppConfig instance or KeyVault instance name 25 | func azServiceFromToken(token string, formatUri string, take int) azServiceHelper { 26 | // ensure preceding slash is trimmed 27 | stringToken := strings.Split(strings.TrimPrefix(token, "/"), "/") 28 | splitToken := []any{} 29 | // recast []string slice to an []any 30 | for _, st := range stringToken { 31 | splitToken = append(splitToken, st) 32 | } 33 | 34 | uri := fmt.Sprintf(formatUri, splitToken[0:take]...) 35 | return azServiceHelper{serviceUri: uri, token: strings.Join(stringToken[take:], "/")} 36 | } 37 | -------------------------------------------------------------------------------- /internal/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "io" 5 | "log/slog" 6 | ) 7 | 8 | type ILogger interface { 9 | SetLevel(level Level) 10 | Debug(msg string, args ...any) 11 | Info(msg string, args ...any) 12 | Error(msg string, args ...any) 13 | } 14 | 15 | type Opts func(*Logger) 16 | 17 | type Logger struct { 18 | // ILogger 19 | leveler *slog.LevelVar 20 | w *slog.Logger 21 | } 22 | 23 | func New(w io.Writer, opts ...Opts) *Logger { 24 | leveler := &slog.LevelVar{} 25 | leveler.Set(slog.LevelError) 26 | l := &Logger{ 27 | leveler: leveler, 28 | w: slog.New(slog.NewTextHandler(w, &slog.HandlerOptions{Level: leveler})), 29 | } 30 | return l 31 | } 32 | 33 | type Level string 34 | 35 | func (lvl Level) getSlogLevel() slog.Level { 36 | switch lvl { 37 | case "debug": 38 | return slog.LevelDebug 39 | case "error": 40 | return slog.LevelError 41 | case "info": 42 | return slog.LevelInfo 43 | } 44 | return 8 45 | } 46 | 47 | const ( 48 | DebugLvl Level = "debug" 49 | InfoLvl Level = "info" 50 | ErrorLvl Level = "error" 51 | ) 52 | 53 | func (l *Logger) SetLevel(level Level) { 54 | l.leveler.Set(level.getSlogLevel()) 55 | } 56 | 57 | func (l *Logger) Debug(msg string, args ...any) { 58 | l.w.Debug(msg, args...) 59 | } 60 | func (l *Logger) Error(msg string, args ...any) { 61 | l.w.Error(msg, args...) 62 | 63 | } 64 | func (l *Logger) Info(msg string, args ...any) { 65 | l.w.Info(msg, args...) 66 | } 67 | -------------------------------------------------------------------------------- /cmd/configmanager/retrieve.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | type retrieveTokenFlags struct { 10 | tokens []string 11 | path string 12 | } 13 | 14 | func newRetrieveCmd(rootCmd *Root) { 15 | f := &retrieveTokenFlags{} 16 | 17 | retrieveCmd := &cobra.Command{ 18 | Use: "retrieve", 19 | Aliases: []string{"r", "fetch", "get"}, 20 | Short: `Retrieves a value for token(s) specified`, 21 | Long: `Retrieves a value for token(s) specified and optionally writes to a file or to stdout in a bash compliant export KEY=VAL syntax`, 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | cu, outputWriter, err := cmdutilsInit(rootCmd, cmd, f.path) 24 | if err != nil { 25 | return err 26 | } 27 | defer outputWriter.Close() 28 | return cu.GenerateFromCmd(f.tokens) 29 | }, 30 | PreRunE: func(cmd *cobra.Command, args []string) error { 31 | if len(f.tokens) < 1 { 32 | return fmt.Errorf("must include at least 1 token") 33 | } 34 | return nil 35 | }, 36 | } 37 | retrieveCmd.PersistentFlags().StringArrayVarP(&f.tokens, "token", "t", []string{}, "Token pointing to a config/secret variable. This can be specified multiple times.") 38 | _ = retrieveCmd.MarkPersistentFlagRequired("token") 39 | retrieveCmd.PersistentFlags().StringVarP(&f.path, "path", "p", "./app.env", "Path where to write out the replaced a config/secret variables. Special value of stdout can be used to return the output to stdout e.g. -p stdout, unix style output only") 40 | rootCmd.Cmd.AddCommand(retrieveCmd) 41 | } 42 | -------------------------------------------------------------------------------- /cmd/configmanager/configmanager_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "log/slog" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | cmd "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | ) 14 | 15 | type cmdTestInput struct { 16 | args []string 17 | errored bool 18 | exactOutput string 19 | output []string 20 | logLevel slog.Level // 8 for error -4 debug, 0 for info 21 | } 22 | type levelSetter int 23 | 24 | func (l levelSetter) Level() int { 25 | return 8 26 | } 27 | func cmdRunTestHelper(t *testing.T, testInput *cmdTestInput) { 28 | t.Helper() 29 | 30 | leveler := &slog.LevelVar{} 31 | leveler.Set(testInput.logLevel) 32 | 33 | logErr := &bytes.Buffer{} 34 | logger := log.New(logErr) 35 | cmd := cmd.NewRootCmd(logger) 36 | os.Args = append([]string{os.Args[0]}, testInput.args...) 37 | errOut := &bytes.Buffer{} 38 | stdOut := &bytes.Buffer{} 39 | cmd.Cmd.SetArgs(testInput.args) 40 | cmd.Cmd.SetErr(errOut) 41 | cmd.Cmd.SetOut(stdOut) 42 | 43 | if err := cmd.Execute(context.TODO()); err != nil { 44 | if testInput.errored { 45 | return 46 | } 47 | t.Fatalf("\ngot: %v\nwanted \n", err) 48 | } 49 | 50 | if testInput.errored && errOut.Len() < 1 { 51 | t.Errorf("\ngot: nil\nwanted an error to be thrown") 52 | } 53 | if len(testInput.output) > 0 { 54 | for _, v := range testInput.output { 55 | if !strings.Contains(stdOut.String(), v) { 56 | t.Errorf("\ngot: %s\vnot found in: %v", stdOut.String(), v) 57 | } 58 | } 59 | } 60 | if testInput.exactOutput != "" && stdOut.String() != testInput.exactOutput { 61 | t.Errorf("output mismatch\ngot: %s\n\nwanted: %s", stdOut.String(), testInput.exactOutput) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /cmd/configmanager/fromfileinput.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | type fromStrFlags struct { 11 | input string 12 | path string 13 | } 14 | 15 | func newFromStrCmd(rootCmd *Root) { 16 | 17 | f := &fromStrFlags{} 18 | 19 | fromstrCmd := &cobra.Command{ 20 | Use: "string-input", 21 | Aliases: []string{"fromstr", "getfromstr"}, 22 | Short: `Retrieves all found token values in a specified string input`, 23 | Long: `Retrieves all found token values in a specified string input, can be a file, stdin, or string. Writes the output to a specified output`, 24 | RunE: func(cmd *cobra.Command, args []string) error { 25 | inputReader, err := cmdutils.GetReader(cmd, f.input) 26 | if err != nil { 27 | return err 28 | } 29 | cu, outputWriter, err := cmdutilsInit(rootCmd, cmd, f.path) 30 | if err != nil { 31 | return err 32 | } 33 | defer outputWriter.Close() 34 | 35 | return cu.GenerateStrOut(inputReader, f.input == f.path) 36 | }, 37 | PreRunE: func(cmd *cobra.Command, args []string) error { 38 | if len(f.input) < 1 { 39 | return fmt.Errorf("must include input") 40 | } 41 | return nil 42 | }, 43 | } 44 | 45 | fromstrCmd.PersistentFlags().StringVarP(&f.input, "input", "i", "", `Path to file which contents will be read in or the contents of a string 46 | inside a variable to be searched for tokens. 47 | If value is a valid path it will open it if not it will accept the string as an input. 48 | e.g. -i /some/file or -i $"(cat /som/file)", are both valid input values`) 49 | _ = fromstrCmd.MarkPersistentFlagRequired("input") 50 | fromstrCmd.PersistentFlags().StringVarP(&f.path, "path", "p", "./app.env", `Path where to write out the 51 | replaced a config/secret variables. Special value of stdout can be used to return the output to stdout e.g. -p stdout, 52 | unix style output only`) 53 | rootCmd.Cmd.AddCommand(fromstrCmd) 54 | } 55 | -------------------------------------------------------------------------------- /internal/log/log_test.go: -------------------------------------------------------------------------------- 1 | package log_test 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 9 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 10 | ) 11 | 12 | func Test_LogInfo(t *testing.T) { 13 | tests := map[string]struct { 14 | level log.Level 15 | logMethod log.Level 16 | message string 17 | expect string 18 | }{ 19 | "info at debug": { 20 | level: log.DebugLvl, 21 | logMethod: log.InfoLvl, 22 | message: "write me out...", 23 | expect: `level=INFO msg="write me out..."`, 24 | }, 25 | "error at debug": { 26 | level: log.DebugLvl, 27 | logMethod: log.ErrorLvl, 28 | message: "write me out...", 29 | expect: `level=ERROR msg="write me out..."`, 30 | }, 31 | "debug at debug": { 32 | level: log.DebugLvl, 33 | logMethod: log.DebugLvl, 34 | message: "write me out...", 35 | expect: `level=DEBUG msg="write me out..."`, 36 | }, 37 | "debug at info": { 38 | level: log.InfoLvl, 39 | logMethod: log.DebugLvl, 40 | message: "write me out...", 41 | expect: ``, 42 | }, 43 | "info at error": { 44 | level: log.ErrorLvl, 45 | logMethod: log.InfoLvl, 46 | message: "write me out...", 47 | expect: ``, 48 | }, 49 | } 50 | for name, tt := range tests { 51 | t.Run(name, func(t *testing.T) { 52 | // overwrite writer for testing 53 | buf := &bytes.Buffer{} 54 | // overwrite logger for testing without timestamp 55 | logger := log.New(buf) 56 | logger.SetLevel(tt.level) 57 | 58 | switch tt.logMethod { 59 | case "debug": 60 | logger.Debug(tt.message) 61 | case "info": 62 | logger.Info(tt.message) 63 | case "error": 64 | logger.Error(tt.message) 65 | } 66 | got := buf.String() 67 | if !strings.Contains(got, tt.expect) { 68 | t.Errorf(testutils.TestPhrase, got, tt.expect) 69 | } 70 | if len(tt.expect) == 0 && len(got) > 0 { 71 | t.Errorf(testutils.TestPhraseWithContext, "no output expected", got, tt.expect) 72 | } 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /.github/workflows/release_container.yml: -------------------------------------------------------------------------------- 1 | name: Publish Container 2 | 3 | on: 4 | workflow_run: 5 | workflows: ['Lint and Test'] 6 | types: 7 | - completed 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: write 13 | packages: write 14 | 15 | jobs: 16 | set-version-tag: 17 | if: ${{ github.event.workflow_run.head_branch == 'main' }} 18 | runs-on: ubuntu-24.04 19 | outputs: 20 | semVer: ${{ steps.gitversion.outputs.semVer }} 21 | steps: 22 | - uses: actions/checkout@v4 23 | # get version 24 | - uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | - name: Install GitVersion 28 | uses: gittools/actions/gitversion/setup@v3.0 29 | with: 30 | versionSpec: '5.x' 31 | - name: Set SemVer Version 32 | uses: gittools/actions/gitversion/execute@v3.0 33 | id: gitversion 34 | 35 | build-and-push: 36 | runs-on: ubuntu-latest 37 | needs: set-version-tag 38 | env: 39 | SEMVER: ${{ needs.set-version-tag.outputs.semVer }} 40 | steps: 41 | - name: Check out repository 42 | uses: actions/checkout@v3 43 | 44 | - name: Log in to GitHub Container Registry 45 | uses: docker/login-action@v2 46 | with: 47 | registry: ghcr.io 48 | # GITHUB_TOKEN is automatically provided in GitHub Actions 49 | username: ${{ github.actor }} 50 | password: ${{ secrets.GITHUB_TOKEN }} 51 | 52 | - name: Set up QEMU (for multi-arch builds, optional) 53 | uses: docker/setup-qemu-action@v3 54 | 55 | - name: Set up Docker Buildx 56 | uses: docker/setup-buildx-action@v3 57 | 58 | - name: Build and push Docker image 59 | uses: docker/build-push-action@v6 60 | with: 61 | context: . 62 | push: true 63 | build-args: Version=${{ needs.set-version-tag.outputs.semVer }},Revision=${{ github.sha }} 64 | tags: | 65 | ghcr.io/ensono/eirctl:${{ needs.set-version-tag.outputs.semVer }} 66 | platforms: linux/amd64,linux/arm64 # adjust as needed 67 | -------------------------------------------------------------------------------- /internal/store/paramstore.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 7 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 8 | "github.com/aws/aws-sdk-go-v2/aws" 9 | awsConf "github.com/aws/aws-sdk-go-v2/config" 10 | "github.com/aws/aws-sdk-go-v2/service/ssm" 11 | ) 12 | 13 | type paramStoreApi interface { 14 | GetParameter(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) 15 | } 16 | 17 | type ParamStore struct { 18 | svc paramStoreApi 19 | ctx context.Context 20 | logger log.ILogger 21 | config *ParamStrConfig 22 | token *config.ParsedTokenConfig 23 | } 24 | 25 | type ParamStrConfig struct { 26 | // reserved for potential future use 27 | } 28 | 29 | func NewParamStore(ctx context.Context, logger log.ILogger) (*ParamStore, error) { 30 | cfg, err := awsConf.LoadDefaultConfig(ctx) 31 | if err != nil { 32 | logger.Error("unable to load SDK config, %v\n%w", err, ErrClientInitialization) 33 | return nil, err 34 | } 35 | c := ssm.NewFromConfig(cfg) 36 | 37 | return &ParamStore{ 38 | svc: c, 39 | logger: logger, 40 | ctx: ctx, 41 | }, nil 42 | } 43 | 44 | func (imp *ParamStore) SetToken(token *config.ParsedTokenConfig) { 45 | storeConf := &ParamStrConfig{} 46 | _ = token.ParseMetadata(storeConf) 47 | imp.token = token 48 | imp.config = storeConf 49 | } 50 | 51 | func (imp *ParamStore) Token() (string, error) { 52 | imp.logger.Info("%s", "Concrete implementation ParameterStore") 53 | imp.logger.Info("ParamStore Token: %s", imp.token.String()) 54 | 55 | input := &ssm.GetParameterInput{ 56 | Name: aws.String(imp.token.StoreToken()), 57 | WithDecryption: aws.Bool(true), 58 | } 59 | ctx, cancel := context.WithCancel(imp.ctx) 60 | defer cancel() 61 | 62 | result, err := imp.svc.GetParameter(ctx, input) 63 | if err != nil { 64 | imp.logger.Error(implementationNetworkErr, config.ParamStorePrefix, err, imp.token.StoreToken()) 65 | return "", err 66 | } 67 | 68 | if result.Parameter.Value != nil { 69 | return *result.Parameter.Value, nil 70 | } 71 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 72 | return "", nil 73 | } 74 | -------------------------------------------------------------------------------- /docs/adding-provider.md: -------------------------------------------------------------------------------- 1 | # Adding a provider 2 | 3 | Add Token Prefix 4 | 5 | ```go 6 | const ( 7 | // AWS SecretsManager prefix 8 | SecretMgrPrefix ImplementationPrefix = "AWSSECRETS" 9 | // AWS Parameter Store prefix 10 | ParamStorePrefix ImplementationPrefix = "AWSPARAMSTR" 11 | // Azure Key Vault Secrets prefix 12 | AzKeyVaultSecretsPrefix ImplementationPrefix = "AZKVSECRET" 13 | // Hashicorp Vault prefix 14 | HashicorpVaultPrefix ImplementationPrefix = "VAULT" 15 | // GcpSecrets 16 | GcpSecretsPrefix ImplementationPrefix = "GCPSECRETS" 17 | ) 18 | ``` 19 | 20 | ```go 21 | var ( 22 | // default varPrefix used by the replacer function 23 | // any token must beging with one of these else 24 | // it will be skipped as not a replaceable token 25 | VarPrefix = map[ImplementationPrefix]bool{SecretMgrPrefix: true, ParamStorePrefix: true, AzKeyVaultSecretsPrefix: true, GcpSecretsPrefix: true, HashicorpVaultPrefix: true} // <-- ADD here 26 | ) 27 | ``` 28 | 29 | ensure your implementation satisfy the `genVarsStrategy` interface 30 | 31 | ```go 32 | type genVarsStrategy interface { 33 | tokenVal(rs *retrieveStrategy) (s string, e error) 34 | setTokenVal(s string) 35 | } 36 | ``` 37 | 38 | Even if the native type is K/V return a marshalled version of the JSON as the rest of the flow will decide how to present it back to the final consumer. 39 | 40 | Custom properties inside the GetValue request, you could specify your own Config struct for the provider, e.g. HashiVault implementation 41 | 42 | ```go 43 | // VaultConfig holds the parseable metadata struct 44 | type VaultConfig struct { 45 | Version string `json:"version"` 46 | Role string `json:"iam_role"` 47 | } 48 | ``` 49 | 50 | You could then use it on the backingStore object 51 | 52 | ```go 53 | type VaultStore struct { 54 | svc hashiVaultApi 55 | ctx context.Context 56 | config *VaultConfig 57 | token string 58 | } 59 | ``` 60 | 61 | On initialize of the instance or in the setTokenVal method (see GCPSecrets or AWSSecrets/ParamStore examples). 62 | 63 | ```go 64 | storeConf := &VaultConfig{} 65 | initialToken := ParseMetadata(token, storeConf) 66 | imp := &VaultStore{ 67 | ctx: ctx, 68 | config: storeConf, 69 | } 70 | ``` 71 | 72 | Where the initialToken is the original Token without the metadata in brackets and the `storeConf` pointer will have been filled with any of the parsed metadata and used in the actual provider implementation, see any of the providers for a sample implementation. 73 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_run: 5 | workflows: ['CI'] 6 | types: 7 | - completed 8 | branches: 9 | - master 10 | - main 11 | 12 | permissions: 13 | contents: write 14 | 15 | jobs: 16 | set-version: 17 | runs-on: ubuntu-latest 18 | if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }} 19 | container: 20 | image: mcr.microsoft.com/dotnet/sdk:6.0 21 | outputs: 22 | semVer: ${{ steps.gitversion.outputs.semVer }} 23 | steps: 24 | - uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | - name: install deps 28 | run: | 29 | apt-get update && apt-get install -y jq git 30 | git config --global --add safe.directory "$GITHUB_WORKSPACE" 31 | git config user.email ${{ github.actor }}-ci@gha.org 32 | git config user.name ${{ github.actor }} 33 | - name: Install GitVersion 34 | uses: gittools/actions/gitversion/setup@v3.0.0 35 | with: 36 | versionSpec: '5.x' 37 | - name: Set SemVer Version 38 | uses: gittools/actions/gitversion/execute@v3.0.0 39 | id: gitversion 40 | 41 | release: 42 | runs-on: ubuntu-latest 43 | needs: set-version 44 | env: 45 | SEMVER: ${{ needs.set-version.outputs.semVer }} 46 | steps: 47 | - uses: actions/checkout@v4 48 | with: 49 | fetch-depth: 1 50 | 51 | - name: Install Eirctl 52 | uses: ensono/actions/eirctl-setup@v0.3.1 53 | with: 54 | version: 0.7.6 55 | isPrerelease: false 56 | 57 | - name: build binary 58 | run: | 59 | VERSION=${SEMVER} REVISION=$GITHUB_SHA eirctl run pipeline build:bin 60 | 61 | - name: Release binary 62 | uses: softprops/action-gh-release@v2 63 | with: 64 | tag_name: v${{ needs.set-version.outputs.semVer }} 65 | # TODO: add additional info to the release 66 | generate_release_notes: true 67 | token: ${{ secrets.GITHUB_TOKEN }} 68 | files: ./dist/* 69 | prerelease: false 70 | 71 | - name: release library 72 | run: | 73 | git config --global --add safe.directory "$GITHUB_WORKSPACE" 74 | git config user.email ${{ github.actor }}-ci@gha.org 75 | git config user.name ${{ github.actor }} 76 | VERSION=${SEMVER} REVISION=$GITHUB_SHA eirctl run tag 77 | -------------------------------------------------------------------------------- /internal/store/gcpsecrets.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | gcpsecrets "cloud.google.com/go/secretmanager/apiv1" 8 | gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" 9 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 10 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 11 | "github.com/googleapis/gax-go/v2" 12 | ) 13 | 14 | type gcpSecretsApi interface { 15 | AccessSecretVersion(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) 16 | } 17 | 18 | type GcpSecrets struct { 19 | svc gcpSecretsApi 20 | logger log.ILogger 21 | ctx context.Context 22 | config *GcpSecretsConfig 23 | close func() error 24 | token *config.ParsedTokenConfig 25 | } 26 | 27 | type GcpSecretsConfig struct { 28 | Version string `json:"version"` 29 | } 30 | 31 | func NewGcpSecrets(ctx context.Context, logger log.ILogger) (*GcpSecrets, error) { 32 | 33 | c, err := gcpsecrets.NewClient(ctx) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &GcpSecrets{ 38 | svc: c, 39 | logger: logger, 40 | ctx: ctx, 41 | close: c.Close, 42 | }, nil 43 | } 44 | 45 | func (imp *GcpSecrets) SetToken(token *config.ParsedTokenConfig) { 46 | storeConf := &GcpSecretsConfig{} 47 | _ = token.ParseMetadata(storeConf) 48 | imp.token = token 49 | imp.config = storeConf 50 | } 51 | 52 | func (imp *GcpSecrets) Token() (string, error) { 53 | // Close client currently as new one would be created per iteration 54 | defer func() { 55 | _ = imp.close() 56 | }() 57 | 58 | imp.logger.Info("Concrete implementation GcpSecrets") 59 | imp.logger.Info("GcpSecrets Token: %s", imp.token.String()) 60 | 61 | version := "latest" 62 | if imp.config.Version != "" { 63 | version = imp.config.Version 64 | } 65 | 66 | imp.logger.Info("Getting Secret: %s @version: %s", imp.token, version) 67 | 68 | input := &gcpsecretspb.AccessSecretVersionRequest{ 69 | Name: fmt.Sprintf("%s/versions/%s", imp.token.StoreToken(), version), 70 | } 71 | 72 | ctx, cancel := context.WithCancel(imp.ctx) 73 | defer cancel() 74 | 75 | result, err := imp.svc.AccessSecretVersion(ctx, input) 76 | 77 | if err != nil { 78 | imp.logger.Error(implementationNetworkErr, imp.token.Prefix(), err, imp.token.String()) 79 | return "", err 80 | } 81 | if result.Payload != nil { 82 | return string(result.Payload.Data), nil 83 | } 84 | 85 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 86 | return "", nil 87 | } 88 | -------------------------------------------------------------------------------- /internal/cmdutils/postprocessor_test.go: -------------------------------------------------------------------------------- 1 | package cmdutils_test 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" 9 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 10 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 11 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 12 | ) 13 | 14 | func postprocessorHelper(t *testing.T) { 15 | t.Helper() 16 | 17 | } 18 | func Test_ConvertToExportVars(t *testing.T) { 19 | tests := map[string]struct { 20 | rawMap generator.ParsedMap 21 | expectStr string 22 | expectLength int 23 | }{ 24 | "number included": {generator.ParsedMap{"foo": "BAR", "num": 123}, `export FOO='BAR'`, 2}, 25 | "strings only": {generator.ParsedMap{"foo": "BAR", "num": "a123"}, `export FOO='BAR'`, 2}, 26 | "numbers only": {generator.ParsedMap{"foo": 123, "num": 456}, `export FOO=123`, 2}, 27 | "map inside response": {generator.ParsedMap{"map": `{"foo":"bar","baz":"qux"}`, "num": 123}, `export FOO='bar'`, 3}, 28 | } 29 | 30 | for name, tt := range tests { 31 | t.Run(name, func(t *testing.T) { 32 | pp := cmdutils.PostProcessor{ProcessedMap: tt.rawMap, Config: config.NewConfig()} 33 | got := pp.ConvertToExportVar() 34 | 35 | if got == nil { 36 | t.Errorf(testutils.TestPhrase, got, "not nil") 37 | } 38 | if len(got) != tt.expectLength { 39 | t.Errorf(testutils.TestPhrase, len(got), tt.expectLength) 40 | } 41 | st := strings.Join(got, "\n") 42 | if !strings.Contains(st, tt.expectStr) { 43 | t.Errorf(testutils.TestPhrase, st, tt.expectStr) 44 | } 45 | 46 | // check FlushToFile 47 | tw := bytes.NewBuffer([]byte{}) 48 | pp.FlushOutToFile(tw) 49 | readBuffer := tw.Bytes() 50 | if len(readBuffer) == 0 { 51 | t.Errorf(testutils.TestPhraseWithContext, "buffer should be filled", string(readBuffer), tt.expectStr) 52 | } 53 | 54 | }) 55 | } 56 | } 57 | 58 | func Test_StrToWriter(t *testing.T) { 59 | ttests := map[string]struct { 60 | input string 61 | }{ 62 | "matches": {`export FOO=BAR`}, 63 | "multiline": {`export FOO=BAR\nBUX=GED`}, 64 | } 65 | for name, tt := range ttests { 66 | t.Run(name, func(t *testing.T) { 67 | want := tt.input 68 | tw := bytes.NewBuffer([]byte{}) 69 | pp := cmdutils.PostProcessor{} 70 | pp.StrToFile(tw, tt.input) 71 | readBuffer := tw.Bytes() 72 | if string(readBuffer) != want { 73 | t.Errorf(testutils.TestPhraseWithContext, "incorrectly written buffer stream", string(readBuffer), want) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /internal/store/secretsmanager.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 7 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 8 | "github.com/aws/aws-sdk-go-v2/aws" 9 | awsconf "github.com/aws/aws-sdk-go-v2/config" 10 | "github.com/aws/aws-sdk-go-v2/service/secretsmanager" 11 | ) 12 | 13 | type secretsMgrApi interface { 14 | GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) 15 | } 16 | 17 | type SecretsMgr struct { 18 | svc secretsMgrApi 19 | ctx context.Context 20 | logger log.ILogger 21 | config *SecretsMgrConfig 22 | token *config.ParsedTokenConfig 23 | } 24 | 25 | type SecretsMgrConfig struct { 26 | Version string `json:"version"` 27 | } 28 | 29 | func NewSecretsMgr(ctx context.Context, logger log.ILogger) (*SecretsMgr, error) { 30 | cfg, err := awsconf.LoadDefaultConfig(ctx) 31 | if err != nil { 32 | logger.Error("unable to load SDK config, %v\n%w", err, ErrClientInitialization) 33 | return nil, err 34 | } 35 | c := secretsmanager.NewFromConfig(cfg) 36 | 37 | return &SecretsMgr{ 38 | svc: c, 39 | logger: logger, 40 | ctx: ctx, 41 | }, nil 42 | 43 | } 44 | 45 | func (imp *SecretsMgr) SetToken(token *config.ParsedTokenConfig) { 46 | storeConf := &SecretsMgrConfig{} 47 | if err := token.ParseMetadata(storeConf); err != nil { 48 | imp.logger.Error("parse token error %v", err) 49 | } 50 | imp.token = token 51 | imp.config = storeConf 52 | } 53 | 54 | func (imp *SecretsMgr) Token() (string, error) { 55 | imp.logger.Info("Concrete implementation SecretsManager") 56 | imp.logger.Debug("SecretsManager Token: %s", imp.token.String()) 57 | 58 | version := "AWSCURRENT" 59 | if imp.config.Version != "" { 60 | version = imp.config.Version 61 | } 62 | 63 | imp.logger.Info("Getting Secret: %s @version: %s", imp.token, version) 64 | 65 | input := &secretsmanager.GetSecretValueInput{ 66 | SecretId: aws.String(imp.token.StoreToken()), 67 | VersionStage: aws.String(version), 68 | } 69 | 70 | ctx, cancel := context.WithCancel(imp.ctx) 71 | defer cancel() 72 | 73 | result, err := imp.svc.GetSecretValue(ctx, input) 74 | if err != nil { 75 | imp.logger.Error(implementationNetworkErr, imp.token.Prefix(), err, imp.token.String()) 76 | return "", err 77 | } 78 | 79 | if result.SecretString != nil { 80 | return *result.SecretString, nil 81 | } 82 | 83 | if len(result.SecretBinary) > 0 { 84 | return string(result.SecretBinary), nil 85 | } 86 | 87 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 88 | return "", nil 89 | } 90 | -------------------------------------------------------------------------------- /internal/store/azkeyvault.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Azure KeyVault implementation 3 | **/ 4 | package store 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 10 | "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | ) 14 | 15 | type kvApi interface { 16 | GetSecret(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) 17 | } 18 | 19 | type KvScrtStore struct { 20 | svc kvApi 21 | ctx context.Context 22 | logger log.ILogger 23 | token *config.ParsedTokenConfig 24 | config *AzKvConfig 25 | strippedToken string 26 | } 27 | 28 | // AzKvConfig takes any metadata from the token 29 | // Version is the only 30 | type AzKvConfig struct { 31 | Version string `json:"version"` 32 | } 33 | 34 | // NewKvScrtStore returns a KvScrtStore 35 | // requires `AZURE_SUBSCRIPTION_ID` environment variable to be present to successfully work 36 | func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger log.ILogger) (*KvScrtStore, error) { 37 | 38 | storeConf := &AzKvConfig{} 39 | _ = token.ParseMetadata(storeConf) 40 | 41 | backingStore := &KvScrtStore{ 42 | ctx: ctx, 43 | logger: logger, 44 | config: storeConf, 45 | token: token, 46 | } 47 | 48 | srvInit := azServiceFromToken(token.StoreToken(), "https://%s.vault.azure.net", 1) 49 | backingStore.strippedToken = srvInit.token 50 | 51 | cred, err := azidentity.NewDefaultAzureCredential(nil) 52 | if err != nil { 53 | logger.Error("failed to get credentials: %v", err) 54 | return nil, err 55 | } 56 | 57 | c, err := azsecrets.NewClient(srvInit.serviceUri, cred, nil) 58 | if err != nil { 59 | logger.Error("%v\n%w", err, ErrClientInitialization) 60 | return nil, err 61 | } 62 | 63 | backingStore.svc = c 64 | return backingStore, nil 65 | 66 | } 67 | 68 | // setToken already happens in AzureKVClient in the constructor 69 | func (implmt *KvScrtStore) SetToken(token *config.ParsedTokenConfig) {} 70 | 71 | func (imp *KvScrtStore) Token() (string, error) { 72 | imp.logger.Info("Concrete implementation AzKeyVault Secret") 73 | imp.logger.Info("AzKeyVault Token: %s", imp.token.String()) 74 | 75 | ctx, cancel := context.WithCancel(imp.ctx) 76 | defer cancel() 77 | 78 | // secretVersion as "" => latest 79 | // imp.config.Version will default `""` if not specified 80 | s, err := imp.svc.GetSecret(ctx, imp.strippedToken, imp.config.Version, nil) 81 | if err != nil { 82 | imp.logger.Error(implementationNetworkErr, imp.token.Prefix(), err, imp.token.String()) 83 | return "", err 84 | } 85 | if s.Value != nil { 86 | return *s.Value, nil 87 | } 88 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 89 | return "", nil 90 | } 91 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [master, main] 6 | pull_request: 7 | branches: [master, main] 8 | 9 | permissions: 10 | contents: write 11 | statuses: write 12 | checks: write 13 | pull-requests: write 14 | 15 | jobs: 16 | set-version: 17 | runs-on: ubuntu-latest 18 | container: 19 | image: mcr.microsoft.com/dotnet/sdk:6.0 20 | outputs: 21 | semVer: ${{ steps.gitversion.outputs.semVer }} 22 | steps: 23 | - uses: actions/checkout@v4 24 | with: 25 | fetch-depth: 0 26 | - name: install deps 27 | run: | 28 | apt-get update && apt-get install -y jq git 29 | git config --global --add safe.directory "$GITHUB_WORKSPACE" 30 | git config user.email ${{ github.actor }}-ci@gha.org 31 | git config user.name ${{ github.actor }} 32 | - name: Install GitVersion 33 | uses: gittools/actions/gitversion/setup@v3.0.0 34 | with: 35 | versionSpec: "5.x" 36 | - name: Set SemVer Version 37 | uses: gittools/actions/gitversion/execute@v3.0.0 38 | id: gitversion 39 | 40 | - name: echo VERSIONS 41 | run: | 42 | echo "REVISION -> $GITHUB_SHA" 43 | echo "VERSION -> $GITVERSION_SEMVER" 44 | 45 | test: 46 | runs-on: ubuntu-latest 47 | needs: set-version 48 | env: 49 | SEMVER: ${{ needs.set-version.outputs.semVer }} 50 | GIT_TAG: ${{ needs.set-version.outputs.semVer }} 51 | GOVCS: false 52 | steps: 53 | - uses: actions/checkout@v4 54 | with: 55 | fetch-depth: 1 56 | 57 | - name: Install Eirctl 58 | uses: ensono/actions/eirctl-setup@v0.3.1 59 | with: 60 | version: 0.9.3 61 | isPrerelease: false 62 | 63 | - name: Run Lint 64 | run: | 65 | eirctl run pipeline lints 66 | - name: Run Tests 67 | run: | 68 | eirctl run pipeline gha:unit:test 69 | 70 | - name: Publish Junit style Test Report 71 | uses: mikepenz/action-junit-report@v4 72 | if: always() # always run even if the previous step fails 73 | with: 74 | report_paths: "**/.coverage/report-junit.xml" 75 | 76 | - name: Analyze with SonarCloud 77 | uses: SonarSource/sonarqube-scan-action@v6 78 | env: 79 | # Needed to get PR information 80 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 81 | # Generate a token on Sonarcloud.io, add it to the secrets of this repo with the name SONAR_TOKEN (Settings > Secrets > Actions > add new repository secret) 82 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 83 | with: 84 | projectBaseDir: . 85 | args: > 86 | -Dsonar.projectVersion=${{ needs.set-version.outputs.semVer }} 87 | -Dsonar.working.directory=.scannerwork 88 | -Dsonar.scm.provider=git 89 | -------------------------------------------------------------------------------- /docs/commands.md: -------------------------------------------------------------------------------- 1 | # Commands 2 | 3 | 4 | ## Usage 5 | 6 | ```bash 7 | configmanager CLI for retrieving config or secret variables. 8 | Using a specific tokens as an array item 9 | 10 | Usage: 11 | configmanager [command] 12 | 13 | Available Commands: 14 | completion Generate the autocompletion script for the specified shell 15 | help Help about any command 16 | insert Not yet implemented 17 | retrieve Retrieves a value for token(s) specified 18 | string-input Retrieves all found token values in a specified string input 19 | version Get version number configmanager 20 | 21 | Flags: 22 | -h, --help help for configmanager 23 | -k, --key-separator string Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1 (default "|") 24 | -s, --token-separator string Separator to use to mark concrete store and the key within it (default "#") 25 | -v, --verbose Verbosity level 26 | ``` 27 | 28 | ### retrieve 29 | 30 | Useful for retrieving a series of tokens in CI or before app start 31 | 32 | ```bash 33 | configmanager retrieve --token AWSSECRETS#/appxyz/service1-password --token AWSPARAMSTR#/appxyz/service2-password 34 | source app.env 35 | ``` 36 | 37 | This will have written to a defaul out path `app.env` in current directory the below contents 38 | 39 | ```bash 40 | export SERVICE1_PASSWORD='somepass!$@sdhf' 41 | export SERVICE2_PASSWORD='somepa22$!$' 42 | ``` 43 | 44 | Once sourced you could delete the file, however the environment variables will persist in the process info `/proc/someprocess` 45 | 46 | ```bash 47 | rm -f app.env 48 | ./startapp 49 | ``` 50 | 51 | By default the output path is `app.env` relative to the exec binary. 52 | 53 | This can be overridden by passing in the `--path` param. 54 | 55 | ```bash 56 | configmanager retrieve --token AWSSECRETS#/appxyz/service1-password --token AWSPARAMSTR#/appxyz/service12-settings --path /some/path/app.env 57 | source /some/path/app.env 58 | ./startapp # psuedo script to start an application 59 | ``` 60 | 61 | Alternatively you can set the path as stdout which will reduce the need to save and source the env from file. 62 | 63 | >!Warning! about eval - if you are retrieving secrets from sources you don't control the input of - best to stick wtih the file approach and then delete the file. 64 | 65 | ```bash 66 | eval "$(configmanager r -t AWSSECRETS#/appxyz/service1-password -t AWSPARAMSTR#/appxyz/service12-settings -p stdout)" && ./.ignore-out.sh 67 | ``` 68 | 69 | ### string-input 70 | 71 | Replaces all the occurences of tokens inside strings and writes them back out to a file provided. 72 | 73 | This method can be used with entire application property files such as `application.yml` or `application.properties` for springboot apps or netcore app config in which ever format. 74 | 75 | The `fromstr` (alias for `string-input`) respects all indentations so can be used on contents of a file of any type 76 | 77 | -------------------------------------------------------------------------------- /internal/cmdutils/postprocessor.go: -------------------------------------------------------------------------------- 1 | package cmdutils 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 9 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 10 | ) 11 | 12 | // PostProcessor 13 | // processes the rawMap and outputs the result 14 | // depending on cmdline options 15 | type PostProcessor struct { 16 | ProcessedMap generator.ParsedMap 17 | Config *config.GenVarsConfig 18 | outString []string 19 | } 20 | 21 | // ConvertToExportVar assigns the k/v out 22 | // as unix style export key=val pairs separated by `\n` 23 | func (p *PostProcessor) ConvertToExportVar() []string { 24 | for k, v := range p.ProcessedMap { 25 | rawKeyToken := strings.Split(k, "/") // assumes a path like token was used 26 | topLevelKey := rawKeyToken[len(rawKeyToken)-1] 27 | trm := generator.ParsedMap{} 28 | if parsedOk := generator.IsParsed(v, trm); parsedOk { 29 | // if is a map 30 | // try look up on key if separator defined 31 | normMap := p.envVarNormalize(trm) 32 | p.exportVars(normMap) 33 | continue 34 | } 35 | p.exportVars(generator.ParsedMap{topLevelKey: v}) 36 | } 37 | return p.outString 38 | } 39 | 40 | // envVarNormalize 41 | func (p *PostProcessor) envVarNormalize(pmap generator.ParsedMap) generator.ParsedMap { 42 | normalizedMap := make(generator.ParsedMap) 43 | for k, v := range pmap { 44 | normalizedMap[p.normalizeKey(k)] = v 45 | } 46 | return normalizedMap 47 | } 48 | 49 | func (p *PostProcessor) exportVars(exportMap generator.ParsedMap) { 50 | 51 | for k, v := range exportMap { 52 | // NOTE: \n line ending is not totally cross platform 53 | t := fmt.Sprintf("%T", v) 54 | switch t { 55 | case "string": 56 | p.outString = append(p.outString, fmt.Sprintf("export %s='%s'", p.normalizeKey(k), v)) 57 | default: 58 | p.outString = append(p.outString, fmt.Sprintf("export %s=%v", p.normalizeKey(k), v)) 59 | } 60 | } 61 | } 62 | 63 | // normalizeKeys returns env var compatible key 64 | func (p *PostProcessor) normalizeKey(k string) string { 65 | // the order of replacer pairs matters less 66 | // as the Replace builds a node tree without overlapping matches 67 | replacer := strings.NewReplacer([]string{" ", "", "@", "", "!", "", "-", "_", p.Config.KeySeparator(), "__"}...) 68 | return strings.ToUpper(replacer.Replace(k)) 69 | } 70 | 71 | // FlushOutToFile saves contents to file provided 72 | // in the config input into the generator 73 | // default location is ./app.env 74 | // 75 | // can also be to stdout or another file location 76 | func (p *PostProcessor) FlushOutToFile(w io.Writer) error { 77 | return p.flushToFile(w, listToString(p.outString)) 78 | } 79 | 80 | // StrToFile writes a provided string to the writer 81 | func (p *PostProcessor) StrToFile(w io.Writer, str string) error { 82 | return p.flushToFile(w, str) 83 | } 84 | 85 | func (p *PostProcessor) flushToFile(f io.Writer, str string) error { 86 | if _, err := f.Write([]byte(str)); err != nil { 87 | return err 88 | } 89 | return nil 90 | } 91 | 92 | func listToString(strList []string) string { 93 | return strings.Join(strList, "\n") 94 | } 95 | -------------------------------------------------------------------------------- /cmd/configmanager/configmanager.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/DevLabFoundry/configmanager/v2" 9 | "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" 10 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 12 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | var ( 17 | Version string = "0.0.1" 18 | Revision string = "1111aaaa" 19 | ) 20 | 21 | type rootCmdFlags struct { 22 | verbose bool 23 | tokenSeparator string 24 | keySeparator string 25 | enableEnvSubst bool 26 | } 27 | 28 | type Root struct { 29 | Cmd *cobra.Command 30 | logger log.ILogger 31 | rootFlags *rootCmdFlags 32 | } 33 | 34 | func NewRootCmd(logger log.ILogger) *Root { //channelOut, channelErr io.Writer 35 | rc := &Root{ 36 | Cmd: &cobra.Command{ 37 | Use: config.SELF_NAME, 38 | Short: fmt.Sprintf("%s CLI for retrieving and inserting config or secret variables", config.SELF_NAME), 39 | Long: fmt.Sprintf(`%s CLI for retrieving config or secret variables. 40 | Using a specific tokens as an array item`, config.SELF_NAME), 41 | SilenceUsage: true, 42 | Version: fmt.Sprintf("%s-%s", Version, Revision), 43 | }, 44 | logger: logger, 45 | rootFlags: &rootCmdFlags{}, 46 | } 47 | 48 | rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.verbose, "verbose", "v", false, "Verbosity level") 49 | rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "#", "Separator to use to mark concrete store and the key within it") 50 | rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1") 51 | rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.enableEnvSubst, "enable-envsubst", "e", false, "Enable envsubst on input. This will fail on any unset or empty variables") 52 | addSubCmds(rc) 53 | return rc 54 | } 55 | 56 | // addSubCmds assigns the subcommands to the parent/root command 57 | func addSubCmds(rootCmd *Root) { 58 | newFromStrCmd(rootCmd) 59 | newRetrieveCmd(rootCmd) 60 | newInsertCmd(rootCmd) 61 | } 62 | 63 | func (rc *Root) Execute(ctx context.Context) error { 64 | return rc.Cmd.ExecuteContext(ctx) 65 | } 66 | 67 | func cmdutilsInit(rootCmd *Root, cmd *cobra.Command, path string) (*cmdutils.CmdUtils, io.WriteCloser, error) { 68 | 69 | outputWriter, err := cmdutils.GetWriter(path) 70 | if err != nil { 71 | return nil, nil, err 72 | } 73 | 74 | cm := configmanager.New(cmd.Context()) 75 | cm.Config.WithTokenSeparator(rootCmd.rootFlags.tokenSeparator).WithOutputPath(path).WithKeySeparator(rootCmd.rootFlags.keySeparator).WithEnvSubst(rootCmd.rootFlags.enableEnvSubst) 76 | gnrtr := generator.NewGenerator(cmd.Context(), func(gv *generator.GenVars) { 77 | if rootCmd.rootFlags.verbose { 78 | rootCmd.logger.SetLevel(log.DebugLvl) 79 | } 80 | gv.Logger = rootCmd.logger 81 | }).WithConfig(cm.Config) 82 | cm.WithGenerator(gnrtr) 83 | return cmdutils.New(cm, rootCmd.logger, outputWriter), outputWriter, nil 84 | } 85 | -------------------------------------------------------------------------------- /internal/store/azappconf.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Azure App Config implementation 3 | **/ 4 | package store 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "time" 10 | 11 | "github.com/Azure/azure-sdk-for-go/sdk/azcore" 12 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 13 | "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 15 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 16 | ) 17 | 18 | // appConfApi 19 | // uses this package https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig 20 | type appConfApi interface { 21 | GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) 22 | } 23 | 24 | type AzAppConf struct { 25 | svc appConfApi 26 | ctx context.Context 27 | config *AzAppConfConfig 28 | token *config.ParsedTokenConfig 29 | strippedToken string 30 | logger log.ILogger 31 | } 32 | 33 | // AzAppConfConfig is the azure conf service specific config 34 | // and it is parsed from the token metadata 35 | type AzAppConfConfig struct { 36 | Label string `json:"label"` 37 | Etag *azcore.ETag `json:"etag"` 38 | AcceptDateTime *time.Time `json:"acceptedDateTime"` 39 | } 40 | 41 | // NewAzAppConf 42 | func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger log.ILogger) (*AzAppConf, error) { 43 | storeConf := &AzAppConfConfig{} 44 | if err := token.ParseMetadata(storeConf); err != nil { 45 | return nil, err 46 | } 47 | 48 | backingStore := &AzAppConf{ 49 | ctx: ctx, 50 | config: storeConf, 51 | token: token, 52 | logger: logger, 53 | } 54 | srvInit := azServiceFromToken(token.StoreToken(), "https://%s.azconfig.io", 1) 55 | backingStore.strippedToken = srvInit.token 56 | 57 | cred, err := azidentity.NewDefaultAzureCredential(nil) 58 | if err != nil { 59 | logger.Error("failed to get credentials: %v", err) 60 | return nil, err 61 | } 62 | 63 | c, err := azappconfig.NewClient(srvInit.serviceUri, cred, nil) 64 | if err != nil { 65 | logger.Error("failed to init the client: %v", err) 66 | return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) 67 | } 68 | 69 | backingStore.svc = c 70 | return backingStore, nil 71 | 72 | } 73 | 74 | // setTokenVal sets the token 75 | func (implmt *AzAppConf) SetToken(token *config.ParsedTokenConfig) {} 76 | 77 | // tokenVal in AZ App Config 78 | // label can be specified 79 | // From this point then normal rules of configmanager apply, 80 | // including keySeperator and lookup. 81 | func (imp *AzAppConf) Token() (string, error) { 82 | imp.logger.Info("Concrete implementation AzAppConf") 83 | imp.logger.Info("AzAppConf Token: %s", imp.token.String()) 84 | 85 | ctx, cancel := context.WithCancel(imp.ctx) 86 | defer cancel() 87 | opts := &azappconfig.GetSettingOptions{} 88 | 89 | // assign any metadatas from the token 90 | if imp.config.Label != "" { 91 | opts.Label = &imp.config.Label 92 | } 93 | 94 | if imp.config.Etag != nil { 95 | opts.OnlyIfChanged = imp.config.Etag 96 | } 97 | 98 | s, err := imp.svc.GetSetting(ctx, imp.strippedToken, opts) 99 | if err != nil { 100 | imp.logger.Error(implementationNetworkErr, config.AzAppConfigPrefix, err, imp.strippedToken) 101 | return "", fmt.Errorf("token: %s, error: %v. %w", imp.strippedToken, err, ErrRetrieveFailed) 102 | } 103 | if s.Value != nil { 104 | return *s.Value, nil 105 | } 106 | imp.logger.Error("token: %v, %w", imp.token.String(), ErrEmptyResponse) 107 | return "", nil 108 | } 109 | -------------------------------------------------------------------------------- /eirctl.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/Ensono/eirctl/refs/heads/main/schemas/schema_v1.json 2 | 3 | output: prefixed 4 | debug: false 5 | 6 | import: 7 | - https://raw.githubusercontent.com/Ensono/eirctl/refs/tags/0.9.3/shared/build/go/eirctl.yaml 8 | 9 | contexts: 10 | bash: 11 | container: 12 | name: mirror.gcr.io/bash:5.0.18-alpine3.22 13 | 14 | pipelines: 15 | gha:unit:test: 16 | - pipeline: test:unit 17 | env: 18 | ROOT_PKG_NAME: github.com/DevLabFoundry 19 | - task: sonar:coverage:prep 20 | depends_on: test:unit 21 | 22 | show_coverage: 23 | - pipeline: test:unit 24 | - task: show:coverage 25 | depends_on: test:unit 26 | 27 | build:bin: 28 | - task: clean 29 | - task: go:build:binary 30 | depends_on: clean 31 | 32 | tasks: 33 | show:coverage: 34 | description: Opens the current coverage viewer for the the configmanager utility. 35 | command: go tool cover -html=.coverage/out 36 | 37 | show_docs: 38 | description: | 39 | Opens a webview with godoc running 40 | Already filters the packages to this one and enables 41 | internal/private package documentation 42 | # go install golang.org/x/tools/cmd/godoc@latest 43 | command: | 44 | open http://localhost:6060/pkg/github.com/DevLabFoundry/configmanager/v2/?m=all 45 | godoc -notes "BUG|TODO" -play -http=:6060 46 | 47 | go:build:binary: 48 | context: go1x 49 | description: | 50 | Generates binaries in a dist folder 51 | Generates all the binaries for the configmanager utility. 52 | command: 53 | - | 54 | mkdir -p .deps 55 | unset GOTOOLCHAIN 56 | ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${VERSION}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${REVISION}\" -extldflags -static" 57 | GOPATH=/eirctl/.deps GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} CGO_ENABLED=0 go build -mod=readonly -buildvcs=false -ldflags="$ldflags" \ 58 | -o ./dist/configmanager-${BUILD_GOOS}-${BUILD_GOARCH}${BUILD_SUFFIX} ./cmd 59 | echo "---" 60 | echo "Built: configmanager-${BUILD_GOOS}-${BUILD_GOARCH}${BUILD_SUFFIX}" 61 | echo "Version: ${VERSION}" 62 | reset_context: true 63 | variations: 64 | - BUILD_GOOS: darwin 65 | BUILD_GOARCH: amd64 66 | BUILD_SUFFIX: "" 67 | - BUILD_GOOS: darwin 68 | BUILD_GOARCH: arm64 69 | BUILD_SUFFIX: "" 70 | - BUILD_GOOS: linux 71 | BUILD_GOARCH: amd64 72 | BUILD_SUFFIX: "" 73 | - BUILD_GOOS: linux 74 | BUILD_GOARCH: arm64 75 | BUILD_SUFFIX: "" 76 | - BUILD_GOOS: windows 77 | BUILD_GOARCH: amd64 78 | BUILD_SUFFIX: ".exe" 79 | - BUILD_GOOS: windows 80 | BUILD_GOARCH: arm64 81 | BUILD_SUFFIX: ".exe" 82 | - BUILD_GOOS: windows 83 | BUILD_GOARCH: "386" 84 | BUILD_SUFFIX: ".exe" 85 | required: 86 | env: 87 | - VERSION 88 | - REVISION 89 | 90 | sonar:coverage:prep: 91 | context: bash 92 | command: 93 | - | 94 | sed -i 's|github.com/DevLabFoundry/configmanager/v2/||g' .coverage/out 95 | echo "Coverage file first 20 lines after conversion:" 96 | head -20 .coverage/out 97 | echo "Coverage file line count:" 98 | wc -l .coverage/out 99 | 100 | tag: 101 | description: | 102 | Usage `eirctl tag GIT_TAG=2111dsfsdfa REVISION=as2342432` 103 | 104 | command: | 105 | git tag -a ${VERSION} -m "ci tag release" ${REVISION} 106 | git push origin ${VERSION} 107 | required: 108 | env: 109 | - VERSION 110 | - REVISION 111 | 112 | 113 | -------------------------------------------------------------------------------- /examples/examples.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/DevLabFoundry/configmanager/v2" 9 | ) 10 | 11 | const DO_STUFF_WITH_VALS_HERE = "connstring:user@%v:host=%s/someschema..." 12 | 13 | // retrieveExample uses the standard Retrieve method on the API 14 | // this will return generator.ParsedMap which can be later used for more complex use cases 15 | func retrieveExample() { 16 | cm := configmanager.New(context.TODO()) 17 | cm.Config.WithTokenSeparator("://") 18 | 19 | pm, err := cm.Retrieve([]string{"token1", "token2"}) 20 | 21 | if err != nil { 22 | panic(err) 23 | } 24 | 25 | // put in a loop for many config params 26 | // or use the helper methods to return a yaml replaced struct 27 | // 28 | if pwd, ok := pm["token1"]; ok { 29 | if host, ok := pm["token2"]; ok { 30 | fmt.Println(fmt.Sprintf(DO_STUFF_WITH_VALS_HERE, pwd, fmt.Sprintf("%s", host))) 31 | } 32 | } 33 | } 34 | 35 | // retrieveStringOut accepts a string as an input 36 | func retrieveStringOut() { 37 | cm := configmanager.New(context.TODO()) 38 | // JSON Marshal K8s CRD into 39 | exampleK8sCrdMarshalled := `apiVersion: crd.foo.custom/v1alpha1 40 | kind: CustomFooCrd 41 | metadata: 42 | name: foo 43 | namespace: bar 44 | spec: 45 | name: baz 46 | secret_val: AWSSECRETS#/customfoo/secret-val 47 | owner: test_10016@example.com 48 | ` 49 | pm, err := cm.RetrieveWithInputReplaced(exampleK8sCrdMarshalled) 50 | 51 | if err != nil { 52 | panic(err) 53 | } 54 | fmt.Println(pm) 55 | } 56 | 57 | // ConfigTokenReplace uses configmanager to replace all occurences of 58 | // replaceable tokens inside a []byte 59 | // this is a re-useable method on all controllers 60 | // will just ignore any non specs without tokens 61 | func SpecConfigTokenReplace[T any](inputType T) (*T, error) { 62 | outType := new(T) 63 | rawBytes, err := json.Marshal(inputType) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | cm := configmanager.New(context.TODO()) 69 | // use custom token separator 70 | cm.Config.WithTokenSeparator("://") 71 | 72 | replaced, err := cm.RetrieveWithInputReplaced(string(rawBytes)) 73 | if err != nil { 74 | return nil, err 75 | } 76 | if err := json.Unmarshal([]byte(replaced), outType); err != nil { 77 | return nil, err 78 | } 79 | return outType, nil 80 | } 81 | 82 | // Example 83 | func exampleRetrieveYamlUnmarshalled() { 84 | 85 | type config struct { 86 | DbHost string `yaml:"dbhost"` 87 | Username string `yaml:"user"` 88 | Password string `yaml:"pass"` 89 | } 90 | configMarshalled := ` 91 | user: AWSPARAMSTR:///int-test/pocketbase/config|user 92 | pass: AWSPARAMSTR:///int-test/pocketbase/config|pwd 93 | dbhost: AWSPARAMSTR:///int-test/pocketbase/config|host 94 | ` 95 | 96 | appConf := &config{} 97 | cm := configmanager.New(context.TODO()) 98 | // use custom token separator inline with future releases 99 | cm.Config.WithTokenSeparator("://") 100 | err := cm.RetrieveUnmarshalledFromYaml([]byte(configMarshalled), appConf) 101 | if err != nil { 102 | panic(err) 103 | } 104 | fmt.Println(appConf.DbHost) 105 | fmt.Println(appConf.Username) 106 | fmt.Println(appConf.Password) 107 | } 108 | 109 | // ### exampleRetrieveYamlMarshalled 110 | func exampleRetrieveYamlMarshalled() { 111 | type config struct { 112 | DbHost string `yaml:"dbhost"` 113 | Username string `yaml:"user"` 114 | Password string `yaml:"pass"` 115 | } 116 | 117 | appConf := &config{ 118 | DbHost: "AWSPARAMSTR:///int-test/pocketbase/config|host", 119 | Username: "AWSPARAMSTR:///int-test/pocketbase/config|user", 120 | Password: "AWSPARAMSTR:///int-test/pocketbase/config|pwd", 121 | } 122 | 123 | cm := configmanager.New(context.TODO()) 124 | cm.Config.WithTokenSeparator("://") 125 | err := cm.RetrieveMarshalledYaml(appConf) 126 | if err != nil { 127 | panic(err) 128 | } 129 | if appConf.DbHost == "AWSPARAMSTR:///int-test/pocketbase/config|host" { 130 | panic(fmt.Errorf("value of DbHost should have been replaced with a value from token")) 131 | } 132 | fmt.Println(appConf.DbHost) 133 | fmt.Println(appConf.Username) 134 | fmt.Println(appConf.Password) 135 | } 136 | -------------------------------------------------------------------------------- /internal/store/aztablestorage.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Azure TableStore implementation 3 | **/ 4 | package store 5 | 6 | import ( 7 | "context" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "strings" 12 | 13 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 14 | "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" 15 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 16 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 17 | ) 18 | 19 | var ErrIncorrectlyStructuredToken = errors.New("incorrectly structured token") 20 | 21 | // tableStoreApi 22 | // uses this package https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/data/aztables 23 | type tableStoreApi interface { 24 | GetEntity(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) 25 | } 26 | 27 | type AzTableStore struct { 28 | svc tableStoreApi 29 | ctx context.Context 30 | logger log.ILogger 31 | config *AzTableStrgConfig 32 | token *config.ParsedTokenConfig 33 | // token only without table indicators 34 | // key only 35 | strippedToken string 36 | } 37 | 38 | type AzTableStrgConfig struct { 39 | Format string `json:"format"` 40 | } 41 | 42 | // NewAzTableStore 43 | func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logger log.ILogger) (*AzTableStore, error) { 44 | 45 | storeConf := &AzTableStrgConfig{} 46 | _ = token.ParseMetadata(storeConf) 47 | // initialToken := config.ParseMetadata(token, storeConf) 48 | backingStore := &AzTableStore{ 49 | ctx: ctx, 50 | logger: logger, 51 | config: storeConf, 52 | token: token, 53 | } 54 | 55 | srvInit := azServiceFromToken(token.StoreToken(), "https://%s.table.core.windows.net/%s", 2) 56 | backingStore.strippedToken = srvInit.token 57 | 58 | cred, err := azidentity.NewDefaultAzureCredential(nil) 59 | if err != nil { 60 | logger.Error("failed to get credentials: %v", err) 61 | return nil, err 62 | } 63 | 64 | c, err := aztables.NewClient(srvInit.serviceUri, cred, nil) 65 | if err != nil { 66 | logger.Error("failed to init the client: %v", err) 67 | return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) 68 | } 69 | 70 | backingStore.svc = c 71 | return backingStore, nil 72 | } 73 | 74 | // setToken already happens in the constructor 75 | func (implmt *AzTableStore) SetToken(token *config.ParsedTokenConfig) {} 76 | 77 | // tokenVal in AZ table storage if an Entity contains the `value` property 78 | // we attempt to extract it and return. 79 | // 80 | // From this point then normal rules of configmanager apply, 81 | // including keySeperator and lookup. 82 | func (imp *AzTableStore) Token() (string, error) { 83 | imp.logger.Info("AzTableSTore Token: %s", imp.token.String()) 84 | imp.logger.Info("Concrete implementation AzTableSTore") 85 | 86 | ctx, cancel := context.WithCancel(imp.ctx) 87 | defer cancel() 88 | 89 | // split the token for partition and rowKey 90 | pKey, rKey, err := azTableStoreTokenSplitter(imp.strippedToken) 91 | if err != nil { 92 | return "", err 93 | } 94 | 95 | s, err := imp.svc.GetEntity(ctx, pKey, rKey, &aztables.GetEntityOptions{}) 96 | if err != nil { 97 | imp.logger.Error(implementationNetworkErr, config.AzTableStorePrefix, err, imp.strippedToken) 98 | return "", fmt.Errorf(implementationNetworkErr+" %w", config.AzTableStorePrefix, err, imp.token.StoreToken(), ErrRetrieveFailed) 99 | } 100 | if len(s.Value) > 0 { 101 | // check for `value` property in entity 102 | checkVal := make(map[string]interface{}) 103 | _ = json.Unmarshal(s.Value, &checkVal) 104 | if checkVal["value"] != nil { 105 | return fmt.Sprintf("%v", checkVal["value"]), nil 106 | } 107 | return string(s.Value), nil 108 | } 109 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 110 | return "", nil 111 | } 112 | 113 | func azTableStoreTokenSplitter(token string) (partitionKey, rowKey string, err error) { 114 | splitToken := strings.Split(strings.TrimPrefix(token, "/"), "/") 115 | if len(splitToken) < 2 { 116 | return "", "", fmt.Errorf("token: %s - could not be correctly destructured to pluck the partition and row keys\n%w", token, ErrIncorrectlyStructuredToken) 117 | } 118 | partitionKey = splitToken[0] 119 | rowKey = splitToken[1] 120 | // naked return to save having to define another struct 121 | return 122 | } 123 | -------------------------------------------------------------------------------- /internal/store/paramstore_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 13 | "github.com/aws/aws-sdk-go-v2/service/ssm" 14 | "github.com/aws/aws-sdk-go-v2/service/ssm/types" 15 | ) 16 | 17 | // var ( 18 | // tsuccessParam = "someVal" 19 | // tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} 20 | // ) 21 | 22 | type mockParamApi func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) 23 | 24 | func (m mockParamApi) GetParameter(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { 25 | return m(ctx, params, optFns...) 26 | } 27 | 28 | func awsParamtStoreCommonGetChecker(t *testing.T, params *ssm.GetParameterInput) { 29 | if params.Name == nil { 30 | t.Fatal("expect name to not be nil") 31 | } 32 | 33 | if strings.Contains(*params.Name, "#") { 34 | t.Errorf("incorrectly stripped token separator") 35 | } 36 | 37 | if strings.Contains(*params.Name, string(config.ParamStorePrefix)) { 38 | t.Errorf("incorrectly stripped prefix") 39 | } 40 | 41 | if !*params.WithDecryption { 42 | t.Fatal("expect WithDecryption to not be false") 43 | } 44 | } 45 | 46 | func Test_GetParamStore(t *testing.T) { 47 | t.Parallel() 48 | 49 | var ( 50 | tsuccessParam = "someVal" 51 | // tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} 52 | ) 53 | tests := map[string]struct { 54 | token string 55 | keySeparator string 56 | tokenSeparator string 57 | expect string 58 | mockClient func(t *testing.T) paramStoreApi 59 | config *config.GenVarsConfig 60 | }{ 61 | "successVal": {"AWSPARAMSTR#/token/1", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { 62 | return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { 63 | t.Helper() 64 | awsParamtStoreCommonGetChecker(t, params) 65 | return &ssm.GetParameterOutput{ 66 | Parameter: &types.Parameter{Value: &tsuccessParam}, 67 | }, nil 68 | }) 69 | }, config.NewConfig(), 70 | }, 71 | "successVal with keyseparator": {"AWSPARAMSTR#/token/1|somekey", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { 72 | return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { 73 | t.Helper() 74 | awsParamtStoreCommonGetChecker(t, params) 75 | 76 | if strings.Contains(*params.Name, "|somekey") { 77 | t.Errorf("incorrectly stripped key separator") 78 | } 79 | 80 | return &ssm.GetParameterOutput{ 81 | Parameter: &types.Parameter{Value: &tsuccessParam}, 82 | }, nil 83 | }) 84 | }, config.NewConfig(), 85 | }, 86 | "errored": {"AWSPARAMSTR#/token/1", "|", "#", "unable to retrieve", func(t *testing.T) paramStoreApi { 87 | return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { 88 | t.Helper() 89 | awsParamtStoreCommonGetChecker(t, params) 90 | return nil, fmt.Errorf("unable to retrieve") 91 | }) 92 | }, config.NewConfig(), 93 | }, 94 | "nil to empty": {"AWSPARAMSTR#/token/1", "|", "#", "", func(t *testing.T) paramStoreApi { 95 | return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { 96 | t.Helper() 97 | awsParamtStoreCommonGetChecker(t, params) 98 | return &ssm.GetParameterOutput{ 99 | Parameter: &types.Parameter{Value: nil}, 100 | }, nil 101 | }) 102 | }, config.NewConfig(), 103 | }, 104 | } 105 | for name, tt := range tests { 106 | t.Run(name, func(t *testing.T) { 107 | 108 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) 109 | 110 | impl, err := NewParamStore(context.TODO(), log.New(io.Discard)) 111 | if err != nil { 112 | t.Errorf(testutils.TestPhrase, err.Error(), nil) 113 | } 114 | impl.svc = tt.mockClient(t) 115 | impl.SetToken(token) 116 | got, err := impl.Token() 117 | if err != nil { 118 | if err.Error() != tt.expect { 119 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 120 | } 121 | return 122 | } 123 | if got != tt.expect { 124 | t.Errorf(testutils.TestPhrase, got, tt.expect) 125 | } 126 | }) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /internal/store/secretsmanager_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 13 | "github.com/aws/aws-sdk-go-v2/service/secretsmanager" 14 | ) 15 | 16 | type mockSecretsApi func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) 17 | 18 | func (m mockSecretsApi) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 19 | return m(ctx, params, optFns...) 20 | } 21 | 22 | func awsSecretsMgrGetChecker(t *testing.T, params *secretsmanager.GetSecretValueInput) { 23 | if params.VersionStage == nil { 24 | t.Fatal("expect name to not be nil") 25 | } 26 | 27 | if strings.Contains(*params.SecretId, "#") { 28 | t.Errorf("incorrectly stripped token separator") 29 | } 30 | 31 | if strings.Contains(*params.SecretId, string(config.SecretMgrPrefix)) { 32 | t.Errorf("incorrectly stripped prefix") 33 | } 34 | } 35 | 36 | func Test_GetSecretMgr(t *testing.T) { 37 | t.Parallel() 38 | 39 | tsuccessSecret := "dsgkbdsf" 40 | 41 | tests := map[string]struct { 42 | token string 43 | keySeparator string 44 | tokenSeparator string 45 | expect string 46 | mockClient func(t *testing.T) secretsMgrApi 47 | config *config.GenVarsConfig 48 | }{ 49 | "success": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { 50 | return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 51 | t.Helper() 52 | awsSecretsMgrGetChecker(t, params) 53 | return &secretsmanager.GetSecretValueOutput{ 54 | SecretString: &tsuccessSecret, 55 | }, nil 56 | }) 57 | }, config.NewConfig(), 58 | }, 59 | "success with version": {"AWSSECRETS#/token/1[version=123]", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { 60 | return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 61 | t.Helper() 62 | awsSecretsMgrGetChecker(t, params) 63 | return &secretsmanager.GetSecretValueOutput{ 64 | SecretString: &tsuccessSecret, 65 | }, nil 66 | }) 67 | }, config.NewConfig(), 68 | }, 69 | "success with binary": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { 70 | return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 71 | t.Helper() 72 | awsSecretsMgrGetChecker(t, params) 73 | return &secretsmanager.GetSecretValueOutput{ 74 | SecretBinary: []byte(tsuccessSecret), 75 | }, nil 76 | }) 77 | }, config.NewConfig(), 78 | }, 79 | "errored": {"AWSSECRETS#/token/1", "|", "#", "unable to retrieve secret", func(t *testing.T) secretsMgrApi { 80 | return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 81 | t.Helper() 82 | awsSecretsMgrGetChecker(t, params) 83 | return nil, fmt.Errorf("unable to retrieve secret") 84 | }) 85 | }, config.NewConfig(), 86 | }, 87 | "ok but empty": {"AWSSECRETS#/token/1", "|", "#", "", func(t *testing.T) secretsMgrApi { 88 | return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { 89 | t.Helper() 90 | awsSecretsMgrGetChecker(t, params) 91 | return &secretsmanager.GetSecretValueOutput{ 92 | SecretString: nil, 93 | }, nil 94 | }) 95 | }, config.NewConfig(), 96 | }, 97 | } 98 | for name, tt := range tests { 99 | t.Run(name, func(t *testing.T) { 100 | 101 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) 102 | 103 | impl, _ := NewSecretsMgr(context.TODO(), log.New(io.Discard)) 104 | impl.svc = tt.mockClient(t) 105 | 106 | impl.SetToken(token) 107 | got, err := impl.Token() 108 | if err != nil { 109 | if err.Error() != tt.expect { 110 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 111 | } 112 | return 113 | } 114 | if got != tt.expect { 115 | t.Errorf(testutils.TestPhrase, got, tt.expect) 116 | } 117 | }) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /internal/cmdutils/cmdutils.go: -------------------------------------------------------------------------------- 1 | // pacakge Cmdutils 2 | // 3 | // Wraps around the ConfigManager library 4 | // with additional postprocessing capabilities for 5 | // output management when used with cli flags. 6 | package cmdutils 7 | 8 | import ( 9 | "fmt" 10 | "io" 11 | "os" 12 | "strings" 13 | 14 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 15 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 16 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 17 | "github.com/spf13/cobra" 18 | ) 19 | 20 | type configManagerIface interface { 21 | RetrieveWithInputReplaced(input string) (string, error) 22 | Retrieve(tokens []string) (generator.ParsedMap, error) 23 | GeneratorConfig() *config.GenVarsConfig 24 | } 25 | 26 | type CmdUtils struct { 27 | logger log.ILogger 28 | configManager configManagerIface 29 | outputWriter io.WriteCloser 30 | tempOutputWriter io.WriteCloser 31 | } 32 | 33 | func New(confManager configManagerIface, logger log.ILogger, writer io.WriteCloser) *CmdUtils { 34 | return &CmdUtils{ 35 | logger: logger, 36 | configManager: confManager, 37 | outputWriter: writer, 38 | } 39 | } 40 | 41 | // GenerateFromTokens is a helper cmd method to call from retrieve command 42 | func (c *CmdUtils) GenerateFromCmd(tokens []string) error { 43 | return c.generateFromToken(tokens) 44 | } 45 | 46 | // generateFromToken 47 | func (c *CmdUtils) generateFromToken(tokens []string) error { 48 | pm, err := c.configManager.Retrieve(tokens) 49 | if err != nil { 50 | // return full error to terminal if no tokens were parsed 51 | if len(pm) < 1 { 52 | return err 53 | } 54 | c.logger.Error("%v", err) 55 | } 56 | // Conver to ExportVars and flush to file 57 | pp := &PostProcessor{ProcessedMap: pm, Config: c.configManager.GeneratorConfig()} 58 | pp.ConvertToExportVar() 59 | return pp.FlushOutToFile(c.outputWriter) 60 | } 61 | 62 | // Generate a replaced string from string input command 63 | // 64 | // returns a non empty string if move of temp file is required 65 | func (c *CmdUtils) GenerateStrOut(input io.Reader, inputOutputIsSame bool) error { 66 | 67 | // outputs and inputs match and are file paths 68 | if inputOutputIsSame { 69 | c.logger.Debug("overwrite mode on") 70 | // create a temp file 71 | tempfile, err := os.CreateTemp(os.TempDir(), "configmanager") 72 | if err != nil { 73 | return err 74 | } 75 | defer os.Remove(tempfile.Name()) 76 | c.logger.Debug("tmp file created: %s", tempfile.Name()) 77 | c.tempOutputWriter = tempfile 78 | defer c.tempOutputWriter.Close() 79 | return c.generateFromStrOutOverwrite(input, tempfile.Name()) 80 | } 81 | 82 | return c.generateStrOutFromInput(input, c.outputWriter) 83 | } 84 | 85 | // generateFromStrOutOverwrite uses the same file for input as output 86 | // requires additional consideration and must create a temp file 87 | // and then write contents from temp to actual target 88 | // otherwise, two open file operations would be targeting same descriptor 89 | // will cause issues and inconsistent writes 90 | func (c *CmdUtils) generateFromStrOutOverwrite(input io.Reader, outtemp string) error { 91 | 92 | if err := c.generateStrOutFromInput(input, c.tempOutputWriter); err != nil { 93 | return err 94 | } 95 | tr, err := os.ReadFile(outtemp) 96 | if err != nil { 97 | return err 98 | } 99 | // move temp file to output path 100 | if _, err := c.outputWriter.Write(tr); err != nil { 101 | return err 102 | } 103 | return nil 104 | } 105 | 106 | // generateStrOutFromInput takes a reader and writer as input 107 | func (c *CmdUtils) generateStrOutFromInput(input io.Reader, writer io.Writer) error { 108 | 109 | b, err := io.ReadAll(input) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | str, err := c.configManager.RetrieveWithInputReplaced(string(b)) 115 | if err != nil { 116 | return err 117 | } 118 | pp := &PostProcessor{} 119 | 120 | return pp.StrToFile(writer, str) 121 | } 122 | 123 | type WriterCloserWrapper struct { 124 | io.Writer 125 | } 126 | 127 | func (swc *WriterCloserWrapper) Close() error { 128 | return nil 129 | } 130 | 131 | func GetWriter(outputpath string) (io.WriteCloser, error) { 132 | outputWriter := &WriterCloserWrapper{os.Stdout} 133 | if outputpath != "stdout" { 134 | return os.Create(outputpath) 135 | } 136 | return outputWriter, nil 137 | } 138 | 139 | func GetReader(cmd *cobra.Command, inputpath string) (io.Reader, error) { 140 | inputReader := cmd.InOrStdin() 141 | if inputpath != "-" && inputpath != "" { 142 | if _, err := os.Stat(inputpath); os.IsNotExist(err) { 143 | return strings.NewReader(inputpath), nil 144 | } 145 | return os.Open(inputpath) 146 | } 147 | return inputReader, nil 148 | } 149 | 150 | // UploadTokensWithVals takes in a map of key/value pairs and uploads them 151 | func (c *CmdUtils) UploadTokensWithVals(tokens map[string]string) error { 152 | return fmt.Errorf("notYetImplemented") 153 | } 154 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | 19 | 20 | ## Working with environemt variables 21 | 22 | 23 | 24 | ## Working with files 25 | 26 | Then you can access the single values like this `AWSSECRETS#/appxyz/service1-db-config|host` ==> `export SERVICE1_DB_CONFIG__HOST='db.internal'` 27 | 28 | Alternatively if you are `configmanager`-ing a file via the fromstr command and the input is something like this: 29 | 30 | (YAML) 31 | 32 | ```yaml 33 | app: 34 | name: xyz 35 | db: 36 | host: AWSSECRETS#/appxyz/service1-db-config|host 37 | port: AWSSECRETS#/appxyz/service1-db-config|port 38 | pass: AWSSECRETS#/appxyz/service1-db-config|pass 39 | ``` 40 | 41 | which would result in this 42 | 43 | ```yaml 44 | app: 45 | name: xyz 46 | db: 47 | host: db.internal 48 | port: 3306 49 | pass: sUp3$ecr3T! 50 | ``` 51 | 52 | If your config parameter matches the config interface, you can also leave the entire token to point to the `db` key 53 | 54 | ```yaml 55 | app: 56 | name: xyz 57 | db: AWSSECRETS#/appxyz/service1-db-config 58 | ``` 59 | 60 | result: 61 | 62 | ```yaml 63 | app: 64 | name: xyz 65 | db: { 66 | "host": "db.internal", 67 | "port": 3306, 68 | "pass": "sUp3$ecr3T!", 69 | } 70 | ``` 71 | 72 | ## Go API Examples 73 | 74 | One of the sample use cases includes implementation in a K8s controller. 75 | 76 | E.g. your Custom CRD stores some values in plain text that should really be secrets/nonpublic config parameters - something like this can be invoked from inside the controller code using the generator pkg API. 77 | 78 | See [examples](./examples/examples.go) for more examples and tests for sample input/usage 79 | 80 | ```go 81 | package main 82 | 83 | import ( 84 | "context" 85 | "fmt" 86 | 87 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 88 | "github.com/DevLabFoundry/configmanager/v2" 89 | ) 90 | 91 | func main() { 92 | cm := &configmanager.ConfigManager{} 93 | cnf := generator.NewConfig() 94 | // JSON Marshal K8s CRD into 95 | exampleK8sCrdMarshalled := `apiVersion: crd.foo.custom/v1alpha1 96 | kind: CustomFooCrd 97 | metadata: 98 | name: foo 99 | namespace: bar 100 | spec: 101 | name: baz 102 | secret_val: AWSSECRETS#/customfoo/secret-val 103 | owner: test_10016@example.com 104 | ` 105 | pm, err := cm.RetrieveWithInputReplaced(exampleK8sCrdMarshalled, *cnf) 106 | 107 | if err != nil { 108 | panic(err) 109 | } 110 | fmt.Println(pm) 111 | } 112 | ``` 113 | 114 | Above example would ensure that you can safely store config/secret values on a CRD in plain text. 115 | 116 | Or using go1.19+ [generics example](https://github.com/dnitsch/reststrategy/blob/d14ccec2b29bff646678ab9cf1775c0e93308569/controller/controller.go#L353). 117 | 118 | > Beware logging out the CRD after tokens have been replaced. 119 | 120 | Samlpe call to retrieve from inside an app/serverless function to only grab the relevant values from config. 121 | 122 | ```go 123 | package main 124 | 125 | import ( 126 | "context" 127 | "fmt" 128 | "log" 129 | "os" 130 | 131 | "github.com/DevLabFoundry/configmanager/v2" 132 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 133 | ) 134 | 135 | var ( 136 | DB_CONNECTION_STRING string = "someuser:%v@tcp(%s:3306)/someschema" 137 | DB_PASSWORD_SECRET_PATH string = os.Getenv("DB_PASSWORD_TOKEN") 138 | DB_HOST_URL string = os.Getenv("DB_URL_TOKEN") 139 | ) 140 | 141 | func main() { 142 | connString, err := credentialString(context.TODO, DB_PASSWORD_SECRET_PATH, DB_HOST_URL) 143 | if err != nil { 144 | log.Fatal(err) 145 | } 146 | 147 | } 148 | 149 | func credentialString(ctx context.Context, pwdToken, hostToken string) (string, error) { 150 | 151 | cnf := generator.NewConfig() 152 | 153 | pm, err := configmanager.Retrieve([]string{pwdToken, hostToken}, *cnf) 154 | 155 | if err != nil { 156 | return "", err 157 | } 158 | if pwd, ok := pm[pwdToken]; ok { 159 | if host, ok := pm[hostToken]; ok { 160 | return fmt.Sprintf(DB_CONNECTION_STRING, pwd, host), nil 161 | } 162 | } 163 | 164 | return "", fmt.Errorf("unable to find value via token") 165 | } 166 | ``` 167 | -------------------------------------------------------------------------------- /internal/strategy/strategy.go: -------------------------------------------------------------------------------- 1 | // Package strategy is a strategy pattern wrapper around the store implementations 2 | // 3 | // NOTE: this may be refactored out into the store package directly 4 | package strategy 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "fmt" 10 | "sync" 11 | 12 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/store" 15 | ) 16 | 17 | var ErrTokenInvalid = errors.New("invalid token - cannot get prefix") 18 | 19 | // StrategyFunc 20 | type StrategyFunc func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) 21 | 22 | // StrategyFuncMap 23 | type StrategyFuncMap map[config.ImplementationPrefix]StrategyFunc 24 | 25 | func defaultStrategyFuncMap(logger log.ILogger) map[config.ImplementationPrefix]StrategyFunc { 26 | return map[config.ImplementationPrefix]StrategyFunc{ 27 | config.AzTableStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 28 | return store.NewAzTableStore(ctx, token, logger) 29 | }, 30 | config.AzAppConfigPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 31 | return store.NewAzAppConf(ctx, token, logger) 32 | }, 33 | config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 34 | return store.NewGcpSecrets(ctx, logger) 35 | }, 36 | config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 37 | return store.NewSecretsMgr(ctx, logger) 38 | }, 39 | config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 40 | return store.NewParamStore(ctx, logger) 41 | }, 42 | config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 43 | return store.NewKvScrtStore(ctx, token, logger) 44 | }, 45 | config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 46 | return store.NewVaultStore(ctx, token, logger) 47 | }, 48 | } 49 | } 50 | 51 | type strategyFnMap struct { 52 | mu sync.Mutex 53 | funcMap StrategyFuncMap 54 | } 55 | type RetrieveStrategy struct { 56 | implementation store.Strategy 57 | config config.GenVarsConfig 58 | strategyFuncMap strategyFnMap 59 | } 60 | type Opts func(*RetrieveStrategy) 61 | 62 | // New 63 | func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *RetrieveStrategy { 64 | rs := &RetrieveStrategy{ 65 | config: config, 66 | strategyFuncMap: strategyFnMap{mu: sync.Mutex{}, funcMap: defaultStrategyFuncMap(logger)}, 67 | } 68 | // overwrite or add any options/defaults set above 69 | for _, o := range opts { 70 | o(rs) 71 | } 72 | 73 | return rs 74 | } 75 | 76 | // WithStrategyFuncMap Adds custom implementations for prefix 77 | // 78 | // Mainly used for testing 79 | // NOTE: this may lead to eventual optional configurations by users 80 | func WithStrategyFuncMap(funcMap StrategyFuncMap) Opts { 81 | return func(rs *RetrieveStrategy) { 82 | for prefix, implementation := range funcMap { 83 | rs.strategyFuncMap.mu.Lock() 84 | defer rs.strategyFuncMap.mu.Unlock() 85 | rs.strategyFuncMap.funcMap[config.ImplementationPrefix(prefix)] = implementation 86 | } 87 | } 88 | } 89 | 90 | func (rs *RetrieveStrategy) setImplementation(strategy store.Strategy) { 91 | rs.implementation = strategy 92 | } 93 | 94 | func (rs *RetrieveStrategy) setTokenVal(s *config.ParsedTokenConfig) { 95 | rs.implementation.SetToken(s) 96 | } 97 | 98 | func (rs *RetrieveStrategy) getTokenValue() (string, error) { 99 | return rs.implementation.Token() 100 | } 101 | 102 | type TokenResponse struct { 103 | value string 104 | key *config.ParsedTokenConfig 105 | Err error 106 | } 107 | 108 | func (tr *TokenResponse) Key() *config.ParsedTokenConfig { 109 | return tr.key 110 | } 111 | 112 | func (tr *TokenResponse) Value() string { 113 | return tr.value 114 | } 115 | 116 | // retrieveSpecificCh wraps around the specific strategy implementation 117 | // and publishes results to a channel 118 | func (rs *RetrieveStrategy) RetrieveByToken(ctx context.Context, impl store.Strategy, tokenConf *config.ParsedTokenConfig) *TokenResponse { 119 | cr := &TokenResponse{} 120 | cr.Err = nil 121 | cr.key = tokenConf 122 | rs.setImplementation(impl) 123 | rs.setTokenVal(tokenConf) 124 | s, err := rs.getTokenValue() 125 | if err != nil { 126 | cr.Err = err 127 | return cr 128 | } 129 | cr.value = s 130 | return cr 131 | } 132 | 133 | func (rs *RetrieveStrategy) SelectImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 134 | if token == nil { 135 | return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) 136 | } 137 | 138 | if store, found := rs.strategyFuncMap.funcMap[token.Prefix()]; found { 139 | return store(ctx, token) 140 | } 141 | 142 | return nil, fmt.Errorf("implementation not found for input string: %s", token) 143 | } 144 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/DevLabFoundry/configmanager/v2 2 | 3 | go 1.25.5 4 | 5 | require ( 6 | cloud.google.com/go/secretmanager v1.16.0 7 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 8 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 9 | github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig v1.2.0 10 | github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.1 11 | github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 12 | github.com/aws/aws-sdk-go-v2 v1.40.1 13 | github.com/aws/aws-sdk-go-v2/config v1.32.3 14 | github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.40.3 15 | github.com/aws/aws-sdk-go-v2/service/ssm v1.67.5 16 | github.com/go-test/deep v1.1.1 17 | github.com/googleapis/gax-go/v2 v2.15.0 18 | github.com/hashicorp/vault/api v1.22.0 19 | github.com/hashicorp/vault/api/auth/aws v0.11.0 20 | github.com/spf13/cobra v1.10.2 21 | github.com/spyzhov/ajson v0.9.6 22 | gopkg.in/yaml.v3 v3.0.1 23 | ) 24 | 25 | require github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 // indirect 26 | 27 | require ( 28 | cloud.google.com/go/auth v0.17.0 // indirect 29 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 30 | cloud.google.com/go/compute/metadata v0.9.0 // indirect 31 | cloud.google.com/go/iam v1.5.3 // indirect 32 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect 33 | github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect 34 | github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect 35 | github.com/a8m/envsubst v1.4.3 36 | github.com/aws/aws-sdk-go v1.55.8 // indirect 37 | github.com/aws/aws-sdk-go-v2/credentials v1.19.3 // indirect 38 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect 39 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect 40 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect 41 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect 42 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect 43 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect 46 | github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect 47 | github.com/aws/smithy-go v1.24.0 // indirect 48 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 49 | github.com/fatih/color v1.18.0 // indirect 50 | github.com/felixge/httpsnoop v1.0.4 // indirect 51 | github.com/go-jose/go-jose/v4 v4.1.3 // indirect 52 | github.com/go-logr/logr v1.4.3 // indirect 53 | github.com/go-logr/stdr v1.2.2 // indirect 54 | github.com/golang-jwt/jwt/v5 v5.3.0 // indirect 55 | github.com/google/s2a-go v0.1.9 // indirect 56 | github.com/google/uuid v1.6.0 // indirect 57 | github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect 58 | github.com/hashicorp/errwrap v1.1.0 // indirect 59 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 60 | github.com/hashicorp/go-hclog v1.6.3 // indirect 61 | github.com/hashicorp/go-multierror v1.1.1 // indirect 62 | github.com/hashicorp/go-retryablehttp v0.7.8 // indirect 63 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 64 | github.com/hashicorp/go-secure-stdlib/awsutil v0.3.0 // indirect 65 | github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect 66 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 67 | github.com/hashicorp/go-sockaddr v1.0.7 // indirect 68 | github.com/hashicorp/go-uuid v1.0.3 // indirect 69 | github.com/hashicorp/hcl v1.0.1-vault-7 // indirect 70 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 71 | github.com/jmespath/go-jmespath v0.4.0 // indirect 72 | github.com/kylelemons/godebug v1.1.0 // indirect 73 | github.com/mattn/go-colorable v0.1.14 // indirect 74 | github.com/mattn/go-isatty v0.0.20 // indirect 75 | github.com/mitchellh/go-homedir v1.1.0 // indirect 76 | github.com/mitchellh/mapstructure v1.5.0 // indirect 77 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 78 | github.com/pkg/errors v0.9.1 // indirect 79 | github.com/ryanuber/go-glob v1.0.0 // indirect 80 | github.com/spf13/pflag v1.0.10 // indirect 81 | go.opentelemetry.io/auto/sdk v1.2.1 // indirect 82 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect 83 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect 84 | go.opentelemetry.io/otel v1.38.0 // indirect 85 | go.opentelemetry.io/otel/metric v1.38.0 // indirect 86 | go.opentelemetry.io/otel/trace v1.38.0 // indirect 87 | golang.org/x/crypto v0.45.0 // indirect 88 | golang.org/x/net v0.47.0 // indirect 89 | golang.org/x/oauth2 v0.33.0 // indirect 90 | golang.org/x/sync v0.18.0 // indirect 91 | golang.org/x/sys v0.38.0 // indirect 92 | golang.org/x/text v0.31.0 // indirect 93 | golang.org/x/time v0.14.0 // indirect 94 | google.golang.org/api v0.257.0 // indirect 95 | google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect 96 | google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect 97 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect 98 | google.golang.org/grpc v1.77.0 // indirect 99 | google.golang.org/protobuf v1.36.10 // indirect 100 | ) 101 | -------------------------------------------------------------------------------- /internal/store/hashivault.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | 14 | vault "github.com/hashicorp/vault/api" 15 | auth "github.com/hashicorp/vault/api/auth/aws" 16 | ) 17 | 18 | // vaultHelper provides a broken up string 19 | type vaultHelper struct { 20 | path string 21 | token string 22 | } 23 | 24 | type hashiVaultApi interface { 25 | Get(ctx context.Context, secretPath string) (*vault.KVSecret, error) 26 | GetVersion(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) 27 | } 28 | 29 | type VaultStore struct { 30 | svc hashiVaultApi 31 | ctx context.Context 32 | logger log.ILogger 33 | config *VaultConfig 34 | token *config.ParsedTokenConfig 35 | strippedToken string 36 | } 37 | 38 | // VaultConfig holds the parseable metadata struct 39 | type VaultConfig struct { 40 | Version string `json:"version"` 41 | Role string `json:"iam_role"` 42 | } 43 | 44 | func NewVaultStore(ctx context.Context, token *config.ParsedTokenConfig, logger log.ILogger) (*VaultStore, error) { 45 | storeConf := &VaultConfig{} 46 | _ = token.ParseMetadata(storeConf) 47 | imp := &VaultStore{ 48 | ctx: ctx, 49 | logger: logger, 50 | config: storeConf, 51 | token: token, 52 | } 53 | 54 | config := vault.DefaultConfig() 55 | vt := splitToken(token.StoreToken()) 56 | imp.strippedToken = vt.token 57 | client, err := vault.NewClient(config) 58 | if err != nil { 59 | return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) 60 | } 61 | 62 | if strings.HasPrefix(os.Getenv("VAULT_TOKEN"), "aws_iam") { 63 | awsclient, err := newVaultStoreWithAWSAuthIAM(client, storeConf.Role) 64 | if err != nil { 65 | return nil, err 66 | } 67 | client = awsclient 68 | } 69 | imp.svc = client.KVv2(vt.path) 70 | return imp, nil 71 | } 72 | 73 | // newVaultStoreWithAWSAuthIAM returns an initialised client with AWSIAMAuth 74 | // EC2 auth type is not supported currently 75 | func newVaultStoreWithAWSAuthIAM(client *vault.Client, role string) (*vault.Client, error) { 76 | if len(role) < 1 { 77 | return nil, fmt.Errorf("role provided is empty, EC2 auth not supported") 78 | } 79 | awsAuth, err := auth.NewAWSAuth( 80 | auth.WithRole(role), 81 | ) 82 | if err != nil { 83 | return nil, fmt.Errorf("unable to initialize AWS auth method: %s. %w", err, ErrClientInitialization) 84 | } 85 | 86 | authInfo, err := client.Auth().Login(context.Background(), awsAuth) 87 | 88 | if err != nil { 89 | return nil, fmt.Errorf("unable to login to AWS auth method: %s. %w", err, ErrClientInitialization) 90 | } 91 | if authInfo == nil { 92 | return nil, fmt.Errorf("no auth info was returned after login") 93 | } 94 | 95 | return client, nil 96 | } 97 | 98 | // setTokenVal 99 | // imp.token is already set in the Vault constructor 100 | // 101 | // This happens inside the New func call 102 | // due to the way the client needs to be 103 | // initialised with a mountpath 104 | // and mountpath is part of the token so it is set then 105 | func (imp *VaultStore) SetToken(token *config.ParsedTokenConfig) {} 106 | 107 | // getTokenValue implements the underlying techonology 108 | // token retrieval and returns a stringified version 109 | // of the secret 110 | func (imp *VaultStore) Token() (string, error) { 111 | imp.logger.Info("%s", "Concrete implementation HashiVault") 112 | imp.logger.Info("Getting Secret: %s", imp.token) 113 | 114 | ctx, cancel := context.WithCancel(imp.ctx) 115 | defer cancel() 116 | 117 | secret, err := imp.getSecret(ctx, imp.strippedToken, imp.config.Version) 118 | if err != nil { 119 | imp.logger.Error(implementationNetworkErr, imp.token.Prefix(), err, imp.token.String()) 120 | return "", err 121 | } 122 | 123 | if secret.Data != nil { 124 | resp, err := marshall(secret.Data) 125 | if err != nil { 126 | imp.logger.Error("marshalling error: %s", err.Error()) 127 | return "", err 128 | } 129 | imp.logger.Debug("marhalled kvv2: %s", resp) 130 | return resp, nil 131 | } 132 | 133 | imp.logger.Error("value retrieved but empty for token: %v", imp.token) 134 | return "", nil 135 | } 136 | 137 | func (imp *VaultStore) getSecret(ctx context.Context, token string, version string) (*vault.KVSecret, error) { 138 | if version != "" { 139 | v, err := strconv.Atoi(version) 140 | if err != nil { 141 | return nil, fmt.Errorf("unable to parse version into an integer: %s", err.Error()) 142 | } 143 | return imp.svc.GetVersion(ctx, token, v) 144 | } 145 | return imp.svc.Get(ctx, token) 146 | } 147 | 148 | func splitToken(token string) vaultHelper { 149 | vh := vaultHelper{} 150 | // split token to extract the mount path 151 | s := strings.Split(strings.TrimPrefix(token, "/"), "___") 152 | // grab token and trim prefix if slash 153 | vh.token = strings.TrimPrefix(strings.Join(s[1:], ""), "/") 154 | // assign mount path as extracted from input token 155 | vh.path = s[0] 156 | return vh 157 | } 158 | 159 | // marshall converts map[string]any into a JSON 160 | // object. Secrets should only be a single level 161 | // deep. 162 | func marshall(secret map[string]any) (string, error) { 163 | b, err := json.Marshal(secret) 164 | if err != nil { 165 | return "", err 166 | } 167 | return string(b), nil 168 | } 169 | -------------------------------------------------------------------------------- /internal/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 7 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 8 | ) 9 | 10 | func Test_SelfName(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | }{ 14 | { 15 | name: "configmanager", 16 | }, 17 | } 18 | for _, tt := range tests { 19 | t.Run(tt.name, func(t *testing.T) { 20 | if tt.name != config.SELF_NAME { 21 | t.Error("self name does not match") 22 | } 23 | }) 24 | } 25 | } 26 | 27 | func Test_MarshalMetadata_with_label_struct_succeeds(t *testing.T) { 28 | type labelMeta struct { 29 | Label string `json:"label"` 30 | } 31 | 32 | ttests := map[string]struct { 33 | config *config.GenVarsConfig 34 | rawToken string 35 | wantLabel string 36 | wantMetaStrippedToken string 37 | }{ 38 | "when provider expects label on token and label exists": { 39 | config.NewConfig().WithTokenSeparator("://"), 40 | `AZTABLESTORE://basjh/dskjuds/123|d88[label=dev]`, 41 | "dev", 42 | "basjh/dskjuds/123", 43 | }, 44 | "when provider expects label on token and label does not exist": { 45 | config.NewConfig().WithTokenSeparator("://"), 46 | `AZTABLESTORE://basjh/dskjuds/123|d88[someother=dev]`, 47 | "", 48 | "basjh/dskjuds/123", 49 | }, 50 | "no metadata found": { 51 | config.NewConfig().WithTokenSeparator("://"), 52 | `AZTABLESTORE://basjh/dskjuds/123|d88`, 53 | "", 54 | "basjh/dskjuds/123", 55 | }, 56 | "no metadata found incorrect marker placement": { 57 | config.NewConfig().WithTokenSeparator("://"), 58 | `AZTABLESTORE://basjh/dskjuds/123|d88]asdas=bar[`, 59 | "", 60 | "basjh/dskjuds/123", 61 | }, 62 | "no metadata found incorrect marker placement and no key separator": { 63 | config.NewConfig().WithTokenSeparator("://"), 64 | `AZTABLESTORE://basjh/dskjuds/123]asdas=bar[`, 65 | "", 66 | "basjh/dskjuds/123]asdas=bar[", 67 | }, 68 | "no end found incorrect marker placement and no key separator": { 69 | config.NewConfig().WithTokenSeparator("://"), 70 | `AZTABLESTORE://basjh/dskjuds/123[asdas=bar`, 71 | "", 72 | "basjh/dskjuds/123[asdas=bar", 73 | }, 74 | "no start found incorrect marker placement and no key separator": { 75 | config.NewConfig().WithTokenSeparator("://"), 76 | `AZTABLESTORE://basjh/dskjuds/123]asdas=bar]`, 77 | "", 78 | "basjh/dskjuds/123]asdas=bar]", 79 | }, 80 | "metadata is in the middle of path lookup": { 81 | config.NewConfig().WithTokenSeparator("://"), 82 | `AZTABLESTORE://basjh/dskjuds/123[label=bar]|lookup`, 83 | "bar", 84 | "basjh/dskjuds/123", 85 | }, 86 | } 87 | for name, tt := range ttests { 88 | t.Run(name, func(t *testing.T) { 89 | inputTyp := &labelMeta{} 90 | got, err := config.NewParsedTokenConfig(tt.rawToken, *tt.config) 91 | 92 | if err != nil { 93 | t.Fatalf("got an error on NewParsedTokenconfig (%s)\n", tt.rawToken) 94 | } 95 | 96 | if got == nil { 97 | t.Errorf(testutils.TestPhraseWithContext, "Unable to parse token", nil, config.ParsedTokenConfig{}) 98 | } 99 | 100 | got.ParseMetadata(inputTyp) 101 | 102 | if got.StoreToken() != tt.wantMetaStrippedToken { 103 | t.Errorf(testutils.TestPhraseWithContext, "Token does not match", got.StripMetadata(), tt.wantMetaStrippedToken) 104 | } 105 | 106 | if inputTyp.Label != tt.wantLabel { 107 | t.Errorf(testutils.TestPhraseWithContext, "Metadata Label does not match", inputTyp.Label, tt.wantLabel) 108 | } 109 | }) 110 | } 111 | } 112 | 113 | func Test_TokenParser_config(t *testing.T) { 114 | type mockConfAwsSecrMgr struct { 115 | Version string `json:"version"` 116 | } 117 | ttests := map[string]struct { 118 | input string 119 | expPrefix config.ImplementationPrefix 120 | expLookupKeys string 121 | expStoreToken string 122 | expString string // fullToken 123 | expMetadataVersion string 124 | }{ 125 | "bare": {"AWSSECRETS://foo/bar", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar", ""}, 126 | "with metadata version": {"AWSSECRETS://foo/bar[version=123]", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar[version=123]", "123"}, 127 | "with keys lookup and label": {"AWSSECRETS://foo/bar|key1.key2[version=123]", config.SecretMgrPrefix, "key1.key2", "foo/bar", "AWSSECRETS://foo/bar|key1.key2[version=123]", "123"}, 128 | "with keys lookup and longer token": {"AWSSECRETS://foo/bar|key1.key2]version=123]", config.SecretMgrPrefix, "key1.key2]version=123]", "foo/bar", "AWSSECRETS://foo/bar|key1.key2]version=123]", ""}, 129 | "with keys lookup but no keys": {"AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", config.SecretMgrPrefix, "", "foo/bar/sdf/sddd.90dsfsd", "AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", "123"}, 130 | } 131 | for name, tt := range ttests { 132 | t.Run(name, func(t *testing.T) { 133 | conf := &mockConfAwsSecrMgr{} 134 | got, _ := config.NewParsedTokenConfig(tt.input, *config.NewConfig()) 135 | got.ParseMetadata(conf) 136 | 137 | if got.LookupKeys() != tt.expLookupKeys { 138 | t.Errorf(testutils.TestPhrase, got.LookupKeys(), tt.expLookupKeys) 139 | } 140 | if got.StoreToken() != tt.expStoreToken { 141 | t.Errorf(testutils.TestPhrase, got.StoreToken(), tt.expLookupKeys) 142 | } 143 | if got.String() != tt.expString { 144 | t.Errorf(testutils.TestPhrase, got.String(), tt.expString) 145 | } 146 | if got.Prefix() != tt.expPrefix { 147 | t.Errorf(testutils.TestPhrase, got.Prefix(), tt.expPrefix) 148 | } 149 | if conf.Version != tt.expMetadataVersion { 150 | t.Errorf(testutils.TestPhrase, conf.Version, tt.expMetadataVersion) 151 | } 152 | }) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /internal/store/azkeyvault_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 14 | ) 15 | 16 | func Test_azSplitToken(t *testing.T) { 17 | tests := []struct { 18 | name string 19 | token string 20 | expect azServiceHelper 21 | }{ 22 | { 23 | name: "simple_with_preceding_slash", 24 | token: "/test-vault/somejsontest", 25 | expect: azServiceHelper{ 26 | serviceUri: "https://test-vault.vault.azure.net", 27 | token: "somejsontest", 28 | }, 29 | }, 30 | { 31 | name: "missing_initial_slash", 32 | token: "test-vault/somejsontest", 33 | expect: azServiceHelper{ 34 | serviceUri: "https://test-vault.vault.azure.net", 35 | token: "somejsontest", 36 | }, 37 | }, 38 | { 39 | name: "missing_initial_slash_multislash_secretname", 40 | token: "test-vault/some/json/test", 41 | expect: azServiceHelper{ 42 | serviceUri: "https://test-vault.vault.azure.net", 43 | token: "some/json/test", 44 | }, 45 | }, 46 | { 47 | name: "with_initial_slash_multislash_secretname", 48 | token: "test-vault//some/json/test", 49 | expect: azServiceHelper{ 50 | serviceUri: "https://test-vault.vault.azure.net", 51 | token: "/some/json/test", 52 | }, 53 | }, 54 | } 55 | for _, tt := range tests { 56 | t.Run(tt.name, func(t *testing.T) { 57 | got := azServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) 58 | if got.token != tt.expect.token { 59 | t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) 60 | } 61 | if got.serviceUri != tt.expect.serviceUri { 62 | t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) 63 | } 64 | }) 65 | } 66 | } 67 | 68 | func azKvCommonGetSecretChecker(t *testing.T, name, version, expectedName string) { 69 | if name == "" { 70 | t.Errorf("expect name to not be nil") 71 | } 72 | if name != expectedName { 73 | t.Errorf(testutils.TestPhrase, name, expectedName) 74 | } 75 | 76 | if strings.Contains(name, "#") { 77 | t.Errorf("incorrectly stripped token separator") 78 | } 79 | 80 | if strings.Contains(name, string(config.AzKeyVaultSecretsPrefix)) { 81 | t.Errorf("incorrectly stripped prefix") 82 | } 83 | 84 | if version != "" { 85 | t.Fatal("expect version to be \"\" an empty string ") 86 | } 87 | } 88 | 89 | type mockAzKvSecretApi func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) 90 | 91 | func (m mockAzKvSecretApi) GetSecret(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 92 | return m(ctx, name, version, options) 93 | } 94 | 95 | func TestAzKeyVault(t *testing.T) { 96 | t.Parallel() 97 | 98 | tsuccessParam := "dssdfdweiuyh" 99 | tests := map[string]struct { 100 | token string 101 | expect string 102 | mockClient func(t *testing.T) kvApi 103 | config *config.GenVarsConfig 104 | }{ 105 | "successVal": {"AZKVSECRET#/test-vault//token/1", tsuccessParam, func(t *testing.T) kvApi { 106 | return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 107 | t.Helper() 108 | azKvCommonGetSecretChecker(t, name, "", "/token/1") 109 | resp := azsecrets.GetSecretResponse{} 110 | resp.Value = &tsuccessParam 111 | return resp, nil 112 | }) 113 | }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 114 | }, 115 | "successVal with version": {"AZKVSECRET#/test-vault//token/1[version:123]", tsuccessParam, func(t *testing.T) kvApi { 116 | return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 117 | t.Helper() 118 | azKvCommonGetSecretChecker(t, name, "", "/token/1") 119 | resp := azsecrets.GetSecretResponse{} 120 | resp.Value = &tsuccessParam 121 | return resp, nil 122 | }) 123 | }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 124 | }, 125 | "successVal with keyseparator": {"AZKVSECRET#/test-vault/token/1|somekey", tsuccessParam, func(t *testing.T) kvApi { 126 | return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 127 | t.Helper() 128 | azKvCommonGetSecretChecker(t, name, "", "token/1") 129 | 130 | resp := azsecrets.GetSecretResponse{} 131 | resp.Value = &tsuccessParam 132 | return resp, nil 133 | }) 134 | }, 135 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 136 | }, 137 | "errored": {"AZKVSECRET#/test-vault/token/1|somekey", "unable to retrieve secret", func(t *testing.T) kvApi { 138 | return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 139 | t.Helper() 140 | azKvCommonGetSecretChecker(t, name, "", "token/1") 141 | 142 | resp := azsecrets.GetSecretResponse{} 143 | return resp, fmt.Errorf("unable to retrieve secret") 144 | }) 145 | }, 146 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 147 | }, 148 | "empty": {"AZKVSECRET#/test-vault/token/1|somekey", "", func(t *testing.T) kvApi { 149 | return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { 150 | t.Helper() 151 | azKvCommonGetSecretChecker(t, name, "", "token/1") 152 | 153 | resp := azsecrets.GetSecretResponse{} 154 | return resp, nil 155 | }) 156 | }, 157 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 158 | }, 159 | } 160 | 161 | for name, tt := range tests { 162 | t.Run(name, func(t *testing.T) { 163 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 164 | 165 | impl, err := NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) 166 | if err != nil { 167 | t.Errorf("failed to init azkvstore") 168 | } 169 | 170 | impl.svc = tt.mockClient(t) 171 | got, err := impl.Token() 172 | if err != nil { 173 | if err.Error() != tt.expect { 174 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 175 | } 176 | return 177 | } 178 | 179 | if got != tt.expect { 180 | t.Errorf(testutils.TestPhrase, got, tt.expect) 181 | } 182 | }) 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /internal/store/azappconf_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" 11 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 12 | logger "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 14 | ) 15 | 16 | func azAppConfCommonChecker(t *testing.T, key string, expectedKey string, expectLabel string, opts *azappconfig.GetSettingOptions) { 17 | t.Helper() 18 | if key != expectedKey { 19 | t.Errorf(testutils.TestPhrase, key, expectedKey) 20 | } 21 | 22 | if expectLabel != "" { 23 | if opts == nil { 24 | t.Errorf(testutils.TestPhrase, nil, expectLabel) 25 | } 26 | if *opts.Label != expectLabel { 27 | t.Errorf(testutils.TestPhrase, opts.Label, expectLabel) 28 | } 29 | } 30 | } 31 | 32 | type mockAzAppConfApi func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) 33 | 34 | func (m mockAzAppConfApi) GetSetting(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 35 | return m(ctx, key, options) 36 | } 37 | 38 | func Test_AzAppConf_Success(t *testing.T) { 39 | t.Parallel() 40 | tsuccessParam := "somecvla" 41 | 42 | logr := logger.New(&bytes.Buffer{}) 43 | tests := map[string]struct { 44 | token string 45 | expect string 46 | mockClient func(t *testing.T) appConfApi 47 | config *config.GenVarsConfig 48 | }{ 49 | "successVal": { 50 | "AZAPPCONF#/test-app-config-instance/table//token/1", 51 | tsuccessParam, 52 | func(t *testing.T) appConfApi { 53 | return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 54 | azAppConfCommonChecker(t, key, "table//token/1", "", options) 55 | resp := azappconfig.GetSettingResponse{} 56 | resp.Value = &tsuccessParam 57 | return resp, nil 58 | }) 59 | }, 60 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 61 | }, 62 | "successVal with :// token Separator": { 63 | "AZAPPCONF:///test-app-config-instance/conf_key[label=dev]", 64 | tsuccessParam, 65 | func(t *testing.T) appConfApi { 66 | return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 67 | azAppConfCommonChecker(t, key, "conf_key", "dev", options) 68 | resp := azappconfig.GetSettingResponse{} 69 | resp.Value = &tsuccessParam 70 | return resp, nil 71 | }) 72 | }, 73 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), 74 | }, 75 | "successVal with :// token Separator and etag specified": { 76 | "AZAPPCONF:///test-app-config-instance/conf_key[label=dev,etag=sometifdsssdsfdi_string01209222]", 77 | tsuccessParam, 78 | func(t *testing.T) appConfApi { 79 | return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 80 | azAppConfCommonChecker(t, key, "conf_key", "dev", options) 81 | if !options.OnlyIfChanged.Equals("sometifdsssdsfdi_string01209222") { 82 | t.Errorf(testutils.TestPhraseWithContext, "Etag not correctly set", options.OnlyIfChanged, "sometifdsssdsfdi_string01209222") 83 | } 84 | resp := azappconfig.GetSettingResponse{} 85 | resp.Value = &tsuccessParam 86 | return resp, nil 87 | }) 88 | }, 89 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), 90 | }, 91 | "successVal with keyseparator but no val returned": { 92 | "AZAPPCONF#/test-app-config-instance/try_to_find|key_separator.lookup", 93 | "", 94 | func(t *testing.T) appConfApi { 95 | return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 96 | azAppConfCommonChecker(t, key, "try_to_find", "", options) 97 | resp := azappconfig.GetSettingResponse{} 98 | resp.Value = nil 99 | return resp, nil 100 | }) 101 | }, 102 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 103 | }, 104 | } 105 | 106 | for name, tt := range tests { 107 | t.Run(name, func(t *testing.T) { 108 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 109 | 110 | impl, err := NewAzAppConf(context.TODO(), token, logr) 111 | if err != nil { 112 | t.Errorf("failed to init AZAPPCONF") 113 | } 114 | 115 | impl.svc = tt.mockClient(t) 116 | got, err := impl.Token() 117 | if err != nil { 118 | if err.Error() != tt.expect { 119 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 120 | } 121 | return 122 | } 123 | 124 | if got != tt.expect { 125 | t.Errorf(testutils.TestPhrase, got, tt.expect) 126 | } 127 | }) 128 | } 129 | } 130 | 131 | func Test_AzAppConf_Error(t *testing.T) { 132 | t.Parallel() 133 | 134 | logr := logger.New(&bytes.Buffer{}) 135 | 136 | tests := map[string]struct { 137 | token string 138 | expect error 139 | mockClient func(t *testing.T) appConfApi 140 | config *config.GenVarsConfig 141 | }{ 142 | "errored on service method call": { 143 | "AZAPPCONF#/test-app-config-instance/table/token/ok", 144 | ErrRetrieveFailed, 145 | func(t *testing.T) appConfApi { 146 | return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { 147 | t.Helper() 148 | resp := azappconfig.GetSettingResponse{} 149 | return resp, fmt.Errorf("network error") 150 | }) 151 | }, 152 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 153 | }, 154 | } 155 | 156 | for name, tt := range tests { 157 | t.Run(name, func(t *testing.T) { 158 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 159 | impl, err := NewAzAppConf(context.TODO(), token, logr) 160 | if err != nil { 161 | t.Fatal("failed to init AZAPPCONF") 162 | } 163 | impl.svc = tt.mockClient(t) 164 | if _, err := impl.Token(); !errors.Is(err, tt.expect) { 165 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 166 | } 167 | }) 168 | } 169 | } 170 | 171 | func Test_fail_AzAppConf_Client_init(t *testing.T) { 172 | t.Parallel() 173 | 174 | logr := logger.New(&bytes.Buffer{}) 175 | 176 | // this is basically a wrap around test for the url.Parse method in the stdlib 177 | // as that is what the client uses under the hood 178 | token, _ := config.NewParsedTokenConfig("AZAPPCONF:///%25%65%6e%301-._~/") 183 | } 184 | if !errors.Is(err, ErrClientInitialization) { 185 | t.Fatalf(testutils.TestPhraseWithContext, "azappconf client init", err.Error(), ErrClientInitialization.Error()) 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /configmanager.go: -------------------------------------------------------------------------------- 1 | package configmanager 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "regexp" 9 | "slices" 10 | "strings" 11 | 12 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 13 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 14 | "github.com/a8m/envsubst" 15 | "gopkg.in/yaml.v3" 16 | ) 17 | 18 | const ( 19 | TERMINATING_CHAR string = `[^\'\"\s\n\\\,]` 20 | ) 21 | 22 | // generateAPI 23 | type generateAPI interface { 24 | Generate(tokens []string) (generator.ParsedMap, error) 25 | } 26 | 27 | type ConfigManager struct { 28 | Config *config.GenVarsConfig 29 | generator generateAPI 30 | } 31 | 32 | // New returns an initialised instance of ConfigManager 33 | // Uses default config for: 34 | // 35 | // ``` 36 | // outputPath = "" 37 | // keySeparator = "|" 38 | // tokenSeparator = "://" 39 | // ``` 40 | // 41 | // Calling cm.Config.WithXXX() will overwrite the generator config 42 | func New(ctx context.Context) *ConfigManager { 43 | cm := &ConfigManager{} 44 | cm.Config = config.NewConfig() 45 | cm.generator = generator.NewGenerator(ctx).WithConfig(cm.Config) 46 | return cm 47 | } 48 | 49 | // GeneratorConfig 50 | // Returns the gettable generator config 51 | func (c *ConfigManager) GeneratorConfig() *config.GenVarsConfig { 52 | return c.Config 53 | } 54 | 55 | // WithGenerator replaces the generator instance 56 | func (c *ConfigManager) WithGenerator(generator generateAPI) *ConfigManager { 57 | c.generator = generator 58 | return c 59 | } 60 | 61 | // Retrieve gets a rawMap from a set implementation 62 | // will be empty if no matches found 63 | func (c *ConfigManager) Retrieve(tokens []string) (generator.ParsedMap, error) { 64 | return c.retrieve(tokens) 65 | } 66 | 67 | func (c *ConfigManager) retrieve(tokens []string) (generator.ParsedMap, error) { 68 | return c.generator.Generate(tokens) 69 | } 70 | 71 | var ErrEnvSubst = errors.New("envsubst enabled and errored on") 72 | 73 | // RetrieveWithInputReplaced parses given input against all possible token strings 74 | // using regex to grab a list of found tokens in the given string and returns the replaced string 75 | func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) { 76 | // replaces all env vars using strict mode of no unset and no empty 77 | // 78 | // NOTE: this happens before the FindTokens is called 79 | // currently it uses a regex, and envsubst uses a more robust lexer => parser mechanism 80 | // 81 | // NOTE: configmanager needs an own lexer => parser to allow for easier modification extension in the future 82 | if c.GeneratorConfig().EnvSubstEnabled() { 83 | var err error 84 | input, err = envsubst.StringRestrictedNoDigit(input, true, true, false) 85 | if err != nil { 86 | return "", fmt.Errorf("%w\n%v", ErrEnvSubst, err) 87 | } 88 | } 89 | m, err := c.retrieve(FindTokens(input)) 90 | 91 | if err != nil { 92 | return "", err 93 | } 94 | 95 | return replaceString(m, input), nil 96 | } 97 | 98 | // FindTokens extracts all replaceable tokens 99 | // from a given input string 100 | func FindTokens(input string) []string { 101 | tokens := []string{} 102 | for k := range config.VarPrefix { 103 | matches := regexp.MustCompile(regexp.QuoteMeta(string(k))+`.(`+TERMINATING_CHAR+`+)`).FindAllString(input, -1) 104 | tokens = append(tokens, matches...) 105 | } 106 | return tokens 107 | } 108 | 109 | // replaceString fills tokens in a provided input with their actual secret/config values 110 | func replaceString(inputMap generator.ParsedMap, inputString string) string { 111 | 112 | oldNew := []string(nil) 113 | // ordered values by index 114 | for _, ov := range orderedKeysList(inputMap) { 115 | oldNew = append(oldNew, ov, fmt.Sprint(inputMap[ov])) 116 | } 117 | replacer := strings.NewReplacer(oldNew...) 118 | return replacer.Replace(inputString) 119 | } 120 | 121 | func orderedKeysList(inputMap generator.ParsedMap) []string { 122 | mkeys := inputMap.MapKeys() 123 | // order map by keys length so that when passed to the 124 | // replacer it will replace the longest first 125 | // removing the possibility of partially overwriting 126 | // another token with same prefix 127 | // the default sort is ascending 128 | slices.Sort(mkeys) 129 | return mkeys 130 | } 131 | 132 | // RetrieveMarshalledJson 133 | // 134 | // It marshalls an input pointer value of a type with appropriate struct tags in JSON 135 | // marshalls it into a string and runs the appropriate token replacement. 136 | // and fills the same pointer value with the replaced fields. 137 | // 138 | // This is useful for when you have another tool or framework already passing you a known type. 139 | // e.g. a CRD Spec in kubernetes - where you POSTed the json/yaml spec with tokens in it 140 | // but now want to use them with tokens replaced for values in a stateless way. 141 | // 142 | // Enables you to store secrets in CRD Specs and other metadata your controller can use 143 | func (cm *ConfigManager) RetrieveMarshalledJson(input any) error { 144 | 145 | // marshall type into a []byte 146 | // with tokens in a string like object 147 | rawBytes, err := json.Marshal(input) 148 | if err != nil { 149 | return err 150 | } 151 | // run the replacement of tokens for values 152 | replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) 153 | if err != nil { 154 | return err 155 | } 156 | // replace the original pointer value with replaced tokens 157 | if err := json.Unmarshal([]byte(replacedString), input); err != nil { 158 | return err 159 | } 160 | return nil 161 | } 162 | 163 | // RetrieveUnmarshalledFromJson 164 | // It accepts an already marshalled byte slice and pointer to the value type. 165 | // It fills the type with the replaced 166 | func (c *ConfigManager) RetrieveUnmarshalledFromJson(input []byte, output any) error { 167 | replaced, err := c.RetrieveWithInputReplaced(string(input)) 168 | if err != nil { 169 | return err 170 | } 171 | if err := json.Unmarshal([]byte(replaced), output); err != nil { 172 | return err 173 | } 174 | return nil 175 | } 176 | 177 | // RetrieveMarshalledYaml 178 | // 179 | // Same as RetrieveMarshalledJson 180 | func (cm *ConfigManager) RetrieveMarshalledYaml(input any) error { 181 | 182 | // marshall type into a []byte 183 | // with tokens in a string like object 184 | rawBytes, err := yaml.Marshal(input) 185 | if err != nil { 186 | return err 187 | } 188 | // run the replacement of tokens for values 189 | replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) 190 | if err != nil { 191 | return err 192 | } 193 | // replace the original pointer value with replaced tokens 194 | if err := yaml.Unmarshal([]byte(replacedString), input); err != nil { 195 | return err 196 | } 197 | return nil 198 | } 199 | 200 | // RetrieveUnmarshalledFromYaml 201 | // 202 | // Same as RetrieveUnmarshalledFromJson 203 | func (c *ConfigManager) RetrieveUnmarshalledFromYaml(input []byte, output any) error { 204 | replaced, err := c.RetrieveWithInputReplaced(string(input)) 205 | if err != nil { 206 | return err 207 | } 208 | if err := yaml.Unmarshal([]byte(replaced), output); err != nil { 209 | return err 210 | } 211 | return nil 212 | } 213 | -------------------------------------------------------------------------------- /internal/store/gcpsecrets_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 15 | "github.com/googleapis/gax-go/v2" 16 | ) 17 | 18 | type mockGcpSecretsApi func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) 19 | 20 | func (m mockGcpSecretsApi) AccessSecretVersion(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { 21 | return m(ctx, req, opts...) 22 | } 23 | 24 | var TEST_GCP_CREDS = []byte(`{ 25 | "type": "service_account", 26 | "project_id": "xxxxx", 27 | "private_key_id": "yyyyyyyyyyyy", 28 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDf842hcn5Nvp6e\n7yKARaCVIDfLXpKDhRwUOvHMzJ1ioRgQo/kbv1n4yHGCSUFyY6hKGj0HBjaGj5kE\n79H/6Y3dJNGhnsMnxBhHdo+3FI8QF0CHZh460NMZSAJ41UMQSBGssGVsNfyUzXGH\nLc45sIx/Twx3yr1k2GD3E8FlDcKlZqa3xGHf+aipg2X3NxbYi+Sz7Yed+SOMhNHl\ncX6E/TqG9n1aTyIwjMIHscCYarJqURkJxr24ukDroCeMxAfxYTdMvRU2e8pFEdoY\nrgUC88fYfaVI5txJ6j/ZKauKQX9Pa8tSyXJeGva3JYp4VC7V4IyoVviCUgEGWZDN\n6/i3zoF/AgMBAAECggEAcVBCcVYFIkE48SH+Svjv74SFtpj7eSB4vKO2hPFjEOyB\nyKmu+aMwWvjQtiNqwf46wIPWLR+vpxYxTpYpo1sBNMvUZfp2tEA8KKyMuw3j9ThO\npjO9R/UxWrFcztbZP/u3NbFrH/2Q95mbv9IlbnsuG5xbqqEig0wYg+uzBvaXbig3\n/Jr0vLT2BkRCBKQkYGjVZcHlHVLoF7/J8cghFgkV1PGvknOv6/q7qzn9L4TjQIet\nfhrhN8Z1vgFiSYtpjP6YQEUEPSHmCQeD3WzJcnASPpU2uCUwd/z65ltKPnn+rqMt\n6jt9R1S1Ju2ZSjv+kR5fIXzihdOzncyzDDm33c/QwQKBgQD2QDZuzLjTxnhsfGii\nKJDAts+Jqfs/6SeEJcJKtEngj4m7rgzyEjbKVp8qtRHIzglKRWAe62/qzzy2BkKi\nvAd4+ZzmG2SkgypGsKVfjGXVFixz2gtUdmBOmK/TnYsxNT9yTt+rX9IGqKK60q73\nOWl8VsliLIsfvSH7+bqi7sRcXQKBgQDo0VUebyQHoTAXPdzGy2ysrVPDiHcldH0Y\n/hvhQTZwxYaJr3HpOCGol2Xl6zyawuudEQsoQwJ3Li6yeb0YMGiWX77/t+qX3pSn\nkGuoftGaNDV7sLn9UV2y+InF8EL1CasrhG1k5RIuxyfV0w+QUo+E7LpVR5XkbJqT\n9QNKnDQXiwKBgQDvvEYCCqbp7e/xVhEbxbhfFdro4Cat6tRAz+3egrTlvXhO0jzi\nMp9Kz5f3oP5ma0gaGX5hu75icE1fvKqE+d+ghAqe7w5FJzkyRulJI0tEb2jphN7A\n5NoPypBqyZboWjmhlG4mzouPVf/POCuEnk028truDAWJ6by7Lj3oP+HFNQKBgQCc\n5BQ8QiFBkvnZb7LLtGIzq0n7RockEnAK25LmJRAOxs13E2fsBguIlR3x5qgckqY8\nXjPqmd2bet+1HhyzpEuWqkcIBGRum2wJz2T9UxjklbJE/D8Z2i8OYDZX0SUOA8n5\ntXASwduS8lqB2Y1vcHOO3AhlV6xHFnjEpCPnr4PbKQKBgAhQ9D9MPeuz+5yw3yHg\nkvULZRtud+uuaKrOayprN25RTxr9c0erxqnvM7KHeo6/urOXeEa7x2n21kAT0Nch\nkF2RtWBLZKXGZEVBtw1Fw0UKNh4IDgM26dwlzRfTVHCiw6M6dCiTNk9KkP2vlkim\n3QFDSSUp+eBTXA17WkDAQf7w\n-----END PRIVATE KEY-----\n", 29 | "client_email": "foo@project.iam.gserviceaccount.com", 30 | "client_id": "99999911111111", 31 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 32 | "token_uri": "https://oauth2.googleapis.com/token", 33 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 34 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/bla" 35 | }`) 36 | 37 | func fixtureInitMockClient(t *testing.T) struct { 38 | name string 39 | close func() error 40 | delete func(name string) error 41 | } { 42 | 43 | cf, err := os.CreateTemp("", "gcp-creds*") 44 | if err != nil { 45 | t.Fatalf(testutils.TestPhraseWithContext, "unable to set up creds file", err.Error(), nil) 46 | } 47 | if _, err := cf.Write(TEST_GCP_CREDS); err != nil { 48 | t.Fatalf(testutils.TestPhraseWithContext, "unable to write mock creds into file", err.Error(), nil) 49 | } 50 | 51 | resp := struct { 52 | name string 53 | close func() error 54 | delete func(name string) error 55 | }{ 56 | name: cf.Name(), 57 | close: cf.Close, 58 | delete: os.Remove, 59 | } 60 | return resp 61 | } 62 | 63 | func gcpSecretsGetChecker(t *testing.T, req *gcpsecretspb.AccessSecretVersionRequest) { 64 | t.Helper() 65 | if req.Name == "" { 66 | t.Fatal("expect name to not be nil") 67 | } 68 | if strings.Contains(req.Name, "#") { 69 | t.Errorf("incorrectly stripped token separator") 70 | } 71 | if strings.Contains(req.Name, string(config.GcpSecretsPrefix)) { 72 | t.Errorf("incorrectly stripped prefix") 73 | } 74 | } 75 | 76 | func Test_GetGcpSecretVarHappy(t *testing.T) { 77 | // t.Parallel() 78 | 79 | tests := map[string]struct { 80 | token string 81 | expect string 82 | mockClient func(t *testing.T) gcpSecretsApi 83 | config *config.GenVarsConfig 84 | }{ 85 | "success": {"GCPSECRETS#/token/1", "someValue", func(t *testing.T) gcpSecretsApi { 86 | return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { 87 | gcpSecretsGetChecker(t, req) 88 | return &gcpsecretspb.AccessSecretVersionResponse{ 89 | Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, 90 | }, nil 91 | }) 92 | }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), 93 | }, 94 | "success with version": {"GCPSECRETS#/token/1[version=123]", "someValue", func(t *testing.T) gcpSecretsApi { 95 | return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { 96 | gcpSecretsGetChecker(t, req) 97 | return &gcpsecretspb.AccessSecretVersionResponse{ 98 | Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, 99 | }, nil 100 | }) 101 | }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), 102 | }, 103 | "error": {"GCPSECRETS#/token/1", "unable to retrieve secret", func(t *testing.T) gcpSecretsApi { 104 | return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { 105 | gcpSecretsGetChecker(t, req) 106 | return nil, fmt.Errorf("unable to retrieve secret") 107 | }) 108 | }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), 109 | }, 110 | "found but empty": { 111 | "GCPSECRETS#/token/1", 112 | "", 113 | func(t *testing.T) gcpSecretsApi { 114 | return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { 115 | gcpSecretsGetChecker(t, req) 116 | return &gcpsecretspb.AccessSecretVersionResponse{}, nil 117 | }) 118 | }, 119 | config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), 120 | }, 121 | } 122 | for name, tt := range tests { 123 | t.Run(name, func(t *testing.T) { 124 | fixture := fixtureInitMockClient(t) 125 | defer fixture.close() 126 | defer fixture.delete(fixture.name) 127 | 128 | os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fixture.name) 129 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 130 | 131 | impl, err := NewGcpSecrets(context.TODO(), log.New(io.Discard)) 132 | 133 | if err != nil { 134 | t.Errorf(testutils.TestPhrase, err.Error(), nil) 135 | } 136 | 137 | impl.svc = tt.mockClient(t) 138 | impl.close = func() error { return nil } 139 | impl.SetToken(token) 140 | got, err := impl.Token() 141 | 142 | if err != nil { 143 | if err.Error() != tt.expect { 144 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 145 | } 146 | return 147 | } 148 | if got != tt.expect { 149 | t.Errorf(testutils.TestPhrase, got, tt.expect) 150 | } 151 | }) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /pkg/generator/generator.go: -------------------------------------------------------------------------------- 1 | package generator 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "strconv" 9 | "sync" 10 | 11 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/strategy" 14 | "github.com/spyzhov/ajson" 15 | ) 16 | 17 | // GenVars is the main struct holding the 18 | // strategy patterns iface 19 | // any initialised config if overridded with withers 20 | // as well as the final outString and the initial rawMap 21 | // which wil be passed in a loop into a goroutine to perform the 22 | // relevant strategy network calls to the config store implementations 23 | type GenVars struct { 24 | Logger log.ILogger 25 | strategy strategy.StrategyFuncMap 26 | ctx context.Context 27 | config config.GenVarsConfig 28 | // rawMap is the internal object that holds the values 29 | // of original token => retrieved value - decrypted in plain text 30 | // with a mutex RW locker 31 | rawMap tokenMapSafe //ParsedMap 32 | } 33 | 34 | type Opts func(*GenVars) 35 | 36 | // NewGenerator returns a new instance of Generator 37 | // with a default strategy pattern wil be overwritten 38 | // during the first run of a found tokens map 39 | func NewGenerator(ctx context.Context, opts ...Opts) *GenVars { 40 | // defaultStrategy := NewDefatultStrategy() 41 | return newGenVars(ctx, opts...) 42 | } 43 | 44 | func newGenVars(ctx context.Context, opts ...Opts) *GenVars { 45 | m := make(ParsedMap) 46 | conf := config.NewConfig() 47 | g := &GenVars{ 48 | Logger: log.New(io.Discard), 49 | rawMap: tokenMapSafe{ 50 | tokenMap: m, 51 | mu: &sync.Mutex{}, 52 | }, 53 | ctx: ctx, 54 | // return using default config 55 | config: *conf, 56 | } 57 | g.strategy = nil 58 | 59 | // now apply additional opts 60 | for _, o := range opts { 61 | o(g) 62 | } 63 | 64 | return g 65 | } 66 | 67 | // WithStrategyMap 68 | // 69 | // Adds addtional funcs for storageRetrieval used for testing only 70 | func (c *GenVars) WithStrategyMap(sm strategy.StrategyFuncMap) *GenVars { 71 | c.strategy = sm 72 | return c 73 | } 74 | 75 | // WithConfig uses custom config 76 | func (c *GenVars) WithConfig(cfg *config.GenVarsConfig) *GenVars { 77 | // backwards compatibility 78 | if cfg != nil { 79 | c.config = *cfg 80 | } 81 | return c 82 | } 83 | 84 | // WithContext uses caller passed context 85 | func (c *GenVars) WithContext(ctx context.Context) *GenVars { 86 | c.ctx = ctx 87 | return c 88 | } 89 | 90 | // Config gets Config on the GenVars 91 | func (c *GenVars) Config() *config.GenVarsConfig { 92 | return &c.config 93 | } 94 | 95 | // ParsedMap is the internal working object definition and 96 | // the return type if results are not flushed to file 97 | type ParsedMap map[string]any 98 | 99 | func (pm ParsedMap) MapKeys() (keys []string) { 100 | for k := range pm { 101 | keys = append(keys, k) 102 | } 103 | return 104 | } 105 | 106 | type tokenMapSafe struct { 107 | mu *sync.Mutex 108 | tokenMap ParsedMap 109 | } 110 | 111 | func (tms *tokenMapSafe) getTokenMap() ParsedMap { 112 | tms.mu.Lock() 113 | defer tms.mu.Unlock() 114 | return tms.tokenMap 115 | } 116 | 117 | func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { 118 | tms.mu.Lock() 119 | defer tms.mu.Unlock() 120 | // NOTE: still use the metadata in the key 121 | // there could be different versions / labels for the same token and hence different values 122 | // However the JSONpath look up 123 | tms.tokenMap[key.String()] = keySeparatorLookup(key, val) 124 | } 125 | 126 | type rawTokenMap struct { 127 | mu sync.Mutex 128 | tokenMap map[string]*config.ParsedTokenConfig 129 | } 130 | 131 | func newRawTokenMap() *rawTokenMap { 132 | return &rawTokenMap{mu: sync.Mutex{}, tokenMap: map[string]*config.ParsedTokenConfig{}} 133 | } 134 | 135 | func (rtm *rawTokenMap) addToken(name string, parsedToken *config.ParsedTokenConfig) { 136 | rtm.mu.Lock() 137 | defer rtm.mu.Unlock() 138 | rtm.tokenMap[name] = parsedToken 139 | } 140 | 141 | func (rtm *rawTokenMap) mapOfToken() map[string]*config.ParsedTokenConfig { 142 | rtm.mu.Lock() 143 | defer rtm.mu.Unlock() 144 | return rtm.tokenMap 145 | } 146 | 147 | // Generate generates a k/v map of the tokens with their corresponding secret/paramstore values 148 | // the standard pattern of a token should follow a path like string 149 | func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { 150 | 151 | rtm := newRawTokenMap() 152 | for _, token := range tokens { 153 | // TODO: normalize tokens here potentially 154 | // merge any tokens that only differ in keys lookup inside the object 155 | parsedToken, err := config.NewParsedTokenConfig(token, c.config) 156 | if err != nil { 157 | c.Logger.Info(err.Error()) 158 | continue 159 | } 160 | rtm.addToken(token, parsedToken) 161 | } 162 | // pass in default initialised retrieveStrategy 163 | // input should be 164 | if err := c.generate(rtm); err != nil { 165 | return nil, err 166 | } 167 | return c.rawMap.getTokenMap(), nil 168 | } 169 | 170 | // generate checks if any tokens found 171 | // initiates groutines with fixed size channel map 172 | // to capture responses and errors 173 | // generates ParsedMap which includes 174 | func (c *GenVars) generate(rawMap *rawTokenMap) error { 175 | rtm := rawMap.mapOfToken() 176 | if len(rtm) < 1 { 177 | c.Logger.Debug("no replaceable tokens found in input") 178 | return nil 179 | } 180 | 181 | tokenCount := len(rtm) 182 | outCh := make(chan *strategy.TokenResponse, tokenCount) 183 | 184 | // TODO: initialise the singleton serviceContainer 185 | // pass into each goroutine 186 | for _, parsedToken := range rtm { 187 | token := parsedToken // safe closure capture 188 | // take value from config allocation on a per iteration basis 189 | go func() { 190 | s := strategy.New(c.config, c.Logger, strategy.WithStrategyFuncMap(c.strategy)) 191 | storeStrategy, err := s.SelectImplementation(c.ctx, token) 192 | if err != nil { 193 | outCh <- &strategy.TokenResponse{Err: err} 194 | return 195 | } 196 | outCh <- s.RetrieveByToken(c.ctx, storeStrategy, token) 197 | }() 198 | } 199 | 200 | // Fan-in: receive results with pure select 201 | received := 0 202 | for received < tokenCount { 203 | select { 204 | case cr := <-outCh: 205 | if cr == nil { 206 | continue // defensive (shouldn't happen) 207 | } 208 | c.Logger.Debug("cro: %+v", cr) 209 | if cr.Err != nil { 210 | c.Logger.Debug("cr.err %v, for token: %s", cr.Err, cr.Key()) 211 | } else { 212 | c.rawMap.addKeyVal(cr.Key(), cr.Value()) 213 | } 214 | received++ 215 | case <-c.ctx.Done(): 216 | c.Logger.Debug("context done: %v", c.ctx.Err()) 217 | return c.ctx.Err() // propagate context error (cancel/timeout) 218 | } 219 | } 220 | return nil 221 | } 222 | 223 | // IsParsed will try to parse the return found string into 224 | // map[string]string 225 | // If found it will convert that to a map with all keys uppercased 226 | // and any characters 227 | func IsParsed(v any, trm ParsedMap) bool { 228 | str := fmt.Sprint(v) 229 | err := json.Unmarshal([]byte(str), &trm) 230 | return err == nil 231 | } 232 | 233 | // keySeparatorLookup checks if the key contains 234 | // keySeparator character 235 | // If it does contain one then it tries to parse 236 | func keySeparatorLookup(key *config.ParsedTokenConfig, val string) string { 237 | // key has separator 238 | k := key.LookupKeys() 239 | if k == "" { 240 | // c.logger.Info("no keyseparator found") 241 | return val 242 | } 243 | 244 | keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) 245 | if err != nil { 246 | // c.logger.Debug("unable to parse as json object %v", err.Error()) 247 | return val 248 | } 249 | 250 | if len(keys) == 1 { 251 | v := keys[0] 252 | if v.Type() == ajson.String { 253 | str, err := strconv.Unquote(fmt.Sprintf("%v", v)) 254 | if err != nil { 255 | // c.logger.Debug("unable to unquote value: %v returning as is", v) 256 | return fmt.Sprintf("%v", v) 257 | } 258 | return str 259 | } 260 | 261 | return fmt.Sprintf("%v", v) 262 | } 263 | 264 | // c.logger.Info("no value found in json using path expression") 265 | return "" 266 | } 267 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | SELF_NAME = "configmanager" 12 | ) 13 | 14 | const ( 15 | // tokenSeparator used for identifying the end of a prefix and beginning of token 16 | // see notes about special consideration for AZKVSECRET tokens 17 | tokenSeparator = "://" 18 | // keySeparator used for accessing nested objects within the retrieved map 19 | keySeparator = "|" 20 | ) 21 | 22 | type ImplementationPrefix string 23 | 24 | const ( 25 | // AWS SecretsManager prefix 26 | SecretMgrPrefix ImplementationPrefix = "AWSSECRETS" 27 | // AWS Parameter Store prefix 28 | ParamStorePrefix ImplementationPrefix = "AWSPARAMSTR" 29 | // Azure Key Vault Secrets prefix 30 | AzKeyVaultSecretsPrefix ImplementationPrefix = "AZKVSECRET" 31 | // Azure Key Vault Secrets prefix 32 | AzTableStorePrefix ImplementationPrefix = "AZTABLESTORE" 33 | // Azure App Config prefix 34 | AzAppConfigPrefix ImplementationPrefix = "AZAPPCONF" 35 | // Hashicorp Vault prefix 36 | HashicorpVaultPrefix ImplementationPrefix = "VAULT" 37 | // GcpSecrets 38 | GcpSecretsPrefix ImplementationPrefix = "GCPSECRETS" 39 | // Unknown 40 | UnknownPrefix ImplementationPrefix = "UNKNOWN" 41 | ) 42 | 43 | var ( 44 | // default varPrefix used by the replacer function 45 | // any token must beging with one of these else 46 | // it will be skipped as not a replaceable token 47 | VarPrefix = map[ImplementationPrefix]bool{ 48 | SecretMgrPrefix: true, ParamStorePrefix: true, AzKeyVaultSecretsPrefix: true, 49 | GcpSecretsPrefix: true, HashicorpVaultPrefix: true, AzTableStorePrefix: true, 50 | AzAppConfigPrefix: true, UnknownPrefix: true, 51 | } 52 | ) 53 | 54 | // GenVarsConfig defines the input config object to be passed 55 | type GenVarsConfig struct { 56 | outpath string 57 | tokenSeparator string 58 | keySeparator string 59 | enableEnvSubst bool 60 | // parseAdditionalVars func(token string) TokenConfigVars 61 | } 62 | 63 | // NewConfig 64 | func NewConfig() *GenVarsConfig { 65 | return &GenVarsConfig{ 66 | tokenSeparator: tokenSeparator, 67 | keySeparator: keySeparator, 68 | } 69 | } 70 | 71 | // WithOutputPath 72 | func (c *GenVarsConfig) WithOutputPath(out string) *GenVarsConfig { 73 | c.outpath = out 74 | return c 75 | } 76 | 77 | // WithTokenSeparator adds a custom token separator 78 | // token is the actual value of the parameter/secret in the 79 | // provider store 80 | func (c *GenVarsConfig) WithTokenSeparator(tokenSeparator string) *GenVarsConfig { 81 | c.tokenSeparator = tokenSeparator 82 | return c 83 | } 84 | 85 | // WithKeySeparator adds a custom key separotor 86 | func (c *GenVarsConfig) WithKeySeparator(keySeparator string) *GenVarsConfig { 87 | c.keySeparator = keySeparator 88 | return c 89 | } 90 | 91 | // WithKeySeparator adds a custom key separotor 92 | func (c *GenVarsConfig) WithEnvSubst(enabled bool) *GenVarsConfig { 93 | c.enableEnvSubst = enabled 94 | return c 95 | } 96 | 97 | // OutputPath returns the outpath set in the config 98 | func (c *GenVarsConfig) OutputPath() string { 99 | return c.outpath 100 | } 101 | 102 | // TokenSeparator returns the tokenSeparator set in the config 103 | func (c *GenVarsConfig) TokenSeparator() string { 104 | return c.tokenSeparator 105 | } 106 | 107 | // KeySeparator returns the keySeparator set in the config 108 | func (c *GenVarsConfig) KeySeparator() string { 109 | return c.keySeparator 110 | } 111 | 112 | // EnvSubstEnabled returns whether or not envsubst is enabled 113 | func (c *GenVarsConfig) EnvSubstEnabled() bool { 114 | return c.enableEnvSubst 115 | } 116 | 117 | // Config returns the derefed value 118 | func (c *GenVarsConfig) Config() GenVarsConfig { 119 | cc := *c 120 | return cc 121 | } 122 | 123 | // Parsed token config section 124 | 125 | var ErrInvalidTokenPrefix = errors.New("token prefix has no implementation") 126 | 127 | type ParsedTokenConfig struct { 128 | prefix ImplementationPrefix 129 | keySeparator, tokenSeparator string 130 | prefixLessToken, fullToken string 131 | metadataStr, keysPath string 132 | storeToken, metadataLess string 133 | } 134 | 135 | // NewParsedTokenConfig returns a pointer to a new TokenConfig struct 136 | // returns nil if current prefix does not correspond to an Implementation 137 | // 138 | // The caller needs to make sure it is not nil 139 | // TODO: a custom parser would be best here 140 | func NewParsedTokenConfig(token string, config GenVarsConfig) (*ParsedTokenConfig, error) { 141 | ptc := &ParsedTokenConfig{} 142 | prfx := strings.Split(token, config.TokenSeparator())[0] 143 | 144 | // This should already only be a list of properly supported tokens but just in case 145 | if found := VarPrefix[ImplementationPrefix(prfx)]; !found { 146 | return nil, fmt.Errorf("prefix: %s\n%w", prfx, ErrInvalidTokenPrefix) 147 | } 148 | 149 | ptc.keySeparator = config.keySeparator 150 | ptc.tokenSeparator = config.tokenSeparator 151 | ptc.prefix = ImplementationPrefix(prfx) 152 | ptc.fullToken = token 153 | return ptc.new(), nil 154 | } 155 | 156 | func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { 157 | // order must be respected here 158 | // 159 | ptc.prefixLessToken = strings.Replace(ptc.fullToken, fmt.Sprintf("%s%s", ptc.prefix, ptc.tokenSeparator), "", 1) 160 | 161 | // token without metadata and the string itself 162 | ptc.extractMetadataStr() 163 | // token without keys 164 | ptc.keysLookup() 165 | return ptc 166 | } 167 | 168 | func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { 169 | // crude json like builder from key/val tags 170 | // since we are only ever dealing with a string input 171 | // extracted from the token there is little chance panic would occur here 172 | // WATCH THIS SPACE "¯\_(ツ)_/¯" 173 | metaMap := []string{} 174 | for keyVal := range strings.SplitSeq(t.metadataStr, ",") { 175 | mapKeyVal := strings.Split(keyVal, "=") 176 | if len(mapKeyVal) == 2 { 177 | metaMap = append(metaMap, fmt.Sprintf(`"%s":"%s"`, mapKeyVal[0], mapKeyVal[1])) 178 | } 179 | } 180 | 181 | // empty map will be parsed as `{}` still resulting in a valid json 182 | // and successful unmarshalling but default value pointer struct 183 | if err := json.Unmarshal(fmt.Appendf(nil, `{%s}`, strings.Join(metaMap, ",")), metadataTyp); err != nil { 184 | // It would very hard to test this since 185 | // we are forcing the key and value to be strings 186 | // return non-filled pointer 187 | return err 188 | } 189 | return nil 190 | } 191 | 192 | func (t *ParsedTokenConfig) StripPrefix() string { 193 | return t.prefixLessToken 194 | } 195 | 196 | // StripMetadata returns the fullToken without the 197 | // metadata 198 | func (t *ParsedTokenConfig) StripMetadata() string { 199 | return t.metadataLess 200 | } 201 | 202 | // Strip 203 | // 204 | // returns the only the store indicator string 205 | // without any of the configmanager token enrichment: 206 | // 207 | // - metadata 208 | // 209 | // - keySeparator 210 | // 211 | // - keys 212 | // 213 | // - prefix 214 | func (t *ParsedTokenConfig) StoreToken() string { 215 | return t.storeToken 216 | } 217 | 218 | // Full returns the full Token path. 219 | // Including key separator and metadata values 220 | func (t *ParsedTokenConfig) String() string { 221 | return t.fullToken 222 | } 223 | 224 | func (t *ParsedTokenConfig) LookupKeys() string { 225 | return t.keysPath 226 | } 227 | 228 | func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { 229 | return t.prefix 230 | } 231 | 232 | const ( 233 | startMetaStr string = `[` 234 | endMetaStr string = `]` 235 | ) 236 | 237 | // extractMetadataStr returns anything between the start and end 238 | // metadata markers in the token string itself 239 | // returns the token without meta 240 | func (t *ParsedTokenConfig) extractMetadataStr() { 241 | token := t.prefixLessToken 242 | t.metadataLess = token 243 | startIndex := strings.Index(token, startMetaStr) 244 | // token has no startMetaStr 245 | if startIndex == -1 { 246 | return 247 | } 248 | newS := token[startIndex+len(startMetaStr):] 249 | 250 | endIndex := strings.Index(newS, endMetaStr) 251 | // token has no meta end 252 | if endIndex == -1 { 253 | return 254 | } 255 | // metastring extracted 256 | // complete [key=value] has been found 257 | metaString := newS[:endIndex] 258 | t.metadataStr = metaString 259 | // Set Metadataless token 260 | t.metadataLess = strings.ReplaceAll(token, startMetaStr+metaString+endMetaStr, "") 261 | } 262 | 263 | // keysLookup returns the keysLookup path and the string without it 264 | // 265 | // NOTE: metadata was already stripped at this point 266 | func (t *ParsedTokenConfig) keysLookup() { 267 | keysIndex := strings.Index(t.metadataLess, t.keySeparator) 268 | if keysIndex >= 0 { 269 | t.keysPath = t.metadataLess[keysIndex+len(t.keySeparator):] 270 | t.storeToken = t.metadataLess[:keysIndex] 271 | return 272 | } 273 | t.storeToken = t.metadataLess 274 | } 275 | -------------------------------------------------------------------------------- /internal/cmdutils/cmdutils_test.go: -------------------------------------------------------------------------------- 1 | package cmdutils_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 13 | log "github.com/DevLabFoundry/configmanager/v2/internal/log" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 15 | "github.com/DevLabFoundry/configmanager/v2/pkg/generator" 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | type mockCfgMgr struct { 20 | parsedMap generator.ParsedMap 21 | err error 22 | parsedString string 23 | config *config.GenVarsConfig 24 | } 25 | 26 | func (m mockCfgMgr) RetrieveWithInputReplaced(input string) (string, error) { 27 | return m.parsedString, m.err 28 | } 29 | 30 | func (m mockCfgMgr) Retrieve(tokens []string) (generator.ParsedMap, error) { 31 | return m.parsedMap, m.err 32 | } 33 | 34 | func (m mockCfgMgr) GeneratorConfig() *config.GenVarsConfig { 35 | return m.config 36 | } 37 | 38 | func Test_UploadTokens_errors(t *testing.T) { 39 | m := &mockCfgMgr{} 40 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{io.Discard}) 41 | tokenMap := make(map[string]string) 42 | if err := cmd.UploadTokensWithVals(tokenMap); err == nil { 43 | t.Errorf(testutils.TestPhraseWithContext, "NOT YET IMPLEMENTED should fail", nil, "err") 44 | } 45 | } 46 | 47 | func cmdTestHelper(t *testing.T, err error, got []byte, expect []string) { 48 | t.Helper() 49 | if err != nil { 50 | t.Errorf("wanted file to not Error") 51 | } 52 | 53 | if len(got) < 1 { 54 | t.Error("empty file") 55 | } 56 | for _, want := range expect { 57 | if !strings.Contains(string(got), want) { 58 | t.Errorf(testutils.TestPhraseWithContext, "contents not found", string(got), want) 59 | } 60 | } 61 | } 62 | 63 | func Test_GenerateFromCmd(t *testing.T) { 64 | t.Parallel() 65 | 66 | ttests := map[string]struct { 67 | mockMap generator.ParsedMap 68 | tokens []string 69 | expect []string 70 | }{ 71 | "succeeds with 3 tokens": { 72 | generator.ParsedMap{"FOO://bar/qusx": "aksujg", "FOO://bar/lorem": "", "FOO://bar/ducks": "sdhbjk0293"}, 73 | []string{"FOO://bar/qusx", "FOO://bar/lorem", "FOO://bar/ducks"}, 74 | []string{"export QUSX='aksujg'", "export LOREM=''", "export DUCKS='sdhbjk0293'"}, 75 | }, 76 | } 77 | for name, tt := range ttests { 78 | t.Run(name, func(t *testing.T) { 79 | // create a temp file 80 | f, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-token*") 81 | defer os.Remove(f.Name()) 82 | 83 | m := &mockCfgMgr{ 84 | config: config.NewConfig(), 85 | parsedMap: tt.mockMap, 86 | } 87 | 88 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), f) 89 | err := cmd.GenerateFromCmd(tt.tokens) 90 | if err != nil { 91 | t.Fatalf(testutils.TestPhraseWithContext, "generate from cmd tokens", err, nil) 92 | } 93 | 94 | got, err := os.ReadFile(f.Name()) 95 | cmdTestHelper(t, err, got, tt.expect) 96 | }) 97 | } 98 | } 99 | 100 | type mockWriter struct { 101 | w io.Writer 102 | } 103 | 104 | func (m *mockWriter) Write(in []byte) (int, error) { 105 | return m.w.Write(in) 106 | } 107 | 108 | func Test_GenerateStrOut(t *testing.T) { 109 | t.Parallel() 110 | 111 | inputStr := `FOO://bar/qusx FOO://bar/lorem FOO://bar/ducks` 112 | mockParsedStr := `aksujg fooLorem Mighty` 113 | expect := []string{"aksujg", "fooLorem", "Mighty"} 114 | 115 | t.Run("succeeds with input from string and output different", func(t *testing.T) { 116 | tearDown, writer, fileName := func(t *testing.T) (func(), io.WriteCloser, string) { 117 | f, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-string*") 118 | return func() { 119 | f.Close() 120 | os.Remove(f.Name()) 121 | }, f, f.Name() 122 | }(t) 123 | defer tearDown() 124 | 125 | m := &mockCfgMgr{ 126 | config: config.NewConfig(), 127 | parsedString: mockParsedStr, 128 | } 129 | inputReader, _ := cmdutils.GetReader(&cobra.Command{}, inputStr) 130 | // outputWriter, _ := cmdutils.GetWriter(file) 131 | 132 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), writer) 133 | err := cmd.GenerateStrOut(inputReader, false) 134 | if err != nil { 135 | t.Fatalf(testutils.TestPhraseWithContext, "generate from string", err, nil) 136 | } 137 | got, err := os.ReadFile(fileName) 138 | cmdTestHelper(t, err, got, expect) 139 | }) 140 | 141 | t.Run("succeeds output set to stdout", func(t *testing.T) { 142 | m := &mockCfgMgr{ 143 | config: config.NewConfig(), 144 | parsedString: mockParsedStr, 145 | } 146 | inputReader, _ := cmdutils.GetReader(&cobra.Command{}, inputStr) 147 | outputWriter := &bytes.Buffer{} 148 | mw := &mockWriter{w: outputWriter} 149 | 150 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{mw}) 151 | err := cmd.GenerateStrOut(inputReader, false) 152 | if err != nil { 153 | t.Fatalf(testutils.TestPhraseWithContext, "generate from string", err, nil) 154 | } 155 | got, err := io.ReadAll(outputWriter) 156 | cmdTestHelper(t, err, got, expect) 157 | }) 158 | t.Run("succeeds input and output are set to file names", func(t *testing.T) { 159 | inputF, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-string*") 160 | inputF.Write([]byte(inputStr)) 161 | outputF, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-string*") 162 | 163 | defer func() { 164 | os.Remove(inputF.Name()) 165 | os.Remove(outputF.Name()) 166 | }() 167 | 168 | m := &mockCfgMgr{ 169 | config: config.NewConfig(), 170 | parsedString: mockParsedStr, 171 | } 172 | 173 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), outputF) 174 | err := cmd.GenerateStrOut(inputF, false) 175 | if err != nil { 176 | t.Fatalf(testutils.TestPhraseWithContext, "generate from string", err, nil) 177 | } 178 | got, err := os.ReadFile(outputF.Name()) 179 | cmdTestHelper(t, err, got, expect) 180 | }) 181 | 182 | t.Run("succeeds input and output are set to the same file", func(t *testing.T) { 183 | inputF, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-string*") 184 | inputF.Write([]byte(inputStr)) 185 | defer func() { 186 | os.Remove(inputF.Name()) 187 | }() 188 | 189 | m := &mockCfgMgr{ 190 | config: config.NewConfig().WithOutputPath(inputF.Name()), 191 | parsedString: mockParsedStr, 192 | } 193 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), inputF) 194 | err := cmd.GenerateStrOut(inputF, true) 195 | if err != nil { 196 | t.Fatalf(testutils.TestPhraseWithContext, "generate from string", err, nil) 197 | } 198 | got, err := os.ReadFile(inputF.Name()) 199 | cmdTestHelper(t, err, got, expect) 200 | }) 201 | } 202 | 203 | func Test_CmdUtils_Errors_on(t *testing.T) { 204 | t.Run("outputFile wrong", func(t *testing.T) { 205 | _, err := cmdutils.GetWriter("xunknown/file") 206 | if err == nil { 207 | t.Fatal("error not caught") 208 | } 209 | }) 210 | t.Run("REtrieve from tokens in fetching ANY of the tokens", func(t *testing.T) { 211 | m := &mockCfgMgr{ 212 | config: config.NewConfig(), 213 | parsedMap: generator.ParsedMap{}, 214 | err: fmt.Errorf("err in fetching tokens"), 215 | } 216 | 217 | writer := bytes.NewBuffer([]byte{}) 218 | mw := &mockWriter{w: writer} 219 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{mw}) 220 | if err := cmd.GenerateFromCmd([]string{"IMNP://foo"}); err == nil { 221 | t.Errorf(testutils.TestPhraseWithContext, "NOT fetching ANY tokens should error", "err", nil) 222 | } 223 | }) 224 | 225 | t.Run("REtrieve from tokens in fetching SOME of the tokens", func(t *testing.T) { 226 | m := &mockCfgMgr{ 227 | config: config.NewConfig(), 228 | parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, 229 | err: fmt.Errorf("err in fetching tokens"), 230 | } 231 | 232 | writer := bytes.NewBuffer([]byte{}) 233 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{&mockWriter{w: writer}}) 234 | if err := cmd.GenerateFromCmd([]string{"IMNP://foo", "IMNP://foo2"}); err != nil { 235 | t.Errorf(testutils.TestPhraseWithContext, "fetching tokens some erroring should only be logged out", "err", nil) 236 | } 237 | }) 238 | 239 | t.Run("REtrieve from string in fetching SOME of the tokens", func(t *testing.T) { 240 | m := &mockCfgMgr{ 241 | config: config.NewConfig().WithOutputPath("stdout"), 242 | parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, 243 | parsedString: `bar `, 244 | err: fmt.Errorf("err in fetching tokens"), 245 | } 246 | 247 | inputReader, _ := cmdutils.GetReader(&cobra.Command{}, `"IMNP://foo", "IMNP://foo2"`) 248 | 249 | writer := bytes.NewBuffer([]byte{}) 250 | mw := &mockWriter{w: writer} 251 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{mw}) 252 | if err := cmd.GenerateStrOut(inputReader, false); err == nil { 253 | t.Errorf(testutils.TestPhraseWithContext, "fetching tokens some erroring should only be logged out", nil, "err") 254 | } 255 | }) 256 | 257 | t.Run("REtrieve from string in fetching SOME of the tokens with input/output the same", func(t *testing.T) { 258 | inputF, _ := os.CreateTemp(os.TempDir(), "gen-conf-frrom-string*") 259 | inputF.Write([]byte(`"IMNP://foo", "IMNP://foo2"`)) 260 | defer func() { 261 | os.Remove(inputF.Name()) 262 | }() 263 | 264 | m := &mockCfgMgr{ 265 | config: config.NewConfig().WithOutputPath(inputF.Name()), 266 | parsedString: `bar `, 267 | err: fmt.Errorf("err in fetching tokens"), 268 | } 269 | 270 | writer := bytes.NewBuffer([]byte{}) 271 | mw := &mockWriter{w: writer} 272 | cmd := cmdutils.New(m, log.New(&bytes.Buffer{}), &cmdutils.WriterCloserWrapper{mw}) 273 | if err := cmd.GenerateStrOut(inputF, true); err == nil { 274 | t.Errorf(testutils.TestPhraseWithContext, "fetching tokens some erroring should only be logged out", nil, "err") 275 | } 276 | }) 277 | } 278 | -------------------------------------------------------------------------------- /internal/strategy/strategy_test.go: -------------------------------------------------------------------------------- 1 | package strategy_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 11 | log "github.com/DevLabFoundry/configmanager/v2/internal/log" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/store" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/strategy" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 15 | "github.com/go-test/deep" 16 | ) 17 | 18 | type mockGenerate struct { 19 | inToken, value string 20 | err error 21 | } 22 | 23 | func (m mockGenerate) SetToken(s *config.ParsedTokenConfig) { 24 | } 25 | 26 | func (m mockGenerate) Token() (s string, e error) { 27 | return m.value, m.err 28 | } 29 | 30 | var TEST_GCP_CREDS = []byte(`{ 31 | "type": "service_account", 32 | "project_id": "xxxxx", 33 | "private_key_id": "yyyyyyyyyyyy", 34 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDf842hcn5Nvp6e\n7yKARaCVIDfLXpKDhRwUOvHMzJ1ioRgQo/kbv1n4yHGCSUFyY6hKGj0HBjaGj5kE\n79H/6Y3dJNGhnsMnxBhHdo+3FI8QF0CHZh460NMZSAJ41UMQSBGssGVsNfyUzXGH\nLc45sIx/Twx3yr1k2GD3E8FlDcKlZqa3xGHf+aipg2X3NxbYi+Sz7Yed+SOMhNHl\ncX6E/TqG9n1aTyIwjMIHscCYarJqURkJxr24ukDroCeMxAfxYTdMvRU2e8pFEdoY\nrgUC88fYfaVI5txJ6j/ZKauKQX9Pa8tSyXJeGva3JYp4VC7V4IyoVviCUgEGWZDN\n6/i3zoF/AgMBAAECggEAcVBCcVYFIkE48SH+Svjv74SFtpj7eSB4vKO2hPFjEOyB\nyKmu+aMwWvjQtiNqwf46wIPWLR+vpxYxTpYpo1sBNMvUZfp2tEA8KKyMuw3j9ThO\npjO9R/UxWrFcztbZP/u3NbFrH/2Q95mbv9IlbnsuG5xbqqEig0wYg+uzBvaXbig3\n/Jr0vLT2BkRCBKQkYGjVZcHlHVLoF7/J8cghFgkV1PGvknOv6/q7qzn9L4TjQIet\nfhrhN8Z1vgFiSYtpjP6YQEUEPSHmCQeD3WzJcnASPpU2uCUwd/z65ltKPnn+rqMt\n6jt9R1S1Ju2ZSjv+kR5fIXzihdOzncyzDDm33c/QwQKBgQD2QDZuzLjTxnhsfGii\nKJDAts+Jqfs/6SeEJcJKtEngj4m7rgzyEjbKVp8qtRHIzglKRWAe62/qzzy2BkKi\nvAd4+ZzmG2SkgypGsKVfjGXVFixz2gtUdmBOmK/TnYsxNT9yTt+rX9IGqKK60q73\nOWl8VsliLIsfvSH7+bqi7sRcXQKBgQDo0VUebyQHoTAXPdzGy2ysrVPDiHcldH0Y\n/hvhQTZwxYaJr3HpOCGol2Xl6zyawuudEQsoQwJ3Li6yeb0YMGiWX77/t+qX3pSn\nkGuoftGaNDV7sLn9UV2y+InF8EL1CasrhG1k5RIuxyfV0w+QUo+E7LpVR5XkbJqT\n9QNKnDQXiwKBgQDvvEYCCqbp7e/xVhEbxbhfFdro4Cat6tRAz+3egrTlvXhO0jzi\nMp9Kz5f3oP5ma0gaGX5hu75icE1fvKqE+d+ghAqe7w5FJzkyRulJI0tEb2jphN7A\n5NoPypBqyZboWjmhlG4mzouPVf/POCuEnk028truDAWJ6by7Lj3oP+HFNQKBgQCc\n5BQ8QiFBkvnZb7LLtGIzq0n7RockEnAK25LmJRAOxs13E2fsBguIlR3x5qgckqY8\nXjPqmd2bet+1HhyzpEuWqkcIBGRum2wJz2T9UxjklbJE/D8Z2i8OYDZX0SUOA8n5\ntXASwduS8lqB2Y1vcHOO3AhlV6xHFnjEpCPnr4PbKQKBgAhQ9D9MPeuz+5yw3yHg\nkvULZRtud+uuaKrOayprN25RTxr9c0erxqnvM7KHeo6/urOXeEa7x2n21kAT0Nch\nkF2RtWBLZKXGZEVBtw1Fw0UKNh4IDgM26dwlzRfTVHCiw6M6dCiTNk9KkP2vlkim\n3QFDSSUp+eBTXA17WkDAQf7w\n-----END PRIVATE KEY-----\n", 35 | "client_email": "foo@project.iam.gserviceaccount.com", 36 | "client_id": "99999911111111", 37 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 38 | "token_uri": "https://oauth2.googleapis.com/token", 39 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 40 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/bla" 41 | }`) 42 | 43 | func Test_Strategy_Retrieve_succeeds(t *testing.T) { 44 | t.Parallel() 45 | ttests := map[string]struct { 46 | impl func(t *testing.T) store.Strategy 47 | config *config.GenVarsConfig 48 | token string 49 | expect string 50 | }{ 51 | "with mocked implementation AZTABLESTORAGE": { 52 | func(t *testing.T) store.Strategy { 53 | return &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} 54 | }, 55 | config.NewConfig().WithOutputPath("stdout").WithTokenSeparator("://"), 56 | "AZTABLESTORE://mountPath/token", 57 | "bar", 58 | }, 59 | // "error in retrieval": { 60 | // func(t *testing.T) store.Strategy { 61 | // return &mockGenerate{"SOME://mountPath/token", "bar", fmt.Errorf("unable to perform getTokenValue")} 62 | // }, 63 | // config.NewConfig().WithOutputPath("stdout").WithTokenSeparator("://"), 64 | // []string{"SOME://token"}, 65 | // config.AzAppConfigPrefix, 66 | // "unable to perform getTokenValue", 67 | // }, 68 | } 69 | for name, tt := range ttests { 70 | t.Run(name, func(t *testing.T) { 71 | rs := strategy.New(*tt.config, log.New(io.Discard)) 72 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 73 | got := rs.RetrieveByToken(context.TODO(), tt.impl(t), token) 74 | if got.Err != nil { 75 | t.Errorf(testutils.TestPhraseWithContext, "Token response errored", got.Err.Error(), tt.expect) 76 | } 77 | if got.Value() != tt.expect { 78 | t.Errorf(testutils.TestPhraseWithContext, "Value not correct", got.Value(), tt.expect) 79 | } 80 | if got.Key().String() != tt.token { 81 | t.Errorf(testutils.TestPhraseWithContext, "INcorrect Token returned in Key", got.Key().String(), tt.token) 82 | } 83 | }) 84 | } 85 | } 86 | 87 | func Test_CustomStrategyFuncMap_add_own(t *testing.T) { 88 | t.Parallel() 89 | 90 | ttests := map[string]struct { 91 | }{ 92 | "default": {}, 93 | } 94 | for name, _ := range ttests { 95 | t.Run(name, func(t *testing.T) { 96 | called := 0 97 | genVarsConf := config.NewConfig() 98 | token, _ := config.NewParsedTokenConfig("AZTABLESTORE://mountPath/token", *genVarsConf) 99 | 100 | var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { 101 | m := &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} 102 | called++ 103 | return m, nil 104 | } 105 | 106 | s := strategy.New(*genVarsConf, log.New(io.Discard), strategy.WithStrategyFuncMap(strategy.StrategyFuncMap{config.AzTableStorePrefix: custFunc})) 107 | 108 | store, _ := s.SelectImplementation(context.TODO(), token) 109 | _ = s.RetrieveByToken(context.TODO(), store, token) 110 | 111 | if called != 1 { 112 | t.Errorf(testutils.TestPhraseWithContext, "custom func not called", called, 1) 113 | } 114 | }) 115 | } 116 | } 117 | 118 | func Test_SelectImpl_With(t *testing.T) { 119 | 120 | ttests := map[string]struct { 121 | setUpTearDown func() func() 122 | token string 123 | config *config.GenVarsConfig 124 | expect func() store.Strategy 125 | expErr error 126 | }{ 127 | "unknown": { 128 | func() func() { 129 | return func() { 130 | } 131 | }, 132 | "UNKNOWN#foo/bar", 133 | config.NewConfig().WithTokenSeparator("#"), 134 | func() store.Strategy { return nil }, 135 | fmt.Errorf("implementation not found for input string: UNKNOWN#foo/bar"), 136 | }, 137 | "success AZTABLESTORE": { 138 | func() func() { 139 | os.Setenv("AZURE_stuff", "foo") 140 | return func() { 141 | os.Clearenv() 142 | } 143 | }, 144 | "AZTABLESTORE#foo/bar1", 145 | config.NewConfig().WithTokenSeparator("#"), 146 | func() store.Strategy { 147 | token, _ := config.NewParsedTokenConfig("AZTABLESTORE#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) 148 | s, _ := store.NewAzTableStore(context.TODO(), token, log.New(io.Discard)) 149 | return s 150 | }, 151 | nil, 152 | }, 153 | "success AWSPARAMSTR": { 154 | func() func() { 155 | os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") 156 | os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") 157 | return func() { 158 | os.Clearenv() 159 | } 160 | }, 161 | "AWSPARAMSTR#foo/bar1", 162 | config.NewConfig().WithTokenSeparator("#"), 163 | func() store.Strategy { 164 | s, _ := store.NewParamStore(context.TODO(), log.New(io.Discard)) 165 | return s 166 | }, 167 | nil, 168 | }, 169 | "success AWSSECRETS": { 170 | func() func() { 171 | os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") 172 | os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") 173 | return func() { 174 | os.Clearenv() 175 | } 176 | }, 177 | "AWSSECRETS#foo/bar1", 178 | config.NewConfig().WithTokenSeparator("#"), 179 | func() store.Strategy { 180 | s, _ := store.NewSecretsMgr(context.TODO(), log.New(io.Discard)) 181 | return s 182 | }, 183 | nil, 184 | }, 185 | "success AZKVSECRET": { 186 | func() func() { 187 | os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") 188 | os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") 189 | return func() { 190 | os.Clearenv() 191 | } 192 | }, 193 | "AZKVSECRET#foo/bar1", 194 | config.NewConfig().WithTokenSeparator("#"), 195 | func() store.Strategy { 196 | token, _ := config.NewParsedTokenConfig("AZKVSECRET#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) 197 | s, _ := store.NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) 198 | return s 199 | }, 200 | nil, 201 | }, 202 | "success AZAPPCONF": { 203 | func() func() { 204 | return func() { 205 | os.Clearenv() 206 | } 207 | }, 208 | "AZAPPCONF#foo/bar1", 209 | config.NewConfig().WithTokenSeparator("#"), 210 | func() store.Strategy { 211 | token, _ := config.NewParsedTokenConfig("AZAPPCONF#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) 212 | s, _ := store.NewAzAppConf(context.TODO(), token, log.New(io.Discard)) 213 | return s 214 | }, 215 | nil, 216 | }, 217 | "success VAULT": { 218 | func() func() { 219 | os.Setenv("VAULT_", "AAAAAAAAAAAAAAA") 220 | return func() { 221 | os.Clearenv() 222 | } 223 | }, 224 | "VAULT#foo/bar1", 225 | config.NewConfig().WithTokenSeparator("#"), 226 | func() store.Strategy { 227 | token, _ := config.NewParsedTokenConfig("VAULT#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) 228 | s, _ := store.NewVaultStore(context.TODO(), token, log.New(io.Discard)) 229 | return s 230 | }, 231 | nil, 232 | }, 233 | "success GCPSECRETS": { 234 | func() func() { 235 | cf, _ := os.CreateTemp(".", "*") 236 | cf.Write(TEST_GCP_CREDS) 237 | os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", cf.Name()) 238 | return func() { 239 | os.Remove(cf.Name()) 240 | os.Clearenv() 241 | } 242 | }, 243 | "GCPSECRETS#foo/bar1", 244 | config.NewConfig().WithTokenSeparator("#"), 245 | func() store.Strategy { 246 | s, _ := store.NewGcpSecrets(context.TODO(), log.New(io.Discard)) 247 | return s 248 | }, 249 | nil, 250 | }, 251 | // "default Error": { 252 | // func() func() { 253 | // os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") 254 | // os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") 255 | // return func() { 256 | // os.Clearenv() 257 | // } 258 | // }, 259 | // context.TODO(), 260 | // UnknownPrefix, "AWSPARAMSTR://foo/bar", (&GenVarsConfig{}).WithKeySeparator("|").WithTokenSeparator("://"), 261 | // func(t *testing.T, ctx context.Context, conf GenVarsConfig) genVarsStrategy { 262 | // imp, err := NewParamStore(ctx) 263 | // if err != nil { 264 | // t.Errorf(testutils.TestPhraseWithContext, "init impl error", err.Error(), nil) 265 | // } 266 | // return imp 267 | // }, 268 | // }, 269 | } 270 | for name, tt := range ttests { 271 | t.Run(name, func(t *testing.T) { 272 | tearDown := tt.setUpTearDown() 273 | defer tearDown() 274 | want := tt.expect() 275 | rs := strategy.New(*tt.config, log.New(io.Discard)) 276 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 277 | got, err := rs.SelectImplementation(context.TODO(), token) 278 | 279 | if err != nil { 280 | if err.Error() != tt.expErr.Error() { 281 | t.Errorf(testutils.TestPhraseWithContext, "uncaught error", err.Error(), tt.expErr.Error()) 282 | } 283 | return 284 | } 285 | 286 | diff := deep.Equal(got, want) 287 | if diff != nil { 288 | t.Errorf(testutils.TestPhraseWithContext, "reflection of initialised implentations", fmt.Sprintf("%q", got), fmt.Sprintf("%q", want)) 289 | } 290 | }) 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /internal/store/aztablestorage_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" 12 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 13 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 15 | ) 16 | 17 | func azTableStoreCommonChecker(t *testing.T, partitionKey, rowKey, expectedPartitionKey, expectedRowKey string) { 18 | t.Helper() 19 | if partitionKey == "" { 20 | t.Errorf("expect name to not be nil") 21 | } 22 | if partitionKey != expectedPartitionKey { 23 | t.Errorf(testutils.TestPhrase, partitionKey, expectedPartitionKey) 24 | } 25 | 26 | if strings.Contains(partitionKey, string(config.AzTableStorePrefix)) { 27 | t.Errorf("incorrectly stripped prefix") 28 | } 29 | 30 | if rowKey != expectedRowKey { 31 | t.Errorf(testutils.TestPhrase, rowKey, expectedPartitionKey) 32 | } 33 | } 34 | 35 | type mockAzTableStoreApi func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) 36 | 37 | func (m mockAzTableStoreApi) GetEntity(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 38 | return m(ctx, partitionKey, rowKey, options) 39 | } 40 | 41 | func Test_AzTableStore_Success(t *testing.T) { 42 | 43 | tests := map[string]struct { 44 | token string 45 | expect string 46 | mockClient func(t *testing.T) tableStoreApi 47 | config *config.GenVarsConfig 48 | }{ 49 | "successVal": {"AZTABLESTORE#/test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { 50 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 51 | t.Helper() 52 | azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") 53 | resp := aztables.GetEntityResponse{} 54 | resp.Value = []byte("tsuccessParam") 55 | return resp, nil 56 | }) 57 | }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 58 | }, 59 | "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { 60 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 61 | t.Helper() 62 | azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") 63 | resp := aztables.GetEntityResponse{} 64 | resp.Value = []byte("tsuccessParam") 65 | return resp, nil 66 | }) 67 | }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), 68 | }, 69 | "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { 70 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 71 | t.Helper() 72 | azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") 73 | 74 | resp := aztables.GetEntityResponse{} 75 | resp.Value = nil 76 | return resp, nil 77 | }) 78 | }, 79 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 80 | }, 81 | } 82 | 83 | for name, tt := range tests { 84 | t.Run(name, func(t *testing.T) { 85 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 86 | impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) 87 | if err != nil { 88 | t.Errorf("failed to init aztablestore") 89 | } 90 | 91 | impl.svc = tt.mockClient(t) 92 | got, err := impl.Token() 93 | if err != nil { 94 | if err.Error() != tt.expect { 95 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 96 | } 97 | return 98 | } 99 | 100 | if got != tt.expect { 101 | t.Errorf(testutils.TestPhrase, got, tt.expect) 102 | } 103 | }) 104 | } 105 | } 106 | 107 | func Test_azstorage_with_value_property(t *testing.T) { 108 | t.Parallel() 109 | conf := config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://") 110 | ttests := map[string]struct { 111 | token string 112 | expect string 113 | mockClient func(t *testing.T) tableStoreApi 114 | config *config.GenVarsConfig 115 | }{ 116 | "return value property with json like object": { 117 | "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", 118 | "map[bool:true host:foo port:1234]", 119 | func(t *testing.T) tableStoreApi { 120 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 121 | t.Helper() 122 | resp := aztables.GetEntityResponse{Value: []byte(`{"value":{"host":"foo","port":1234,"bool":true}}`)} 123 | return resp, nil 124 | }) 125 | }, 126 | conf, 127 | }, 128 | "return value property with string only": { 129 | "AZTABLESTORE:///test-account/table/partitionkey/rowKey", 130 | "foo.bar.com", 131 | func(t *testing.T) tableStoreApi { 132 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 133 | t.Helper() 134 | resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} 135 | return resp, nil 136 | }) 137 | }, 138 | conf, 139 | }, 140 | "return value property with numeric only": { 141 | "AZTABLESTORE:///test-account/table/partitionkey/rowKey", 142 | "1234", 143 | func(t *testing.T) tableStoreApi { 144 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 145 | t.Helper() 146 | resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} 147 | return resp, nil 148 | }) 149 | }, 150 | conf, 151 | }, 152 | "return value property with boolean only": { 153 | "AZTABLESTORE:///test-account/table/partitionkey/rowKey", 154 | "false", 155 | func(t *testing.T) tableStoreApi { 156 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 157 | t.Helper() 158 | resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} 159 | return resp, nil 160 | }) 161 | }, 162 | conf, 163 | }, 164 | } 165 | for name, tt := range ttests { 166 | t.Run(name, func(t *testing.T) { 167 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 168 | 169 | impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) 170 | if err != nil { 171 | t.Fatal("failed to init aztablestore") 172 | } 173 | 174 | impl.svc = tt.mockClient(t) 175 | 176 | got, err := impl.Token() 177 | if err != nil { 178 | t.Fatalf(testutils.TestPhrase, err.Error(), nil) 179 | } 180 | 181 | if got != tt.expect { 182 | t.Errorf(testutils.TestPhraseWithContext, "AZ Table storage with value property inside entity", fmt.Sprintf("%q", got), fmt.Sprintf("%q", tt.expect)) 183 | } 184 | }) 185 | } 186 | } 187 | 188 | func Test_AzTableStore_Error(t *testing.T) { 189 | t.Parallel() 190 | 191 | tests := map[string]struct { 192 | token string 193 | expect error 194 | mockClient func(t *testing.T) tableStoreApi 195 | config *config.GenVarsConfig 196 | }{ 197 | "errored on token parsing to partiationKey": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { 198 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 199 | t.Helper() 200 | resp := aztables.GetEntityResponse{} 201 | return resp, nil 202 | }) 203 | }, 204 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 205 | }, 206 | "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { 207 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 208 | t.Helper() 209 | resp := aztables.GetEntityResponse{} 210 | return resp, fmt.Errorf("network error") 211 | }) 212 | }, 213 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 214 | }, 215 | 216 | "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { 217 | return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { 218 | t.Helper() 219 | resp := aztables.GetEntityResponse{} 220 | return resp, nil 221 | }) 222 | }, 223 | config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), 224 | }, 225 | } 226 | 227 | for name, tt := range tests { 228 | t.Run(name, func(t *testing.T) { 229 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) 230 | 231 | impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) 232 | if err != nil { 233 | t.Fatal("failed to init aztablestore") 234 | } 235 | 236 | impl.svc = tt.mockClient(t) 237 | if _, err := impl.Token(); !errors.Is(err, tt.expect) { 238 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 239 | } 240 | }) 241 | } 242 | } 243 | 244 | func Test_fail_AzTable_Client_init(t *testing.T) { 245 | // this is basically a wrap around test for the url.Parse method in the stdlib 246 | // as that is what the client uses under the hood 247 | token, _ := config.NewParsedTokenConfig("AZTABLESTORE:///%25%65%6e%301-._~/") 252 | } 253 | if !errors.Is(err, ErrClientInitialization) { 254 | t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), ErrClientInitialization.Error()) 255 | } 256 | } 257 | 258 | func Test_azSplitTokenTableStore(t *testing.T) { 259 | t.Parallel() 260 | 261 | tests := []struct { 262 | name string 263 | token string 264 | expect azServiceHelper 265 | }{ 266 | { 267 | name: "simple_with_preceding_slash", 268 | token: "/test-account/tablename/somejsontest", 269 | expect: azServiceHelper{ 270 | serviceUri: "https://test-account.table.core.windows.net/tablename", 271 | token: "somejsontest", 272 | }, 273 | }, 274 | { 275 | name: "missing_initial_slash", 276 | token: "test-account/tablename/somejsontest", 277 | expect: azServiceHelper{ 278 | serviceUri: "https://test-account.table.core.windows.net/tablename", 279 | token: "somejsontest", 280 | }, 281 | }, 282 | { 283 | name: "missing_initial_slash_multislash_secretname", 284 | token: "test-account/tablename/some/json/test", 285 | expect: azServiceHelper{ 286 | serviceUri: "https://test-account.table.core.windows.net/tablename", 287 | token: "some/json/test", 288 | }, 289 | }, 290 | { 291 | name: "with_initial_slash_multislash_secretname", 292 | token: "test-account/tablename//some/json/test", 293 | expect: azServiceHelper{ 294 | serviceUri: "https://test-account.table.core.windows.net/tablename", 295 | token: "/some/json/test", 296 | }, 297 | }, 298 | } 299 | for _, tt := range tests { 300 | t.Run(tt.name, func(t *testing.T) { 301 | got := azServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) 302 | if got.token != tt.expect.token { 303 | t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) 304 | } 305 | if got.serviceUri != tt.expect.serviceUri { 306 | t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) 307 | } 308 | }) 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Config Manager 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/DevLabFoundry/configmanager/v2.svg)](https://pkg.go.dev/github.com/DevLabFoundry/configmanager/v2) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/DevLabFoundry/configmanager/v2)](https://goreportcard.com/report/github.com/DevLabFoundry/configmanager/v2) 5 | [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=dnitsch_configmanager&metric=bugs)](https://sonarcloud.io/summary/new_code?id=dnitsch_configmanager) 6 | [![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=dnitsch_configmanager&metric=sqale_index)](https://sonarcloud.io/summary/new_code?id=dnitsch_configmanager) 7 | [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=dnitsch_configmanager&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=dnitsch_configmanager) 8 | [![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=dnitsch_configmanager&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=dnitsch_configmanager) 9 | [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=dnitsch_configmanager&metric=coverage)](https://sonarcloud.io/summary/new_code?id=dnitsch_configmanager) 10 | 11 | Package used for retrieving application settings from various sources. 12 | 13 | Currently supported variable and secrets implementations: 14 | 24 | 25 | - [AWS SecretsManager](https://aws.amazon.com/secrets-manager/) 26 | - Implementation Indicator: `AWSSECRETS` 27 | - [AWS ParameterStore](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) 28 | - Implementation Indicator: `AWSPARAMSTR` 29 | - [AzureKeyvault Secrets](https://azure.microsoft.com/en-gb/products/key-vault/) 30 | - Implementation Indicator: `AZKVSECRET` 31 | - see [Special consideration for AZKVSECRET](#special-consideration-for-azkvsecret) around how to structure the token in this case. 32 | - [Azure TableStorage](https://azure.microsoft.com/en-gb/products/storage/tables/) 33 | - Implementation Indicator: `AZTABLESTORE` 34 | - see [Special consideration for AZTABLESTORE](#special-consideration-for-aztablestore) around how to structure the token in this case. 35 | - [Azure AppConfig](https://azure.microsoft.com/en-gb/products/app-configuration/) 36 | - Implementation Indicator: `AZAPPCONF` 37 | - [GCP Secrets](https://cloud.google.com/secret-manager) 38 | - Implementation Indicator: `GCPSECRETS` 39 | - [Hashicorp Vault](https://developer.hashicorp.com/vault/docs/secrets/kv) 40 | - Implementation Indicator: `VAULT` 41 | - using the KvV2 engine endpoint 42 | - see [special consideration hashivault](#special-consideration-for-hashicorpvault) 43 | 44 | The main driver is to use component level configuration objects, if stored in a `"namespaced"` manner e.g. in AWS ParamStore as `/nonprod/component-service-a/configVar`, however this is not a requirement and the param name can be whatever. Though whilst using some sort of a organised manner it will be more straight forward to allow other services to consume certain secrets/params based on resource/access policies. 45 | 46 | > Beware size limitation with certain config/vault implementations. In which case it's best to split certain items up e.g. TLS certs `/nonprod/component-service-a/pub-cert`, `/nonprod/component-service-a/private-cert`, `/nonprod/component-service-a/chain1-cert`, etc... 47 | 48 | Where `configVar` can be either a parseable string `'som3#!S$CRet'` or a number `3306` or a parseable single level JSON object like `{host: ..., pass: ...., port: ...}` which can be returned whole or accessed via a key separator for a specific value. 49 | 50 | ## Use cases 51 | 52 | - Go API 53 | 54 | This can be leveraged from any application written in Go - on start up or at runtime. Secrets/Config items can be retrieved in "bulk" and parsed into a provided type, [see here for examples](./examples/examples.go). 55 | 56 | > BREAKING CHANGE v2.x with the API (see [examples](./examples/examples.go)) 57 | - `generator.NewConfig()` is no longer required. 58 | 59 | ```go 60 | // initialise new configmanager instance 61 | cm := configmanager.New(context.TODO()) 62 | // add additional config to apply on your tokens 63 | cm.Config.WithTokenSeparator("://") 64 | pm, err := cm.Retrieve([]string{"IMPLEMENTATION://token1", "IMPLEMENTATION://token2", "ANOTHER_IMPL://token1"}) 65 | ``` 66 | 67 | - `RetrieveUnmarshalledFromYaml`|`RetrieveUnmarshalledFromJson`|`RetrieveMarshalledJson`|`RetrieveMarshalledYaml` methods are now on the ConfigManager struct, see `exampleRetrieveYamlMarshalled` or `exampleRetrieveYamlUnmarshalled` in [examples](./examples/examples.go) 68 | 69 | - Kubernetes 70 | 71 | Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS#/$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases. 72 | 73 | - VMs 74 | 75 | VM deployments can function in a similar manner by passing in the contents or a path to the source config and the output path so that app at startup time can consume it. 76 | 77 | ## CLI 78 | 79 | ConfigManager comes packaged as a CLI for all major platforms, to see [download/installation](./docs/installation.md) 80 | 81 | For more detailed usage you can run -h with each subcommand and additional info can be found [here](./docs/commands.md) 82 | 83 | ## __Token Config__ 84 | 85 | The token is made up of the following parts: 86 | 87 | _An example token would look like this_ 88 | 89 | #### `AWSSECRETS#/path/to/my/key|lookup.Inside.Object[meta=data]` 90 | 91 | ### Implementation indicator 92 | 93 | The `AWSSECRETS` the strategy identifier to choose the correct provider at runtime. Multiple providers can be referenced in a single run via a CLI or with the API. 94 | 95 | This is not overrideable and must be exactly as it is in the provided list of providers. 96 | 97 | ### __Token Separator__ 98 | 99 | The `#` symbol from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) - used for separating the implementation indicator and the look up value. 100 | 101 | > The default is currently `#` - it will change to `://` to allow for a more natural reading of the "token". you can achieve this behaviour now by either specifying the `-s` to the CLI or ConfigManager Go API. 102 | 103 | ```go 104 | cnf := generator.NewConfig().WithTokenSeparator("://") 105 | ``` 106 | 107 | ### __Provider Secret/Config Path__ 108 | 109 | The `/path/to/my/key` part from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) is the actual path to the item in the backing store. 110 | 111 | See the different special considerations per provider as it different providers will require different implementations. 112 | 113 | ### __Key Separator__ 114 | 115 | __THIS IS OPTIONAL__ 116 | 117 | The `|` symbol from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) is used to specify the key seperator. 118 | 119 | If an item retrieved from a store is JSON parseable map it can be interrogated for further properties inside. 120 | 121 | ### __Look up key__ 122 | 123 | __THIS IS OPTIONAL__ 124 | 125 | The `lookup.Inside.Object` from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) is used to perform a lookup inside the retrieved item IF it is parseable into a `map[string]any` structure. 126 | 127 | Given the below response from a backing store 128 | 129 | ```json 130 | { 131 | "lookup": { 132 | "Inside": { 133 | "Object": { 134 | "host": "db.internal", 135 | "port": 3306, 136 | "pass": "sUp3$ecr3T!", 137 | } 138 | } 139 | } 140 | } 141 | ``` 142 | 143 | The value returned for the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) would be: 144 | 145 | ```json 146 | { 147 | "host": "db.internal", 148 | "port": 3306, 149 | "pass": "sUp3$ecr3T!", 150 | } 151 | ``` 152 | 153 | See [examples of working with files](docs/examples.md#working-with-files) for more details. 154 | 155 | ### Token Metadata Config 156 | 157 | The `[meta=data]` from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) - is the optional metadata about the target in the backing provider 158 | 159 | IT must have this format `[key=value]` - IT IS OPTIONAL 160 | 161 | The `key` and `value` would be provider specific. Meaning that different providers support different config, these values _CAN_ be safely omitted configmanager would just use the defaults where applicable or not specify the additional 162 | 163 | - Hashicorp Vault (VAULT) 164 | - `iam_role` - would be the value of an IAM role ARN to use with AWSClient Authentication. 165 | - `version` - is the version of the secret/configitem to get (should be in an integer format) 166 | 167 | e.g. `VAULT://baz/bar/123|d88[role=arn:aws:iam::1111111:role/i-orchestration,version=1082313]` 168 | 169 | - Azure AppConfig (AZAPPCONF) 170 | - `label` - the label to use whilst retrieving the item 171 | - `etag` - etag value 172 | 173 | e.g. `AZAPPCONF://baz/bar/123|d88[label=dev,etag=aaaaa1082313]` 174 | 175 | - GCP secrets, AWS SEcrets, AZ KeyVault (`GCPSECRETS` , `AWSSECRETS`, `AZKVSECRET`) 176 | they all support the `version` metadata property 177 | 178 | e.g. `GCPSECRETS://baz/bar/123|d88[version=verUUID0000-1123zss]` 179 | 180 | ## Special considerations 181 | 182 | This section outlines the special consideration in token construction on a per provider basis 183 | 184 | ### Special consideration for AZKVSECRET 185 | 186 | For Azure KeyVault the first part of the token needs to be the name of the vault. 187 | 188 | > Azure Go SDK (v2) requires the vault Uri on initializing the client 189 | 190 | `AZKVSECRET#/test-vault//token/1` ==> will use KeyVault implementation to retrieve the `/token/1` from a `test-vault`. 191 | 192 | `AZKVSECRET#/test-vault/no-slash-token-1` ==> will use KeyVault implementation to retrieve the `no-slash-token-1` from a `test-vault`. 193 | 194 | > The preceeding slash to the vault name is optional - `AZKVSECRET#/test-vault/no-slash-token-1` and `AZKVSECRET#test-vault/no-slash-token-1` will both identify the vault of name `test-vault` 195 | 196 | ### Special consideration for AZTABLESTORE 197 | 198 | The token itself must contain all of the following properties, so that it would look like this `AZTABLESTORE://STORAGE_ACCOUNT_NAME/TABLE_NAME/PARTITION_KEY/ROW_KEY`: 199 | 200 | - Storage account name [`STORAGE_ACCOUNT_NAME`] 201 | - Table Name [`TABLE_NAME`] 202 | - > It might make sense to make this table global to the domain or project 203 | - Partition Key [`PARTITION_KEY`] 204 | - > This could correspond to the component/service name 205 | - Row Key [`ROW_KEY`] 206 | - > This could correspond to the property itself or a group of properties 207 | - > e.g. `AZTABLESTORE://globalconfigstorageaccount/domainXyz/serviceXyz/db` => `{"value":{"host":"foo","port":1234,"enabled":true}}` 208 | - > It will continue to work the same way with additional keyseparators inside values. 209 | 210 | > NOTE: if you store a more complex object inside a top level `value` property this will reduce the number of columns and normalize the table - **THE DATA INSIDE THE VALUE MUST BE JSON PARSEABLE** 211 | 212 | All the usual token rules apply e.g. of `keySeparator` 213 | 214 | `AZTABLESTORE://account/app1Config/db/config` => `{host: foo.bar, port: 8891}` 215 | 216 | `AZTABLESTORE://account/app1Config/db/config|host` => `foo.bar` 217 | 218 | ### Special consideration for HashicorpVault 219 | 220 | For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`, 221 | e.g.: `VAULT://secret___demo/configmanager|test` 222 | 223 | or if the secrets are at another location: `VAULT://another/mount/path__config/app1/db` 224 | 225 | The hardcoded separator cannot be modified and you must separate your `mountPath` with `___` (3x `_`) followed by the key to the secret. 226 | 227 | #### AWS IAM auth to vault 228 | 229 | when using Vault in AWS - you can set the value of the `VAULT_TOKEN=aws_iam` this will trigger the AWS Auth login as opposed to using the local token. 230 | 231 | The Hashicorp Vault functions in the same exact way as the other implementations. It will retrieve the JSON object and can be looked up within it by using a key separator. 232 | 233 | ## [Go API](https://pkg.go.dev/github.com/DevLabFoundry/configmanager/v2) 234 | 235 | ## [Examples](docs/examples.md) 236 | 237 | ## Help 238 | 239 | - More implementations should be easily added with a specific implementation under the strategy interface 240 | - see [add additional providers](docs/adding-provider.md) 241 | 242 | - maybe run as cron in the background to perform a periodic sync in case values change? 243 | -------------------------------------------------------------------------------- /internal/store/hashivault_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/DevLabFoundry/configmanager/v2/internal/config" 14 | "github.com/DevLabFoundry/configmanager/v2/internal/log" 15 | "github.com/DevLabFoundry/configmanager/v2/internal/testutils" 16 | vault "github.com/hashicorp/vault/api" 17 | ) 18 | 19 | func TestMountPathExtract(t *testing.T) { 20 | ttests := map[string]struct { 21 | token string 22 | tokenSeparator string 23 | keySeparator string 24 | expect string 25 | }{ 26 | "without leading slash": {"VAULT://secret___/demo/configmanager", "://", "|", "secret"}, 27 | "with leading slash": {"VAULT:///secret___/demo/configmanager", "://", "|", "secret"}, 28 | "with underscore in path name": {"VAULT://_secret___/demo/configmanager", "://", "|", "_secret"}, 29 | "with double underscore in path name": {"VAULT://__secret___/demo/configmanager", "://", "|", "__secret"}, 30 | "with multiple paths in mountpath": {"VAULT://secret/bar/path___/demo/configmanager", "://", "|", "secret/bar/path"}, 31 | } 32 | for name, tt := range ttests { 33 | t.Run(name, func(t *testing.T) { 34 | token, _ := config.NewParsedTokenConfig(tt.token, *config.NewConfig().WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) 35 | got := splitToken(token.StoreToken()) 36 | if got.path != tt.expect { 37 | t.Errorf("got %q, expected %q", got, tt.expect) 38 | } 39 | }) 40 | } 41 | } 42 | 43 | type mockVaultApi struct { 44 | g func(ctx context.Context, secretPath string) (*vault.KVSecret, error) 45 | gv func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) 46 | } 47 | 48 | func (m mockVaultApi) Get(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 49 | return m.g(ctx, secretPath) 50 | } 51 | 52 | func (m mockVaultApi) GetVersion(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { 53 | return m.gv(ctx, secretPath, version) 54 | } 55 | 56 | func TestVaultScenarios(t *testing.T) { 57 | t.Parallel() 58 | 59 | ttests := map[string]struct { 60 | token string 61 | conf *config.GenVarsConfig 62 | expect string 63 | mockClient func(t *testing.T) hashiVaultApi 64 | setupEnv func() func() 65 | }{ 66 | "happy return": {"VAULT://secret___/foo", config.NewConfig(), `{"foo":"test2130-9sd-0ds"}`, 67 | func(t *testing.T) hashiVaultApi { 68 | mv := mockVaultApi{} 69 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 70 | t.Helper() 71 | if secretPath != "foo" { 72 | t.Errorf("got %v; want %s", secretPath, `foo`) 73 | } 74 | m := make(map[string]interface{}) 75 | m["foo"] = "test2130-9sd-0ds" 76 | return &vault.KVSecret{Data: m}, nil 77 | } 78 | return mv 79 | }, 80 | func() func() { 81 | os.Setenv("VAULT_TOKEN", "129378y1231283") 82 | return func() { 83 | os.Clearenv() 84 | } 85 | }, 86 | }, 87 | "incorrect json": {"VAULT://secret___/foo", config.NewConfig(), `json: unsupported type: func() error`, 88 | func(t *testing.T) hashiVaultApi { 89 | mv := mockVaultApi{} 90 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 91 | t.Helper() 92 | if secretPath != "foo" { 93 | t.Errorf("got %v; want %s", secretPath, `foo`) 94 | } 95 | m := make(map[string]interface{}) 96 | m["error"] = func() error { return fmt.Errorf("ddodod") } 97 | return &vault.KVSecret{Data: m}, nil 98 | } 99 | return mv 100 | }, 101 | func() func() { 102 | os.Setenv("VAULT_TOKEN", "129378y1231283") 103 | return func() { 104 | os.Clearenv() 105 | } 106 | }, 107 | }, 108 | "another return": { 109 | "VAULT://secret/engine1___/some/other/foo2", 110 | config.NewConfig(), 111 | `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, 112 | func(t *testing.T) hashiVaultApi { 113 | mv := mockVaultApi{} 114 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 115 | t.Helper() 116 | if secretPath != "some/other/foo2" { 117 | t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) 118 | } 119 | m := make(map[string]interface{}) 120 | m["foo1"] = "test2130-9sd-0ds" 121 | m["foo2"] = "dsfsdf3454456" 122 | return &vault.KVSecret{Data: m}, nil 123 | } 124 | return mv 125 | }, 126 | func() func() { 127 | os.Setenv("VAULT_TOKEN", "129378y1231283") 128 | return func() { 129 | os.Clearenv() 130 | } 131 | }, 132 | }, 133 | "not found": {"VAULT://secret___/foo", config.NewConfig(), `secret not found`, 134 | func(t *testing.T) hashiVaultApi { 135 | mv := mockVaultApi{} 136 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 137 | t.Helper() 138 | if secretPath != "foo" { 139 | t.Errorf("got %v; want %s", secretPath, `foo`) 140 | } 141 | return nil, fmt.Errorf("secret not found") 142 | } 143 | return mv 144 | }, 145 | func() func() { 146 | os.Setenv("VAULT_TOKEN", "129378y1231283") 147 | return func() { 148 | os.Clearenv() 149 | } 150 | }, 151 | }, 152 | "403": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `client 403`, 153 | func(t *testing.T) hashiVaultApi { 154 | mv := mockVaultApi{} 155 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 156 | t.Helper() 157 | if secretPath != "some/other/foo2" { 158 | t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) 159 | } 160 | return nil, fmt.Errorf("client 403") 161 | } 162 | return mv 163 | }, 164 | func() func() { 165 | os.Setenv("VAULT_TOKEN", "129378y1231283") 166 | return func() { 167 | os.Clearenv() 168 | } 169 | }, 170 | }, 171 | "found but empty": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `{}`, func(t *testing.T) hashiVaultApi { 172 | mv := mockVaultApi{} 173 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 174 | t.Helper() 175 | if secretPath != "some/other/foo2" { 176 | t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) 177 | } 178 | m := make(map[string]interface{}) 179 | return &vault.KVSecret{Data: m}, nil 180 | } 181 | return mv 182 | }, 183 | func() func() { 184 | os.Setenv("VAULT_TOKEN", "129378y1231283") 185 | return func() { 186 | os.Clearenv() 187 | } 188 | }, 189 | }, 190 | "found but nil returned": {"VAULT://secret___/some/other/foo2", config.NewConfig(), "", func(t *testing.T) hashiVaultApi { 191 | mv := mockVaultApi{} 192 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 193 | t.Helper() 194 | if secretPath != "some/other/foo2" { 195 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 196 | } 197 | return &vault.KVSecret{Data: nil}, nil 198 | } 199 | return mv 200 | }, 201 | func() func() { 202 | os.Setenv("VAULT_TOKEN", "129378y1231283") 203 | return func() { 204 | os.Clearenv() 205 | } 206 | }, 207 | }, 208 | "version provided correctly": {"VAULT://secret___/some/other/foo2[version=1]", config.NewConfig(), `{"foo2":"dsfsdf3454456"}`, func(t *testing.T) hashiVaultApi { 209 | mv := mockVaultApi{} 210 | mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { 211 | t.Helper() 212 | if secretPath != "some/other/foo2" { 213 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 214 | } 215 | m := make(map[string]interface{}) 216 | m["foo2"] = "dsfsdf3454456" 217 | return &vault.KVSecret{Data: m}, nil 218 | } 219 | return mv 220 | }, 221 | func() func() { 222 | os.Setenv("VAULT_TOKEN", "129378y1231283") 223 | return func() { 224 | os.Clearenv() 225 | } 226 | }, 227 | }, 228 | "version provided but unable to parse": {"VAULT://secret___/some/other/foo2[version=1a]", config.NewConfig(), "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", func(t *testing.T) hashiVaultApi { 229 | mv := mockVaultApi{} 230 | mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { 231 | t.Helper() 232 | if secretPath != "some/other/foo2" { 233 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 234 | } 235 | return nil, nil 236 | } 237 | return mv 238 | }, 239 | func() func() { 240 | os.Setenv("VAULT_TOKEN", "129378y1231283") 241 | return func() { 242 | os.Clearenv() 243 | } 244 | }, 245 | }, 246 | "vault rate limit incorrect": { 247 | "VAULT://secret___/some/other/foo2", 248 | config.NewConfig(), 249 | `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted 250 | failed to initialize the client`, 251 | func(t *testing.T) hashiVaultApi { 252 | mv := mockVaultApi{} 253 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 254 | t.Helper() 255 | if secretPath != "some/other/foo2" { 256 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 257 | } 258 | return &vault.KVSecret{Data: nil}, nil 259 | } 260 | return mv 261 | }, 262 | func() func() { 263 | os.Setenv("VAULT_TOKEN", "") 264 | os.Setenv("VAULT_RATE_LIMIT", "wrong") 265 | return func() { 266 | os.Clearenv() 267 | } 268 | }, 269 | }, 270 | } 271 | 272 | for name, tt := range ttests { 273 | t.Run(name, func(t *testing.T) { 274 | tearDown := tt.setupEnv() 275 | defer tearDown() 276 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) 277 | 278 | impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) 279 | if err != nil { 280 | if err.Error() != tt.expect { 281 | t.Fatalf("failed to init hashivault, %v", err.Error()) 282 | } 283 | return 284 | } 285 | 286 | impl.svc = tt.mockClient(t) 287 | got, err := impl.Token() 288 | if err != nil { 289 | if err.Error() != tt.expect { 290 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 291 | } 292 | return 293 | } 294 | if got != tt.expect { 295 | t.Errorf(testutils.TestPhrase, got, tt.expect) 296 | } 297 | }) 298 | } 299 | } 300 | 301 | func TestAwsIamAuth(t *testing.T) { 302 | ttests := map[string]struct { 303 | token string 304 | conf *config.GenVarsConfig 305 | expect string 306 | mockClient func(t *testing.T) hashiVaultApi 307 | mockHanlder func(t *testing.T) http.Handler 308 | setupEnv func(addr string) func() 309 | }{ 310 | "aws_iam auth no role specified": { 311 | "VAULT://secret___/some/other/foo2[version:1]", config.NewConfig(), 312 | "role provided is empty, EC2 auth not supported", 313 | func(t *testing.T) hashiVaultApi { 314 | mv := mockVaultApi{} 315 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 316 | t.Helper() 317 | if secretPath != "some/other/foo2" { 318 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 319 | } 320 | return &vault.KVSecret{Data: nil}, nil 321 | } 322 | return mv 323 | }, 324 | func(t *testing.T) http.Handler { 325 | return nil 326 | }, 327 | func(_ string) func() { 328 | os.Setenv("VAULT_TOKEN", "aws_iam") 329 | os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") 330 | os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") 331 | os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") 332 | os.Setenv("AWS_REGION", "eu-west-1") 333 | return func() { 334 | os.Clearenv() 335 | } 336 | }, 337 | }, 338 | "aws_iam auth incorrectly formatted request": { 339 | "VAULT://secret___/some/other/foo2[version=1,iam_role=not_a_role]", config.NewConfig(), 340 | `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. 341 | 342 | URL: PUT %s/v1/auth/aws/login 343 | Code: 400. Raw Message: 344 | 345 | incorrect values supplied. failed to initialize the client`, 346 | func(t *testing.T) hashiVaultApi { 347 | mv := mockVaultApi{} 348 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 349 | t.Helper() 350 | if secretPath != "some/other/foo2" { 351 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 352 | } 353 | return &vault.KVSecret{Data: nil}, nil 354 | } 355 | return mv 356 | }, 357 | func(t *testing.T) http.Handler { 358 | mux := http.NewServeMux() 359 | mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { 360 | 361 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 362 | w.WriteHeader(400) 363 | w.Write([]byte(`incorrect values supplied`)) 364 | }) 365 | return mux 366 | }, 367 | func(addr string) func() { 368 | os.Setenv("VAULT_TOKEN", "aws_iam") 369 | os.Setenv("VAULT_ADDR", addr) 370 | os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") 371 | os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") 372 | os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") 373 | os.Setenv("AWS_REGION", "eu-west-1") 374 | return func() { 375 | os.Clearenv() 376 | } 377 | }, 378 | }, 379 | "aws_iam auth success": { 380 | "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), 381 | `{"foo2":"dsfsdf3454456"}`, 382 | func(t *testing.T) hashiVaultApi { 383 | mv := mockVaultApi{} 384 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 385 | t.Helper() 386 | if secretPath != "some/other/foo2" { 387 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 388 | } 389 | m := make(map[string]interface{}) 390 | m["foo2"] = "dsfsdf3454456" 391 | return &vault.KVSecret{Data: m}, nil 392 | } 393 | return mv 394 | }, 395 | func(t *testing.T) http.Handler { 396 | mux := http.NewServeMux() 397 | mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { 398 | 399 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 400 | w.Write([]byte(`{"auth":{"client_token": "fooresddfasdsasad"}}`)) 401 | }) 402 | return mux 403 | }, 404 | func(addr string) func() { 405 | os.Setenv("VAULT_TOKEN", "aws_iam") 406 | os.Setenv("VAULT_ADDR", addr) 407 | os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") 408 | os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") 409 | os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") 410 | os.Setenv("AWS_REGION", "eu-west-1") 411 | return func() { 412 | os.Clearenv() 413 | } 414 | }, 415 | }, 416 | "aws_iam auth no token returned": { 417 | "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), 418 | `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, 419 | func(t *testing.T) hashiVaultApi { 420 | mv := mockVaultApi{} 421 | mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { 422 | t.Helper() 423 | if secretPath != "some/other/foo2" { 424 | t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) 425 | } 426 | m := make(map[string]interface{}) 427 | m["foo2"] = "dsfsdf3454456" 428 | return &vault.KVSecret{Data: m}, nil 429 | } 430 | return mv 431 | }, 432 | func(t *testing.T) http.Handler { 433 | mux := http.NewServeMux() 434 | mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { 435 | 436 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 437 | w.Write([]byte(`{"auth":{}}`)) 438 | }) 439 | return mux 440 | }, 441 | func(addr string) func() { 442 | os.Setenv("VAULT_TOKEN", "aws_iam") 443 | os.Setenv("VAULT_ADDR", addr) 444 | os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") 445 | os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") 446 | os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") 447 | os.Setenv("AWS_REGION", "eu-west-1") 448 | return func() { 449 | os.Clearenv() 450 | } 451 | }, 452 | }, 453 | } 454 | 455 | for name, tt := range ttests { 456 | t.Run(name, func(t *testing.T) { 457 | // 458 | ts := httptest.NewServer(tt.mockHanlder(t)) 459 | tearDown := tt.setupEnv(ts.URL) 460 | defer tearDown() 461 | token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) 462 | 463 | impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) 464 | if err != nil { 465 | // WHAT A CRAP way to do this... 466 | if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { 467 | t.Errorf(testutils.TestPhraseWithContext, "aws iam auth", err.Error(), strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0]) 468 | t.Fatalf("failed to init hashivault, %v", err.Error()) 469 | } 470 | return 471 | } 472 | 473 | impl.svc = tt.mockClient(t) 474 | got, err := impl.Token() 475 | if err != nil { 476 | if err.Error() != tt.expect { 477 | t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) 478 | } 479 | return 480 | } 481 | if got != tt.expect { 482 | t.Errorf(testutils.TestPhrase, got, tt.expect) 483 | } 484 | }) 485 | } 486 | } 487 | --------------------------------------------------------------------------------