├── .gitignore ├── Dockerfile ├── Makefile ├── README.md ├── bin ├── param-api-0.0.1 └── param-api-latest ├── docker └── docker-compose.yml ├── main.go ├── param.go └── param_api /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.3 2 | 3 | RUN apk --update add ca-certificates 4 | 5 | ADD ./bin/param-api-latest /bin/param-api 6 | 7 | CMD ["/bin/param-api"] 8 | 9 | EXPOSE 8080 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Generate tarball with new build of param_api 2 | # 3 | 4 | 5 | all: clean build build-docker 6 | 7 | clean: 8 | @rm -f ./bin/param-api-latest 9 | 10 | build: 11 | @echo Building param-api version $(VERSION) 12 | @env CGO_ENABLED=0 GOOS=linux go build -a -tags netgo -ldflags '-w' -o ./bin/param-api-latest *.go 13 | 14 | build-docker: 15 | @echo Building docker tag $(TAG) in $(AWS_REGION) 16 | @env AWS_REGION=$(AWS_REGION) TAG=$(TAG) docker-compose -f docker/docker-compose.yml build 17 | 18 | .PHONY: all clean build 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Parameter Store API 2 | 3 | A helpful Golang-application that provides a HTTP endpoint to retrieve parameters from AWS Parameter Store. 4 | 5 | > STOP WORRING ABOUT HAVING TO REVISION CONTROL ENV VARIABLES, STOP HAVING TO UPLOAD SECRETS TO S3! 6 | 7 | ## Design 8 | 9 | For AWS paramater store, the entry should be in the format: 10 | ``` 11 | landscape.environment.application_name.KEY_NAME 12 | ``` 13 | 14 | To revision the same entry, edit it, bump the version number in the description, and change the value. 15 | 16 | **NOTE:** 17 | 18 | `AWS Param Store keeps the history of changed values. If you want to use an old version, you simply have to search for the key name with a description of the version you want to access.` 19 | 20 | image 21 | 22 | ## Installation 23 | 24 | **Clone the repo** 25 | 26 | ``` 27 | git clone git@github.com:Sjeanpierre/param_api.git 28 | ``` 29 | 30 | **Build the binary** 31 | 32 | ``` 33 | make build 34 | ``` 35 | 36 | **Build the Docker image** 37 | 38 | ``` 39 | AWS_REGION={SOME_REGION} TAG={SOME_DOCKER_IMAGE_TAG} docker-compose -f docker/docker-compose.yml build 40 | ``` 41 | 42 | **Push the Docker image** 43 | 44 | `You can upload this to ECR, Dockerhub, or any other prefered docker image revision control.` 45 | 46 | ## Implementation Options 47 | 48 | | Type | Description | 49 | | ------------- | ------------- | 50 | | Standalone | You can set this application up by building the Docker image for it and running it on a single server. You will need to allow http requests to that server on port 8080 | 51 | | Multi-Container | You can setup this application through a multi-container approach through AWS Elastic Beanstalk, AWS ECS, or on any server using docker-compose with multiple container definitions. | 52 | 53 | `The param-store-api will return JSON data with the environment variables( key/value ). It's up to you if you want to perform the HTTP request via command line and then export the values, or if you want your application make the HTTP request and interpret the variabls.` 54 | 55 | ## Usage 56 | 57 | **Run the image** 58 | 59 | ``` 60 | AWS_REGION={SOME_REGION} TAG={SOME_DOCKER_IMAGE_TAG} docker-compose -f docker/docker-compose.yml up 61 | ``` 62 | 63 | **Retrieve App config as a single JSON document from AWS SSM parameter store** 64 | 65 | image 66 | -------------------------------------------------------------------------------- /bin/param-api-0.0.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sjeanpierre/param_api/e9f56889e6ef06acd6ba5ac30a97ef12a574a7d5/bin/param-api-0.0.1 -------------------------------------------------------------------------------- /bin/param-api-latest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sjeanpierre/param_api/e9f56889e6ef06acd6ba5ac30a97ef12a574a7d5/bin/param-api-latest -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | param-store-api: 4 | build: 5 | context: ../ 6 | dockerfile: ./Dockerfile 7 | ports: 8 | - "8080:8080" 9 | environment: 10 | - AWS_REGION="${AWS_REGION}" 11 | image: "aws-param-store-api:${TAG}" 12 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/gorilla/handlers" 15 | "github.com/gorilla/mux" 16 | ) 17 | 18 | type Response struct { 19 | status int 20 | Message string 21 | Data interface{} 22 | } 23 | 24 | type paramRequest struct { 25 | Application string 26 | Environment string 27 | Version string 28 | Landscape string 29 | } 30 | 31 | func (p paramRequest) valid() bool { 32 | if p.Application == "" || p.Environment == "" || p.Version == "" || p.Landscape == "" { 33 | return false 34 | } 35 | return true 36 | } 37 | 38 | func (p paramRequest) envPrefix() string { 39 | return fmt.Sprintf("%s.%s.%s", p.Landscape, p.Environment, p.Application) 40 | } 41 | 42 | func (p paramRequest) cacheKey() string { 43 | return fmt.Sprintf("%x", sha256.Sum256([]byte(p.identifier()))) 44 | } 45 | 46 | func (p paramRequest) identifier() string { 47 | return fmt.Sprintf("%s@%s", p.envPrefix(), p.Version) 48 | } 49 | 50 | var ( 51 | DebugMode = false 52 | SingleKeyMode = false 53 | CACHE = make(map[string]Response) 54 | region = os.Getenv("AWS_REGION") 55 | debug = os.Getenv("DEBUG") 56 | SingleKey = os.Getenv("SINGLE_KEY_MODE") 57 | ) 58 | 59 | func main() { 60 | api() 61 | } 62 | 63 | func api() { 64 | router := mux.NewRouter().StrictSlash(true) 65 | registerHandlers(router) 66 | loggedRouter := handlers.LoggingHandler(os.Stdout, router) 67 | log.Println("Validating Config") //todo, validate config 68 | if region == "" { 69 | log.Fatal("Environment variable AWS_REGION undefined") 70 | //todo, check against list of known regions 71 | } 72 | //in debug mode no caching takes place 73 | //logs are produced in greater detail 74 | if debug != "" { 75 | log.Printf("DEBUG flag set to %+v - attempting to parse to boolean", debug) 76 | debugenabled, err := strconv.ParseBool(debug) 77 | if err != nil { 78 | log.Printf("Warning: Could not parse debug flag, value provided was %s\n %s", DebugMode, err.Error()) 79 | log.Println("debug mode: false") 80 | DebugMode = false 81 | } else { 82 | DebugMode = debugenabled 83 | log.Printf("debug mode set to %+v", DebugMode) 84 | } 85 | } 86 | if SingleKey != "" { 87 | sk, err := strconv.ParseBool(SingleKey) 88 | if err != nil { 89 | log.Fatalf("Could not start application, unknown value '%v' set "+ 90 | "for SINGLE_KEY_MODE ENV VAR - true or false required", SingleKey) 91 | } 92 | SingleKeyMode = sk 93 | } 94 | log.Println("Started: Ready to serve") 95 | log.Fatal(http.ListenAndServe(":8080", loggedRouter)) //todo, refactor to make port dynamic 96 | } 97 | 98 | func registerHandlers(r *mux.Router) { 99 | r.NotFoundHandler = http.HandlerFunc(notFoundHandler) 100 | r.HandleFunc("/params", envHandler).Methods("POST") 101 | r.HandleFunc("/service/health", healthHandler).Methods("GET") 102 | } 103 | 104 | func parseParamRequestBody(b io.ReadCloser) paramRequest { 105 | decoder := json.NewDecoder(b) 106 | var p paramRequest 107 | err := decoder.Decode(&p) 108 | if err != nil { 109 | log.Printf("encountered issue decoding request body; %s", err.Error()) 110 | return paramRequest{} 111 | } 112 | return p 113 | } 114 | 115 | func (p paramRequest) getData() map[string]string { 116 | c := ssmClient{NewClient(region)} 117 | if SingleKeyMode { 118 | //paramName := c.WithPrefix(fmt.Sprintf("%s.%s.%s.%s", p.Landscape, p.Environment, p.Application,p.Version)) 119 | //return paramName.IncludeHistory(c).withVersion(p.Version) 120 | return c.SingleParam(fmt.Sprintf("%s.%s.%s.%s", p.Landscape, p.Environment, p.Application,p.Version)) 121 | 122 | } 123 | paramNames := c.WithPrefix(p.envPrefix()) //todo, provide the full known param, is composite key 124 | return paramNames.IncludeHistory(c).withVersion(p.Version) //todo, return error 125 | } 126 | 127 | func healthHandler(w http.ResponseWriter, r *http.Request) { 128 | w.Header().Set("Content-Type", "application/json") 129 | var m = make(map[string]string) 130 | 131 | m["region"] = region 132 | m["single_key"] = fmt.Sprintf("%t", SingleKeyMode) 133 | m["debug"] = fmt.Sprintf("%t", DebugMode) 134 | 135 | resp := Response{status: http.StatusOK, Data: m} //todo, check length of list before returning 136 | 137 | JSONResponseHandler(w, resp) 138 | } 139 | 140 | func envHandler(w http.ResponseWriter, r *http.Request) { 141 | p := parseParamRequestBody(r.Body) 142 | if !p.valid() { 143 | badRequest(w, p) 144 | return 145 | } 146 | log.Printf("Processing request for %s uniquely identified as %+v", p.identifier(), p.cacheKey()) 147 | cached, ok := CACHE[p.cacheKey()] 148 | if ok { 149 | if DebugMode { 150 | log.Println("Bypassing response cache due to debug mode") 151 | } else { 152 | log.Printf("Retrieved parameters from cache") 153 | JSONResponseHandler(w, cached) 154 | return 155 | 156 | } 157 | } 158 | data := p.getData() 159 | resp := Response{status: http.StatusOK, Data: data} //todo, check length of list before returning 160 | //only cache data when elements were found 161 | //possible bug - existing versions where new elements are added will still return cached data 162 | //should not be a problem since container will be restarted upon config changes 163 | //latest is treated as a special version indicator which should not be cached 164 | if len(data) > 0 && p.Version != "latest" { 165 | if DebugMode { 166 | log.Println("Skipping response caching due to debug mode") 167 | } else { 168 | log.Println("Caching result set") 169 | CACHE[p.cacheKey()] = resp 170 | } 171 | } 172 | JSONResponseHandler(w, resp) 173 | } 174 | 175 | func notFoundHandler(w http.ResponseWriter, r *http.Request) { 176 | w.Header().Set("Content-Type", "application/json") 177 | var m = make(map[string]string) 178 | m["error"] = fmt.Sprintf("Route %s not found with method %s, please check request and try again", 179 | r.URL.Path, r.Method) 180 | resp := Response{Data: m, status: http.StatusNotFound} 181 | JSONResponseHandler(w, resp) 182 | } 183 | 184 | func badRequest(w http.ResponseWriter, p paramRequest) { 185 | w.Header().Set("Content-Type", "application/json") 186 | var m = make(map[string]string) 187 | expected := strings.ToLower(fmt.Sprintf("%+v", paramRequest{"STRING", "STRING", "STRING", "STRING"})) 188 | m["error"] = fmt.Sprintf("Bad request, expected: %s, got: %s", expected, strings.ToLower(fmt.Sprintf("%+v", p))) 189 | resp := Response{Data: m, status: http.StatusBadRequest} 190 | JSONResponseHandler(w, resp) 191 | } 192 | 193 | func JSONResponseHandler(w http.ResponseWriter, resp Response) { 194 | w.Header().Set("Content-Type", "application/json") 195 | w.WriteHeader(resp.status) 196 | json.NewEncoder(w).Encode(resp.Data) 197 | } 198 | -------------------------------------------------------------------------------- /param.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/base64" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io/ioutil" 11 | "log" 12 | "strings" 13 | 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/aws/aws-sdk-go/service/ssm" 17 | ) 18 | 19 | type ssmClient struct { 20 | client *ssm.SSM 21 | } 22 | 23 | type parameter struct { 24 | Name string 25 | Versions []paramHistory 26 | } 27 | 28 | type parameters []parameter 29 | 30 | type paramHistory struct { 31 | Value string 32 | Version string 33 | } 34 | 35 | func NewClient(region string) *ssm.SSM { 36 | session := session.Must(session.NewSession()) 37 | if DebugMode { 38 | session.Config.WithRegion(region).WithLogLevel(aws.LogDebugWithHTTPBody) //.WithMaxRetries(2) 39 | } else { 40 | session.Config.WithRegion(region) 41 | } 42 | return ssm.New(session) 43 | } 44 | 45 | var ( 46 | CACHEREF = map[string]map[string]string{} 47 | ) 48 | 49 | func Deserialize(encoded string) (map[string]string, error) { 50 | params := make(map[string]string) 51 | compressed, err := base64.StdEncoding.DecodeString(encoded) 52 | if err != nil { 53 | error := fmt.Sprintf("Error decoding value returned by single key param: %s", err.Error()) 54 | return params, errors.New(error) 55 | } 56 | gz, err := gzip.NewReader(bytes.NewBuffer(compressed)) 57 | defer gz.Close() 58 | if err != nil { 59 | error := fmt.Sprintf("Error decompressing value returned by single key param: %s", err.Error()) 60 | return params, errors.New(error) 61 | } 62 | jsonData, err := ioutil.ReadAll(gz) 63 | if err != nil { 64 | error := fmt.Sprintf("Error reading decompressed json stream: %s", err.Error()) 65 | return params, errors.New(error) 66 | } 67 | err = json.Unmarshal(jsonData, ¶ms) 68 | if err != nil { 69 | error := fmt.Sprintf("Error unmarshalling JSON to struct: %s", err.Error()) 70 | return params, errors.New(error) 71 | } 72 | return params, nil 73 | } 74 | 75 | ////// SINGLE KEY MODE ONLY 76 | ////// 77 | ////// 78 | ////// 79 | func (s ssmClient) SingleParam(paramName string) map[string]string { 80 | empty := make(map[string]string) 81 | 82 | // Get requested parameter 83 | paramData, err := s.CacheRequestedParam(paramName) 84 | if err != nil { 85 | return empty 86 | } 87 | CACHEREF[paramName] = paramData 88 | 89 | // Iterate over param data to get ssm:// references 90 | for i := range CACHEREF[paramName] { 91 | if strings.HasPrefix(CACHEREF[paramName][i], "ssm://") { 92 | // Trim the ssm:// off of the key so we know the full name of the param to get 93 | keyName := strings.Trim(CACHEREF[paramName][i], "ssm://") 94 | 95 | // Request param store for the key above and store it in the CACHEREF 96 | paramReferenceData, err := s.CacheRequestedParam(keyName) 97 | if err != nil { 98 | return empty 99 | } 100 | CACHEREF[keyName] = paramReferenceData 101 | 102 | // If the reference json data has the key required, get the value 103 | if val, ok := CACHEREF[keyName][i]; ok { 104 | CACHEREF[paramName][i] = val 105 | // If the reference json data does not have the key we require, return empty 106 | } else { 107 | log.Printf("Error: Key %s in parameter reference %s was not found.\n", i, keyName) 108 | return empty 109 | } 110 | } 111 | } 112 | return CACHEREF[paramName] 113 | } 114 | 115 | func (s ssmClient) CacheRequestedParam(paramName string) (map[string]string, error) { 116 | empty := make(map[string]string) 117 | 118 | if _, ok := CACHEREF[paramName]; ok { 119 | if DebugMode { 120 | log.Printf("%s is already cached, skipping\n", paramName) 121 | } 122 | return CACHEREF[paramName], nil 123 | } 124 | 125 | pi := &ssm.GetParameterInput{Name: aws.String(paramName), 126 | WithDecryption: aws.Bool(true)} 127 | 128 | r, err := s.client.GetParameter(pi) 129 | if err != nil { 130 | log.Printf("%s, parameter name: %s\n", err.Error(), paramName) 131 | return empty, err 132 | } 133 | 134 | ret, err := Deserialize(*r.Parameter.Value) 135 | if err != nil { 136 | log.Println(err) 137 | return empty, err 138 | } 139 | 140 | return ret, nil 141 | } 142 | 143 | ////// ANYTHING BEYOND THIS POINT IS FOR NON-SINGLE KEY MODE ONLY 144 | ////// 145 | ////// 146 | ////// 147 | func (s ssmClient) WithPrefix(prefix string) parameters { 148 | var names parameters 149 | resp, err := s.ParamList(prefix) 150 | if err != nil { 151 | log.Println("Encountered an error listing params", err) 152 | return parameters{} 153 | } 154 | for _, param := range resp.Parameters { 155 | names = append(names, parameter{*param.Name, []paramHistory{}}) 156 | } 157 | return names 158 | } 159 | 160 | func (s ssmClient) ParamList(filter string) (*ssm.DescribeParametersOutput, error) { 161 | //limit of 50 parameters, unless extra logic is added to paginate 162 | params := &ssm.DescribeParametersInput{ 163 | MaxResults: aws.Int64(50), 164 | Filters: []*ssm.ParametersFilter{ 165 | { 166 | Values: []*string{ 167 | aws.String(filter), 168 | }, 169 | Key: aws.String("Name"), 170 | }, 171 | }, 172 | } 173 | return s.client.DescribeParameters(params) 174 | } 175 | 176 | func (p parameters) IncludeHistory(s ssmClient) parameters { 177 | var params parameters 178 | for _, param := range p { 179 | param.history(s) 180 | params = append(params, param) 181 | } 182 | return params 183 | } 184 | 185 | func (p *parameter) history(s ssmClient) { 186 | //todo, return error 187 | pi := &ssm.GetParametersInput{ 188 | Names: []*string{&p.Name}, 189 | WithDecryption: aws.Bool(true), 190 | } 191 | hpi := &ssm.GetParameterHistoryInput{ 192 | Name: &p.Name, 193 | WithDecryption: aws.Bool(true), 194 | } 195 | resp, err := s.client.GetParameterHistory(hpi) 196 | if err != nil { 197 | fmt.Println(err.Error()) 198 | return 199 | } 200 | r, err := s.client.GetParameters(pi) 201 | if err != nil { 202 | fmt.Println(err.Error()) 203 | return 204 | } 205 | re, err := s.ParamList(p.Name) 206 | if err != nil { 207 | fmt.Println(err.Error()) 208 | return 209 | } 210 | //todo, guard against empty param 211 | //this is being done in order to get the current version description 212 | p.Versions = append(p.Versions, paramHistory{Value: *r.Parameters[0].Value, Version: *re.Parameters[0].Description}) 213 | var hist []paramHistory 214 | var des string 215 | for _, param := range resp.Parameters { 216 | if param.Description != nil { 217 | des = *param.Description 218 | 219 | } 220 | val := *param.Value 221 | hist = append(hist, paramHistory{Value: val, Version: des}) 222 | } 223 | p.Versions = append(p.Versions, hist...) 224 | return 225 | } 226 | 227 | func (p parameters) withVersion(version string) map[string]string { 228 | paramsDoc := make(map[string]string) 229 | //todo, deserialize right here 230 | 231 | for _, param := range p { 232 | ver, err := param.containsVersion(version) 233 | if err != nil { 234 | log.Printf("Error: could not find version: %v for param %v", version, param.Name) 235 | continue 236 | } 237 | if SingleKeyMode { 238 | decodedData, err := Deserialize(ver.Value) 239 | if err != nil { 240 | log.Printf("Could not retrieve single key param: %s", err.Error()) 241 | continue 242 | } 243 | return decodedData 244 | } 245 | ParsedName := strings.Split(param.Name, ".") //todo, check if envName matches ENV VAR regex 246 | envName := ParsedName[len(ParsedName)-1] 247 | paramsDoc[envName] = ver.Value 248 | } 249 | return paramsDoc 250 | } 251 | 252 | func (p parameter) containsVersion(version string) (paramHistory, error) { 253 | for _, v := range p.Versions { 254 | if v.Version == version { 255 | return v, nil 256 | } 257 | } 258 | return paramHistory{}, errors.New("could not find version") 259 | } 260 | -------------------------------------------------------------------------------- /param_api: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sjeanpierre/param_api/e9f56889e6ef06acd6ba5ac30a97ef12a574a7d5/param_api --------------------------------------------------------------------------------