├── src ├── gitutil │ ├── test_data │ │ ├── test_object_storage.json │ │ ├── sandbox_whitelist.json │ │ ├── user.json │ │ ├── test_users2.json │ │ ├── kernels │ │ │ └── spark-kernel-12CPU-24GB │ │ └── test_users1.json │ ├── git_utility.go │ └── git_utility_test.go ├── handlers │ ├── gitHandler.go │ ├── confEnvHandler.go │ ├── kernelHandler.go │ ├── keyHandler.go │ ├── userHandler.go │ └── userHandler_test.go ├── config-data-util │ ├── config_structs.go │ ├── kernel │ │ ├── kernel_test.go │ │ └── kernel.go │ ├── environment │ │ └── environment.go │ ├── data_utility.go │ ├── memfilesystem │ │ └── memfilesystem.go │ ├── key │ │ └── key.go │ └── user │ │ ├── user.go │ │ └── user_test.go └── helpers │ └── helpers.go ├── charts └── service-config │ ├── requirements.yaml │ ├── Chart.yaml │ ├── templates │ ├── service.yaml │ ├── deployment.yaml │ └── image-pull-secret.yaml │ └── values-template.yaml ├── .gitmodules ├── watch.sh ├── .dockerignore ├── .gitignore ├── scripts ├── hook.template ├── common.sh └── githook.sh ├── config.json ├── .helmignore ├── SECURITY.md ├── Jenkinsfile ├── glide.yaml ├── skaffold.yaml ├── JenkinsPod.yaml ├── Makefile ├── service.go └── README.md /src/gitutil/test_data/test_object_storage.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /charts/service-config/requirements.yaml: -------------------------------------------------------------------------------- 1 | # !! File must end with empty line !! 2 | dependencies: 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "service-common-lib"] 2 | path = service-common-lib 3 | url = https://github.ibm.com/AdvancedAnalyticsCanada/service-common-lib.git 4 | -------------------------------------------------------------------------------- /src/handlers/gitHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | conf "config-data-util" 5 | ) 6 | 7 | type GitHandler struct { 8 | Environments conf.MappingToEnv 9 | } 10 | -------------------------------------------------------------------------------- /watch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # watch the java files and continously deploy the service 4 | make linux 5 | skaffold run -p dev 6 | reflex -r "\.go$" -- bash -c 'make linux && skaffold run -p dev' 7 | -------------------------------------------------------------------------------- /charts/service-config/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Helm chart for Kubernetes 3 | icon: https://raw.githubusercontent.com/jenkins-x/jenkins-x-platform/master/images/go.png 4 | name: config-service 5 | version: 1.0.0-SNAPSHOT 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | draft.toml 2 | target/classes 3 | target/generated-sources 4 | target/generated-test-sources 5 | target/maven-archiver 6 | target/maven-status 7 | target/surefire-reports 8 | target/test-classes 9 | target/*.original 10 | charts/ 11 | NOTICE 12 | LICENSE 13 | README.md -------------------------------------------------------------------------------- /src/gitutil/test_data/sandbox_whitelist.json: -------------------------------------------------------------------------------- 1 | { 2 | "whitelist": ["brian.ho@ibm.com", "edward.ren@ibm.com", "hongyi.tian@ibm.com", "judah@ibm.com", "rachel.liu@ibm.com"], 3 | "admin": ["oleg.gorodnitchi@ibm.com", "Zain.Kabani@ibm.com", "zebin.kang@ibm.com", "Mehryar.Maalem@ibm.com"] 4 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp.* 2 | vars*.mk 3 | service-config-data 4 | Dockerfile 5 | secret.yml 6 | tmp-secrets.yaml 7 | 8 | .DS_Store 9 | .idea 10 | credentials.json 11 | tmp* 12 | 13 | charts/service-config/values.yaml 14 | src/github.com/* 15 | vendor/* 16 | creds.json 17 | hook.json 18 | scripts/hook.json 19 | -------------------------------------------------------------------------------- /scripts/hook.template: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web", 3 | "active": true, 4 | "events": ["push"], 5 | "config": { 6 | "url": "http://{{ .ServiceIp }}:{{ .ServicePort }}/{{ .WebhookGit }}", 7 | "content_type": "json", 8 | "insecure_ssl": "0", 9 | "secret": "{{ .WebhookGitSecret }}" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/gitutil/test_data/user.json: -------------------------------------------------------------------------------- 1 | { 2 | "email": "newUser@ibm.com", 3 | "metadata": { 4 | "is_admin": false, 5 | "list_of_team_bucket_mapping": [ 6 | { 7 | "bucket_name": "sample_bucket_2", 8 | "team_name": "fun_team_2", 9 | "cos_instance": "client2_instance" 10 | } 11 | ], 12 | "user_bucket": "abdullas_bucket" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/config-data-util/config_structs.go: -------------------------------------------------------------------------------- 1 | // Package config-data-util defines all of the data structures used in the service 2 | 3 | package config_data_util 4 | 5 | import ( 6 | "config-data-util/environment" 7 | ) 8 | 9 | // An Environment represents the configuration of a particular environment this can be one of 10 | // {sandbox, dev} 11 | 12 | type MappingToEnv map[string]*environment.Environment 13 | -------------------------------------------------------------------------------- /src/config-data-util/kernel/kernel_test.go: -------------------------------------------------------------------------------- 1 | package kernel 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestGetKernels(t *testing.T) { 10 | folder, err := os.Open("../../gitutil/test_data/kernels") 11 | if err != nil { 12 | fmt.Println(err) 13 | } 14 | 15 | info, _ := folder.Readdir(0) 16 | 17 | for index, fileName := range info { 18 | fmt.Println(fileName.Name()) 19 | fmt.Println(index) 20 | } 21 | } -------------------------------------------------------------------------------- /src/config-data-util/kernel/kernel.go: -------------------------------------------------------------------------------- 1 | package kernel 2 | 3 | 4 | type Kernel struct { 5 | Name string 6 | JSON []byte 7 | } 8 | 9 | //// TODO: Finish this 10 | //func GetKernels(environment *conf.Environment) (int, error){ 11 | // //listOfUsers := environment.Users 12 | // //for i := range listOfUsers{ 13 | // // if listOfUsers[i].Email == email { 14 | // // return i,nil 15 | // // } 16 | // //} 17 | // return -1, fmt.Errorf("user not found") 18 | //} -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "service": { 3 | "name" : "service-config-data", 4 | "port" : "8000", 5 | "apiver" : "2", 6 | "backend":{ 7 | "type": "git", 8 | "repo" : "github.com/OlegGorJ/config-data.git", 9 | "webhook_path" : "/webhook_git", 10 | "account" : "", 11 | "token" : "", 12 | "git-creds-file": "credentials.json", 13 | "branches": [ 14 | "dev", 15 | "sandbox", 16 | "qa" 17 | ] 18 | } 19 | 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/gitutil/test_data/test_users2.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "email": "abdullah@ibm.com", 3 | "metadata": { 4 | "is_admin": false, 5 | "list_of_team_bucket_mapping": [] 6 | } 7 | }, 8 | { 9 | "email": "johnny@ibm.com", 10 | "metadata": { 11 | "is_admin": true, 12 | "list_of_team_bucket_mapping": [] 13 | } 14 | }, 15 | { 16 | "email": "oleg@ibm.com", 17 | "metadata": { 18 | "is_admin": true, 19 | "list_of_team_bucket_mapping": [] 20 | } 21 | } 22 | ] -------------------------------------------------------------------------------- /scripts/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BLACK=$(tput setaf 0) 3 | RED=$(tput setaf 1) 4 | GREEN=$(tput setaf 2) 5 | YELLOW=$(tput setaf 3) 6 | LIME_YELLOW=$(tput setaf 190) 7 | POWDER_BLUE=$(tput setaf 153) 8 | BLUE=$(tput setaf 4) 9 | MAGENTA=$(tput setaf 5) 10 | CYAN=$(tput setaf 6) 11 | WHITE=$(tput setaf 7) 12 | BRIGHT=$(tput bold) 13 | NORMAL=$(tput sgr0) 14 | BLINK=$(tput blink) 15 | REVERSE=$(tput smso) 16 | UNDERLINE=$(tput smul) 17 | 18 | export GITAPIURL=api.github.com 19 | export CURL=$(which curl) 20 | -------------------------------------------------------------------------------- /src/config-data-util/environment/environment.go: -------------------------------------------------------------------------------- 1 | package environment 2 | 3 | import ( 4 | "gopkg.in/src-d/go-billy.v4" 5 | "gopkg.in/src-d/go-git.v4" 6 | 7 | "config-data-util/user" 8 | "config-data-util/kernel" 9 | "config-data-util/key" 10 | ) 11 | 12 | 13 | type Environment struct { 14 | Name string 15 | Repository *git.Repository 16 | FileSystem billy.Filesystem 17 | Users user.Users 18 | Kernels []kernel.Kernel 19 | Keys key.Keys 20 | JsonData string 21 | 22 | } 23 | -------------------------------------------------------------------------------- /.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | *.png 23 | 24 | # known compile time folders 25 | target/ 26 | node_modules/ 27 | vendor/ -------------------------------------------------------------------------------- /src/config-data-util/data_utility.go: -------------------------------------------------------------------------------- 1 | package config_data_util 2 | 3 | // TODO: Populating structs based on config_data package in config_structs.go file 4 | // TODO: Think about how you can refactor this into an Interface 5 | 6 | 7 | 8 | //type ConfigCRUD interface { 9 | // Create(environment *Environment, data []byte) 10 | // Read(environment *Environment, id string) (index int, err error) 11 | // Update(environment *Environment, id string, data[]byte) (error) 12 | // Delete(environment *Environment, id string) (error) 13 | //} -------------------------------------------------------------------------------- /src/helpers/helpers.go: -------------------------------------------------------------------------------- 1 | // helpers package contains helper functions that are commonly used across this application 2 | package helpers 3 | 4 | import ( 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | func ReadFromFileToBytes(path string) []byte { 12 | absPath, _ := filepath.Abs(path) 13 | jsonFile, err := os.Open(absPath) 14 | if err != nil { 15 | fmt.Println("Issue with the file being read") 16 | print(err) 17 | } 18 | json_byte_value, _ := ioutil.ReadAll(jsonFile) 19 | return json_byte_value 20 | } 21 | -------------------------------------------------------------------------------- /src/gitutil/test_data/kernels/spark-kernel-12CPU-24GB: -------------------------------------------------------------------------------- 1 | { 2 | "display_name": "PySpark (py3.6/spark2.2.3-0.2/12cpu/24GB)", 3 | "language": "python", 4 | "argv": [ "/opt/conda/bin/python", "-m", "ipykernel", "-f", "{connection_file}" ], 5 | "env": { 6 | "SPARK_HOME": "/usr/local/spark", 7 | "PYSPARK_PYTHON": "/opt/conda/bin/python", 8 | "PYTHONPATH": "/usr/local/spark/python/:/usr/local/spark/python/lib/py4j-0.10.7-src.zip:/home/jovyan/.local/lib/python3.6/site-packages", 9 | "PYTHONSTARTUP": "/usr/local/spark/python/pyspark/shell.py", 10 | "PYSPARK_SUBMIT_ARGS": "--master spark://spark-master.spark:7077 pyspark-shell" 11 | } 12 | } -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Use this section to tell people about which versions of your project are 6 | currently being supported with security updates. 7 | 8 | | Version | Supported | 9 | | ------- | ------------------ | 10 | | 5.1.x | :white_check_mark: | 11 | | 5.0.x | :x: | 12 | | 4.0.x | :white_check_mark: | 13 | | < 4.0 | :x: | 14 | 15 | ## Reporting a Vulnerability 16 | 17 | Use this section to tell people how to report a vulnerability. 18 | 19 | Tell them where to go, how often they can expect to get an update on a 20 | reported vulnerability, what to expect if the vulnerability is accepted or 21 | declined, etc. 22 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | kubernetes { 4 | label 'jenkins-worker' 5 | defaultContainer 'jnlp' 6 | yamlFile 'JenkinsPod.yaml' 7 | } 8 | } 9 | stages { 10 | 11 | /*========================================================================*/ 12 | stage('Deploying') { 13 | when { 14 | branch 'master' 15 | } 16 | steps { 17 | container('helm') { 18 | sh(script: "helm init -c --skip-refresh", label: "Initializing Helm Client") 19 | sh(script: "make deploy", label: "Atomically Deploying Helm Chart") 20 | } 21 | } 22 | } 23 | /*========================================================================*/ 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/oleggorj/service-config-data 2 | import: 3 | - package: github.com/gorilla/mux 4 | - package: github.com/oleggorj/service-common-lib 5 | import: 6 | repo: git@github.com/oleggorj/service-common-lib.git 7 | subpackages: 8 | - common/config 9 | - common/logging 10 | - common/util 11 | - service 12 | - package: github.com/spf13/viper 13 | - package: github.com/tidwall/gjson 14 | - package: gopkg.in/src-d/go-billy.v4 15 | subpackages: 16 | - memfs 17 | - package: gopkg.in/src-d/go-git.v4 18 | subpackages: 19 | - plumbing 20 | - plumbing/object 21 | - storage/memory 22 | - package: github.com/google/go-github/github 23 | testImport: 24 | - package: gotest.tools 25 | subpackages: 26 | - assert 27 | - assert/cmp 28 | -------------------------------------------------------------------------------- /charts/service-config/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ .Values.service.name }} 5 | namespace: {{ .Values.service.K8s_namespace }} 6 | labels: 7 | app: {{ .Values.service.name }} 8 | app.kubernetes.io/name: {{ .Values.service.name }} 9 | app.kubernetes.io/version: "{{ .Values.service.Release }}" 10 | app.kubernetes.io/component: service 11 | app.kubernetes.io/managed-by: helm 12 | app.kubernetes.io/namespace: {{ .Values.service.K8s_namespace }} 13 | spec: 14 | selector: 15 | app: {{ .Values.service.name }} 16 | # clusterIP: None 17 | type: {{ .Values.service.type }} 18 | ports: 19 | - protocol: TCP 20 | # nodePort: {{ .Values.service.nodePort }} 21 | targetPort: {{ .Values.service.externalPort }} 22 | port: {{ .Values.service.containerPort }} 23 | -------------------------------------------------------------------------------- /skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v1beta2 2 | kind: Config 3 | build: 4 | artifacts: 5 | - image: service-config-data 6 | context: . 7 | docker: {} 8 | tagPolicy: 9 | envTemplate: 10 | template: '{{.DOCKER_REGISTRY}}/oleggorj/service-config-data:{{.VERSION}}' 11 | local: {} 12 | deploy: 13 | kubectl: {} 14 | profiles: 15 | - name: dev 16 | build: 17 | artifacts: 18 | - docker: {} 19 | tagPolicy: 20 | envTemplate: 21 | template: '{{.DOCKER_REGISTRY}}/oleggorj/service-config-data:{{.DIGEST_HEX}}' 22 | local: {} 23 | deploy: 24 | helm: 25 | releases: 26 | - name: service-config-data 27 | chartPath: charts/service-config-data 28 | setValueTemplates: 29 | image.repository: '{{.DOCKER_REGISTRY}}/oleggorj/service-config-data' 30 | image.tag: '{{.DIGEST_HEX}}' 31 | -------------------------------------------------------------------------------- /src/config-data-util/memfilesystem/memfilesystem.go: -------------------------------------------------------------------------------- 1 | package memfilesystem 2 | 3 | import ( 4 | "fmt" 5 | "gopkg.in/src-d/go-billy.v4" 6 | "io/ioutil" 7 | ) 8 | 9 | func OverWriteFile(fs billy.Filesystem, path string, dataToWrite []byte) error { 10 | 11 | //fsRef := *fs 12 | err := fs.Remove(path) 13 | if err != nil { 14 | 15 | return fmt.Errorf(fmt.Sprintf("File %s not found", path)) 16 | } 17 | f, _ := fs.Create(path) 18 | _, _ = f.Write(dataToWrite) 19 | _ = f.Close() 20 | 21 | return nil 22 | 23 | } 24 | 25 | func ReadFile(fs billy.Filesystem, path string) ([]byte, error) { 26 | 27 | f, err := fs.Open(path) 28 | if err != nil { 29 | _ = f.Close() 30 | return nil, fmt.Errorf("file does not exist") 31 | } 32 | 33 | bytes, err := ioutil.ReadAll(f) 34 | _ = f.Close() 35 | 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return bytes, err 41 | } -------------------------------------------------------------------------------- /src/handlers/confEnvHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | // 3 | //import ( 4 | // 5 | // "encoding/json" 6 | // log "github.com/oleggorj/service-common-lib/common/logging" 7 | 8 | // "net/http" 9 | //) 10 | // 11 | //type ConfEnvHandler struct { 12 | // Environments conf.MappingToEnv 13 | //} 14 | // 15 | //func (c *ConfEnvHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 16 | // response := make(map[string]conf.Environment) 17 | // for k, v := range c.Environments { 18 | // response[k] = conf.Environment{ 19 | // Name: v.Name, 20 | // Users: v.Users, 21 | // } 22 | // } 23 | // err := json.NewEncoder(rw).Encode(response) 24 | // if err != nil { 25 | // log.Error("ERROR: The response variable in the user environment can't be encoded") 26 | // rw.WriteHeader(http.StatusInternalServerError) 27 | // return 28 | // } 29 | // rw.WriteHeader(http.StatusOK) 30 | // rw.Header().Set("Content-Type", "application/json") 31 | // 32 | //} 33 | -------------------------------------------------------------------------------- /charts/service-config/values-template.yaml: -------------------------------------------------------------------------------- 1 | 2 | cleanup: 3 | Args: 4 | - --cleanup 5 | Annotations: 6 | helm.sh/hook: pre-delete 7 | helm.sh/hook-delete-policy: hook-succeeded 8 | 9 | secretsibmcloud: 10 | enabled: false 11 | imagePullSecrets: image-pull-secret-ibm-cloud 12 | 13 | service: 14 | name: {{ .ServiceName }} 15 | type: LoadBalancer 16 | externalPort: {{ .LBPort }} 17 | containerPort: {{ .ContainerPort }} 18 | nodePort: 31070 19 | replicaCount: 1 20 | 21 | GITACCOUNT: 22 | APITOKEN: 23 | GITREPO: "github.com/OlegGorJ/config-data.git" 24 | 25 | CONFIGFILE: services 26 | Release: "{{ .Release }}" 27 | Environment: "{{ .Env }}" 28 | K8s_namespace: "{{ .Kube_namespace }}" 29 | APIVER: "{{ .ApiVer }}" 30 | #NodeSelector: services 31 | 32 | image: 33 | repository: "{{ .DockerOrg }}/{{ .ServiceName }}:{{ .Release }}" 34 | pullPolicy: Always 35 | 36 | resources: 37 | requests: 38 | cpu: 100m 39 | memory: 100Mi 40 | -------------------------------------------------------------------------------- /JenkinsPod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | jenkins/kube-default: "true" 6 | app: jenkins 7 | component: agent 8 | spec: 9 | serviceAccountName: cicd-jenkins 10 | containers: 11 | - name: jnlp 12 | image: jenkins/jnlp-slave:3.27-1 13 | imagePullPolicy: Always 14 | env: 15 | - name: POD_IP 16 | valueFrom: 17 | fieldRef: 18 | fieldPath: status.podIP 19 | - name: DOCKER_HOST 20 | value: tcp://localhost:2375 21 | - name: helm 22 | image: alpine/helm:2.13.1 23 | command: 24 | - cat 25 | tty: true 26 | - name: dind 27 | image: docker:18.05-dind 28 | securityContext: 29 | privileged: true 30 | volumeMounts: 31 | - name: dind-storage 32 | mountPath: /var/lib/docker 33 | - name: integration-test-image 34 | image: robcherry/docker-chromedriver:headless 35 | command: 36 | - cat 37 | tty: true 38 | volumes: 39 | - name: dind-storage 40 | emptyDir: {} 41 | -------------------------------------------------------------------------------- /src/config-data-util/key/key.go: -------------------------------------------------------------------------------- 1 | package key 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | _ "gopkg.in/src-d/go-billy.v4" 7 | _ "config-data-util/memfilesystem" 8 | log "github.com/oleggorj/service-common-lib/common/logging" 9 | ) 10 | 11 | type Keys []Key 12 | 13 | type Key struct { 14 | Key string 15 | Val string 16 | } 17 | 18 | func (keys *Keys) Init(jsonBuffer []byte) ( error){ 19 | //fmt.Println("AFTER ", string(jsonBuffer)) 20 | keys_arr := make(map[string]interface{}) 21 | err := json.Unmarshal(jsonBuffer, &keys_arr) 22 | if err != nil { 23 | log.Error(err) 24 | return err 25 | } 26 | //var allkeys Keys 27 | for j_key, j_value := range keys_arr { 28 | var k Key 29 | k.Key = j_key 30 | k.Val = fmt.Sprintf("%s", j_value ) 31 | (*keys) = append( (*keys), k) 32 | 33 | fmt.Println("Pair: key - ", j_key, ", val - ", j_value) 34 | } 35 | return nil 36 | } 37 | 38 | func (keys *Keys) Read(key string) (string, error){ 39 | 40 | for i := range *keys{ 41 | if (*keys)[i].Key == key { 42 | return (*keys)[i].Val, nil 43 | } 44 | } 45 | return "", fmt.Errorf("key not found") 46 | } 47 | -------------------------------------------------------------------------------- /src/handlers/kernelHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/gorilla/mux" 6 | "net/http" 7 | "strings" 8 | 9 | log "github.com/oleggorj/service-common-lib/common/logging" 10 | conf "config-data-util" 11 | 12 | ) 13 | 14 | type KernelHandler struct { 15 | Environments conf.MappingToEnv 16 | } 17 | 18 | // TODO: Implement the POST,DELETE, PUT request for Users 19 | func (u *KernelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 20 | 21 | envValue := strings.ToLower(mux.Vars(req)["environment"]) 22 | environment := u.Environments[envValue] 23 | 24 | if environment == nil { 25 | log.Error("ERROR: Environment does not exist") 26 | rw.WriteHeader(http.StatusBadRequest) 27 | return 28 | } 29 | 30 | response := environment.Users 31 | err := json.NewEncoder(rw).Encode(response) 32 | if err != nil { 33 | log.Error("ERROR: The response variable in the user environment can't be encoded") 34 | rw.WriteHeader(http.StatusInternalServerError) 35 | return 36 | } 37 | 38 | rw.WriteHeader(http.StatusOK) 39 | rw.Header().Set("Content-Type", "application/json") 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/gitutil/test_data/test_users1.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "email": "abdullah@ibm.com", 3 | "metadata": { 4 | "is_admin": false, 5 | "list_of_team_bucket_mapping": [ 6 | { 7 | "bucket_name": "sample_bucket_2", 8 | "team_name": "fun_team_2", 9 | "cos_instance": "client2_instance" 10 | } 11 | ], 12 | "user_bucket": "abdullas_bucket" 13 | } 14 | }, 15 | { 16 | "email": "johnny@ibm.com", 17 | "metadata": { 18 | "is_admin": true, 19 | "list_of_team_bucket_mapping": [ 20 | { 21 | "bucket_name": "sample_bucket_1", 22 | "team_name": "fun_team_2", 23 | "cos_instance": "client1_instance" 24 | }, 25 | { 26 | "bucket_name": "sample_bucket_2", 27 | "team_name": "fun_team_2", 28 | "cos_instance": "client2_instance" 29 | } 30 | ], 31 | "user_bucket": "abdullas_bucket" 32 | } 33 | }, 34 | { 35 | "email": "oleg@ibm.com", 36 | "metadata": { 37 | "is_admin": true, 38 | "list_of_team_bucket_mapping": [ 39 | { 40 | "bucket_name": "sample_bucket_3", 41 | "team_name": "fun_team_3", 42 | "cos_instance": "client3_instance" 43 | } 44 | ], 45 | "user_bucket": "abdullas_bucket" 46 | } 47 | } 48 | ] -------------------------------------------------------------------------------- /charts/service-config/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ .Values.service.name }} 5 | namespace: {{ .Values.service.K8s_namespace }} 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: {{ .Values.service.name }} 10 | replicas: {{ .Values.service.replicaCount }} 11 | template: 12 | metadata: 13 | labels: 14 | app: {{ .Values.service.name }} 15 | app.kubernetes.io/name: {{ .Values.service.name }} 16 | app.kubernetes.io/version: "{{ .Values.service.Release }}" 17 | app.kubernetes.io/component: service 18 | app.kubernetes.io/managed-by: helm 19 | app.kubernetes.io/namespace: {{ .Values.service.K8s_namespace }} 20 | spec: 21 | # nodeSelector: 22 | # app: {{ .Values.service.NodeSelector }} 23 | containers: 24 | - name: {{ .Values.service.name }} 25 | image: "{{ .Values.image.repository }}" 26 | imagePullPolicy: {{ .Values.image.pullPolicy }} 27 | resources: 28 | requests: 29 | cpu: {{ .Values.resources.requests.cpu }} 30 | memory: {{ .Values.resources.requests.memory }} 31 | env: 32 | - name: APIVER 33 | value: {{ .Values.service.APIVER }} 34 | 35 | ports: 36 | - containerPort: {{ .Values.service.containerPort }} 37 | 38 | livenessProbe: 39 | httpGet: 40 | path: /healthz 41 | port: {{ .Values.service.externalPort }} 42 | readinessProbe: 43 | httpGet: 44 | path: /readyz 45 | port: {{ .Values.service.externalPort }} 46 | 47 | imagePullSecrets: 48 | - name: {{ .Values.imagePullSecrets }} 49 | -------------------------------------------------------------------------------- /src/handlers/keyHandler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | _ "fmt" 5 | "github.com/gorilla/mux" 6 | "net/http" 7 | "strings" 8 | "encoding/json" 9 | "encoding/xml" 10 | "github.com/tidwall/gjson" 11 | 12 | log "github.com/oleggorj/service-common-lib/common/logging" 13 | conf "config-data-util" 14 | "config-data-util/memfilesystem" 15 | ) 16 | 17 | type KeyHandler struct { 18 | Environments conf.MappingToEnv 19 | } 20 | 21 | // move this to utilis 22 | func IsJSON(str string) bool { 23 | var js json.RawMessage 24 | return json.Unmarshal([]byte(str), &js) == nil 25 | } 26 | func IsXML(data []byte) bool { 27 | return xml.Unmarshal(data, new(interface{})) != nil 28 | } 29 | 30 | func (u *KeyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 31 | outformat := "plain" 32 | v := req.URL.Query() 33 | outformat = v.Get("out") 34 | //serviceDebugFlag := false 35 | appValue := strings.ToLower(mux.Vars(req)["app"]) 36 | envValue := strings.ToLower(mux.Vars(req)["env"]) 37 | keyValue := strings.Replace( mux.Vars(req)["key"] , "@","#",-1) 38 | if appValue == "" || envValue == "" || keyValue == "" { 39 | log.Error("ERROR: , or can not be empty.\n") 40 | rw.WriteHeader(http.StatusNotAcceptable) 41 | return 42 | } 43 | 44 | environment := u.Environments[envValue] 45 | if environment == nil { 46 | log.Error("ERROR: Environment " + envValue + " does not exist") 47 | rw.WriteHeader(http.StatusBadRequest) 48 | return 49 | } 50 | 51 | if req.Method == http.MethodGet { 52 | // read config json 53 | fs := &environment.FileSystem 54 | bytes, err := memfilesystem.ReadFile(*fs, appValue + ".json") 55 | if err != nil { 56 | log.Error(err) 57 | } 58 | 59 | // cleanup new lines 60 | environment.JsonData = strings.Replace( string(bytes), "\n","",-1 ) 61 | // get the value for the key (keyValue) 62 | val := gjson.Get( environment.JsonData , keyValue ) 63 | 64 | var byteData []byte = []byte( val.String() ) 65 | if outformat == "json" { 66 | rw.Header().Set("Content-Type", "application/json") 67 | // don't marshal output if its already json 68 | if IsJSON( val.String() ) == false { byteData, err = json.Marshal( val.String() ) } 69 | }else if outformat == "xml" { 70 | rw.Header().Set("Content-Type", "application/xml") 71 | byteData, err = xml.Marshal( val.String() ) 72 | }else{ 73 | rw.Header().Set("Content-Type", "application/text") 74 | } 75 | 76 | rw.WriteHeader(http.StatusOK) 77 | rw.Write( byteData ) 78 | 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | COMMIT?=$(shell git rev-parse --short HEAD) 2 | BUILD_TIME?=$(shell date -u '+%Y-%m-%d_%H:%M:%S') 3 | GOOS?=linux 4 | GOARCH?=amd64 5 | 6 | include vars-gcp.mk 7 | 8 | APP?=service-config-data 9 | APIVER?=v2 10 | RELEASE?=1.5 11 | IMAGE?=${DOCKER_ORG}/${APP}:${RELEASE} 12 | 13 | ENV?=DEV 14 | 15 | K8S_CHART?=service-config 16 | K8S_NAMESPACE?=dev 17 | NODESELECTOR?=services 18 | 19 | helm: 20 | kubectl create serviceaccount --namespace kube-system tiller 21 | kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller 22 | helm init --service-account tiller --upgrade 23 | 24 | clean: 25 | rm -f ${APP} 26 | 27 | build: clean 28 | echo "GOPATH: " ${GOPATH} 29 | CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build \ 30 | -ldflags "-s -w -X ${PROJECT}/version.Release=${RELEASE} \ 31 | -X ${PROJECT}/version.Commit=${COMMIT} -X ${PROJECT}/version.BuildTime=${BUILD_TIME}" \ 32 | -o ${APP} 33 | 34 | run: container 35 | docker stop ${APP} || true && docker rm ${APP} || true 36 | docker run --name ${APP} -p ${PORT}:${PORT} --rm \ 37 | -e "PORT=${PORT}" \ 38 | $(IMAGE) 39 | 40 | push: container 41 | docker push $(IMAGE) 42 | 43 | container: build 44 | # generate dockerfile from template 45 | for t in $(shell find ./src/github.com/oleggorj/service-common-lib/docker/ -type f -name "Dockerfile.goservice.template"); do \ 46 | cat $$t | \ 47 | sed -E "s/{{ .PORT }}/$(PORT)/g" | \ 48 | sed -E "s/{{ .ServiceName }}/$(APP)/g"; \ 49 | done > ./Dockerfile 50 | docker build -t $(IMAGE) . 51 | rm Dockerfile 52 | rm -f ${APP} 53 | 54 | deployclean: 55 | helm del --purge "${K8S_CHART}-${K8S_NAMESPACE}" 56 | 57 | deploy: 58 | for t in $(shell find ./charts/${K8S_CHART} -type f -name "values-template.yaml"); do \ 59 | cat $$t | \ 60 | sed -E "s/{{ .ServiceName }}/$(APP)/g" | \ 61 | sed -E "s/{{ .Release }}/$(RELEASE)/g" | \ 62 | sed -E "s/{{ .Env }}/$(ENV)/g" | \ 63 | sed -E "s/{{ .Kube_namespace }}/$(K8S_NAMESPACE)/g" | \ 64 | sed -E "s/{{ .ApiVer }}/$(APIVER)/g" | \ 65 | sed -E "s/{{ .LBPort }}/$(LB_EXTERNAL_PORT)/g" | \ 66 | sed -E "s/{{ .ContainerPort }}/$(PORT)/g" | \ 67 | sed -E "s/{{ .DockerOrg }}/$(DOCKER_ORG)/g"; \ 68 | done > ./charts/${K8S_CHART}/values.yaml 69 | helm install --name "${K8S_CHART}-${K8S_NAMESPACE}" --values ./charts/${K8S_CHART}/values.yaml --namespace ${K8S_NAMESPACE} ./charts/${K8S_CHART}/ 70 | echo "Cleaning up temp files.." && rm ./charts/${K8S_CHART}/values.yaml 71 | kubectl get services --all-namespaces | grep ${APP} 72 | ./scripts/githook.sh ${APP} ${LB_EXTERNAL_PORT} webhook_git ${GITUSER} ${GITREPO} ${K8S_NAMESPACE} 73 | 74 | 75 | .PHONY: glide 76 | glide: 77 | ifeq ($(shell command -v glide 2> /dev/null),) 78 | curl https://glide.sh/get | sh 79 | endif 80 | 81 | .PHONY: deps 82 | deps: glide 83 | glide install 84 | -------------------------------------------------------------------------------- /src/gitutil/git_utility.go: -------------------------------------------------------------------------------- 1 | package gitutil 2 | 3 | 4 | import ( 5 | "fmt" 6 | "config-data-util/memfilesystem" 7 | log "github.com/oleggorj/service-common-lib/common/logging" 8 | 9 | "gopkg.in/src-d/go-billy.v4" 10 | "gopkg.in/src-d/go-billy.v4/memfs" 11 | "gopkg.in/src-d/go-git.v4" 12 | "gopkg.in/src-d/go-git.v4/plumbing" 13 | "gopkg.in/src-d/go-git.v4/plumbing/object" 14 | "time" 15 | //"gopkg.in/src-d/go-git.v4/plumbing/object" 16 | "gopkg.in/src-d/go-git.v4/storage/memory" 17 | "io/ioutil" 18 | "os" 19 | //"time" 20 | ) 21 | 22 | // TODO: Write a stub for both functions 23 | type GitCredentials struct { 24 | RepoName string `json:"repo_name"` 25 | Account string `json:"account"` 26 | ApiToken string `json:"api_token"` 27 | } 28 | 29 | func GetRepoFromGit(gitAccount, apiToken, repoName, branch string) (billy.Filesystem, *git.Repository, error) { 30 | url := "" 31 | if gitAccount != "" { 32 | url = fmt.Sprintf("https://%s:%s@%s", gitAccount, apiToken, repoName) 33 | }else{ 34 | url = fmt.Sprintf("https://%s", repoName) 35 | } 36 | log.Info("Cloning " + url) 37 | 38 | fs := memfs.New() 39 | storer := memory.NewStorage() 40 | repo, err := git.Clone(storer, fs, &git.CloneOptions{ 41 | URL: url, 42 | ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), 43 | SingleBranch: true, 44 | Progress: os.Stdout, 45 | }) 46 | if err != nil { 47 | return nil, nil, err 48 | } 49 | 50 | return fs, repo, nil 51 | } 52 | 53 | func GetFileFromRepo(fs billy.Filesystem, file_name string) ([]byte, error) { 54 | 55 | f, err := fs.Open(file_name) 56 | if err != nil { 57 | log.Info("File ", file_name, " doesn't exist") 58 | log.Fatal(err) 59 | return nil, err 60 | } 61 | 62 | contents, err := ioutil.ReadAll(f) 63 | if err != nil { 64 | log.Info("Problem with reading file contents") 65 | log.Info(err) 66 | } 67 | 68 | return contents, nil 69 | } 70 | 71 | // TODO: Finish this 72 | //func UpdateGitRepo(environment *conf.Environment, envName string) error { 73 | // 74 | // log.Info("Updating it repo") 75 | // 76 | // 77 | // w, _ := (*environment).Repository.Worktree() 78 | // 79 | // err := w.Pull(&git.PullOptions{ 80 | // RemoteName: "test", 81 | // }) 82 | // log.Info("Updating it repo") 83 | // 84 | // return err 85 | //} 86 | 87 | // TODO: Add extract kernels utility from memfs 88 | // - Might be better to but this in a different package 89 | func UpdateFileOnGitRepo(repo git.Repository, fs billy.Filesystem, filePath string) error { 90 | w, _ := (repo).Worktree() 91 | bytes, _ := memfilesystem.ReadFile(fs, filePath) 92 | fmt.Println("AFTER ", string(bytes)) 93 | _ , _ = w.Add(filePath) 94 | 95 | _, err := w.Commit("test commit", &git.CommitOptions{ 96 | Author: &object.Signature{ 97 | Name: "serviceaccount-config-data", 98 | Email: "serviceaccount-config-data@gmail.com", 99 | When: time.Now(), 100 | }, 101 | }) 102 | if err != nil { 103 | return fmt.Errorf("Error comitting data to repo") 104 | } 105 | 106 | err = (repo).Push(&git.PushOptions{}) 107 | if err != nil { 108 | log.Info(err) 109 | return fmt.Errorf("Error pushing data to repo") 110 | } 111 | return nil 112 | } 113 | -------------------------------------------------------------------------------- /src/config-data-util/user/user.go: -------------------------------------------------------------------------------- 1 | package user 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "gopkg.in/src-d/go-billy.v4" 7 | "config-data-util/memfilesystem" 8 | log "github.com/oleggorj/service-common-lib/common/logging" 9 | ) 10 | 11 | type Users []User 12 | 13 | type User struct { 14 | Email string `json:"email"` 15 | UserMetadata MetaData `json:"metadata"` 16 | } 17 | 18 | type MetaData struct { 19 | IsAdmin bool `json:"is_admin"` 20 | ListOfTeamBucketMapping []TeamBucketMapping `json:"list_of_team_bucket_mapping"` 21 | UserBucket string `json:"user_bucket"` 22 | } 23 | 24 | type TeamBucketMapping struct { 25 | CosInstance string `json:"cos_instance"` 26 | TeamName string `json:"team_name"` 27 | BucketName string `json:"bucket_name"` 28 | } 29 | 30 | func (users *Users) CreateAllUsers(usersBytes []byte) error { 31 | 32 | var tempUsers Users 33 | err := json.Unmarshal(usersBytes, &tempUsers) 34 | if err != nil { 35 | log.Info("Problem unmarshalling new user data") 36 | log.Info(err) 37 | return err 38 | } 39 | *users = tempUsers 40 | return nil 41 | 42 | } 43 | 44 | func (users *Users) Create(fs *billy.Filesystem, data []byte) error { 45 | 46 | var newUser User 47 | err := json.Unmarshal(data,&newUser) 48 | if err != nil { 49 | log.Info("Problem unmarshalling new user data") 50 | log.Info(err) 51 | log.Fatal(err) 52 | return err 53 | } 54 | 55 | index, _ := users.Read(newUser.Email) 56 | if index != -1 { 57 | return fmt.Errorf("the user already exists") 58 | } 59 | 60 | *users = append(*users, newUser) 61 | userBytes, _ := json.Marshal(users) 62 | 63 | err = memfilesystem.OverWriteFile(*fs, "users.json", userBytes) 64 | if err != nil { 65 | *users = (*users)[:len(*users) - 1] 66 | return err 67 | } 68 | 69 | return nil 70 | } 71 | 72 | func (users *Users) Read(email string) (int, error){ 73 | 74 | for i := range *users{ 75 | if (*users)[i].Email == email { 76 | return i,nil 77 | } 78 | } 79 | return -1, fmt.Errorf("user not found") 80 | } 81 | 82 | func (users *Users) Update(fs *billy.Filesystem, email string, data []byte) error { 83 | 84 | index, err := users.Read(email) 85 | // TODO: Handle error better 86 | if err != nil { 87 | return err 88 | } 89 | 90 | var updatedUser User 91 | err = json.Unmarshal(data,&updatedUser) 92 | 93 | if err != nil { 94 | return err 95 | } 96 | 97 | (*users)[index] = updatedUser 98 | userBytes, _ := json.Marshal(users) 99 | err = memfilesystem.OverWriteFile(*fs, "users.json", userBytes) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | return nil 105 | } 106 | 107 | func (users *Users) Delete(fs *billy.Filesystem, email string) error { 108 | 109 | i, err := users.Read(email) 110 | if err != nil { 111 | return err 112 | } 113 | // Order doesn't matter so we just swap the last one in for the oen we're deleting and shorten the slice 114 | (*users)[i] = (*users)[len(*users) - 1] 115 | *users = (*users)[:len(*users) - 1] 116 | userBytes, _ := json.Marshal(users) 117 | err = memfilesystem.OverWriteFile(*fs, "users.json", userBytes) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /scripts/githook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SERVICE_NAME=$1 3 | SERVICE_PORT=$2 4 | ENDPOINT=$3 5 | GIT_USER=$4 6 | GIT_REPO=$5 7 | KUBE_NS=$6 8 | 9 | # TODO check args 10 | 11 | # get git tocken 12 | TOKEN=$(jq '.git.access_token' creds.json | tr -d '"') 13 | 14 | source ./scripts/common.sh 15 | 16 | 17 | # ARGS: 18 | # 1 - service name 19 | function wait4external_ip(){ 20 | # waits for LB IP to be provisioned 21 | external_ip="" 22 | sp="/-\|" 23 | echo "Waiting for end point..." 24 | echo -n ' ' 25 | while [ -z $external_ip ]; do 26 | echo -ne "\b${sp:i++%${#sp}:1}" 27 | external_ip=$(kubectl get svc $1 --namespace=$KUBE_NS --template="{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}") 28 | [ -z "$external_ip" ] && sleep 1 29 | done 30 | echo -ne "\n" 31 | ret_val=$external_ip 32 | } 33 | 34 | printf $BLUE && printf $BRIGHT && wait4external_ip $SERVICE_NAME && EXTERNAL_IP=$ret_val && printf $NORMAL 35 | printf $GREEN && printf $BRIGHT && echo "Service is running on IP: ${EXTERNAL_IP}:${SERVICE_PORT}" && printf $NORMAL 36 | 37 | # ARGS: 38 | # 1 - Repo 39 | # 2 - user name 40 | # 3 - token 41 | # 4 - IP for hook 42 | function check_githook() { 43 | # list all githooks 44 | ret_val=0 45 | HOOKSLIST=$(${CURL} -s -H "Authorization: token ${3}" -H "Content-Type: application/json" -X GET https://$GITAPIURL/repos/${2}/${1}/hooks ) 46 | if [ 0 -ne $? ]; then 47 | printf $BLUE && printf $RED && printf "Something went wrong.. Curl output: " $HOOKSLIST && printf $NORMAL 48 | ret_val=1 49 | exit 1 50 | fi 51 | 52 | printf $BLUE && printf $BRIGHT 53 | echo "Looking for Githook with IP " ${4} ". Listing existing Githooks:" 54 | for k in $(jq '.[] | .config.url' <<< "$HOOKSLIST"); do 55 | echo "k: " $k 56 | if echo "$k" | grep -q "${4}"; then 57 | echo "Githook with IP " ${4} "already exists, no need to create"; 58 | ret_val=1 59 | fi 60 | done 61 | printf $NORMAL 62 | } 63 | 64 | check_githook $GIT_REPO $GIT_USER $TOKEN $EXTERNAL_IP && githook_flag=$ret_val 65 | if [[ $githook_flag -eq 1 ]] ; then 66 | exit 0 67 | fi 68 | 69 | # ARGS: 70 | # 1 - Repo 71 | # 2 - user name 72 | # 3 - token 73 | # 4 - IP for hook 74 | # 5 - Githook endpoint 75 | # 6 - service port (8000) 76 | function create_githook() { 77 | # generates json body for githook 78 | for t in $(find ./ -type f -name "hook.template"); do \ 79 | cat $t | \ 80 | sed -E "s/{{ .WebhookGitSecret }}/$(jq '.git.git_hook_secret' creds.json | tr -d '"')/g" | \ 81 | sed -E "s/{{ .WebhookGit }}/${5}/g" | \ 82 | sed -E "s/{{ .ServiceIp }}/${4}/g" | \ 83 | sed -E "s/{{ .ServicePort }}/${6}/g"; \ 84 | done > ./hook.json 85 | # create githook 86 | HOOK=$(${CURL} -s -H "Authorization: token ${3}" -H "Content-Type: application/json" -X POST -d @hook.json https://$GITAPIURL/repos/${2}/${1}/hooks) 87 | if jq -e .type >/dev/null 2>&1 <<<"$HOOK"; then 88 | printf $GREEN && printf $BRIGHT && echo "Githook created successfully: " $HOOK && printf $NORMAL 89 | ret_val=0 90 | else 91 | printf $RED && printf $BRIGHT && echo "Something went wrong. Message from curl call: " $HOOK && printf $NORMAL 92 | ret_val=1 93 | fi 94 | rm hook.json 95 | } 96 | 97 | create_githook $GIT_REPO $GIT_USER $TOKEN $EXTERNAL_IP $ENDPOINT $SERVICE_PORT && githook_status=$ret_val 98 | #echo $githook_status 99 | -------------------------------------------------------------------------------- /src/config-data-util/user/user_test.go: -------------------------------------------------------------------------------- 1 | package user 2 | 3 | import ( 4 | "memfilesystem" 5 | 6 | //"encoding/json" 7 | "fmt" 8 | "gopkg.in/src-d/go-billy.v4" 9 | "gopkg.in/src-d/go-billy.v4/memfs" 10 | "gotest.tools/assert" 11 | "io/ioutil" 12 | "os" 13 | "testing" 14 | ) 15 | 16 | type TestingEnv struct { 17 | fs billy.Filesystem 18 | users Users 19 | } 20 | 21 | func generateUsers() *TestingEnv{ 22 | 23 | path := "users.json" 24 | 25 | fs := memfs.New() 26 | f, _ := fs.Create(path) 27 | 28 | jsonFile, err := os.Open("../../gitutil/test_data/test_users1.json") 29 | 30 | if err != nil { 31 | fmt.Println("Issue with the file being read") 32 | print(err) 33 | } 34 | 35 | jsonBytes, _ := ioutil.ReadAll(jsonFile) 36 | listOfUsers := Users{} 37 | 38 | err = (&listOfUsers).CreateAllUsers(jsonBytes) 39 | 40 | if err != nil { 41 | fmt.Println(err) 42 | } 43 | _, err = f.Write(jsonBytes) 44 | 45 | if err != nil { 46 | fmt.Println(err) 47 | } 48 | 49 | testEnv := &TestingEnv{ 50 | fs: fs, 51 | users: listOfUsers, 52 | } 53 | return testEnv 54 | } 55 | 56 | 57 | 58 | func TestUser_Delete(t *testing.T) { 59 | 60 | testEnv := generateUsers() 61 | users := testEnv.users 62 | fs := testEnv.fs 63 | 64 | assert.Equal(t, len(users),3 ) // test initial size 65 | _ = users.Delete(fs,"abdullah@ibm.com") 66 | assert.Equal(t, len(users), 2) 67 | index, err := users.Read("abdullah@ibm.com") 68 | 69 | assert.Equal(t,index,-1) // test that correct user was deleted 70 | 71 | if err == nil { 72 | fmt.Println("error should be here, since user should not exist any more") 73 | t.Failed() 74 | } 75 | } 76 | 77 | func TestUser_Update(t *testing.T) { 78 | 79 | testEnv := generateUsers() 80 | users := testEnv.users 81 | fs := testEnv.fs 82 | 83 | newUserFile, _ := os.Open("../../gitutil/test_data/user.json") 84 | newUser, _ := ioutil.ReadAll(newUserFile) 85 | 86 | _ = users.Update(fs,"abdullah@ibm.com", newUser) 87 | 88 | index, _ := users.Read("abdullah@ibm.com") 89 | 90 | assert.Equal(t, index, -1) 91 | 92 | } 93 | 94 | func TestUser_Create(t *testing.T) { 95 | 96 | testEnv := generateUsers() 97 | users := testEnv.users 98 | fs := testEnv.fs 99 | 100 | newUserFile, _ := os.Open("../../gitutil/test_data/user.json") 101 | newUser, _ := ioutil.ReadAll(newUserFile) 102 | 103 | initialLength := len(users) 104 | 105 | originalBytes, err := memfilesystem.ReadFile(fs, "users.json") 106 | 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | fmt.Println("ORIGINAL ", originalBytes) 112 | 113 | _ = users.Create(fs, newUser) 114 | 115 | 116 | afterBytes, err := memfilesystem.ReadFile(fs, "users.json") 117 | 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | 122 | fmt.Println("AFTER ", afterBytes) 123 | 124 | 125 | 126 | //fmt.Println(string(afterBytes)) 127 | 128 | 129 | assert.Equal(t, len(users), initialLength+1) 130 | assert.Equal(t, users[len(users)-1].Email, "newUser@ibm.com") 131 | 132 | } 133 | 134 | func TestUser_Read(t *testing.T) { 135 | 136 | testEnv := generateUsers() 137 | users := testEnv.users 138 | 139 | userToFind := "johnny@ibm.com" 140 | index, err := users.Read(userToFind) 141 | if err != nil { 142 | t.Fatal(err) 143 | } 144 | foundUser := testEnv.users[index] 145 | 146 | assert.Equal(t, foundUser.Email, userToFind) 147 | 148 | } 149 | -------------------------------------------------------------------------------- /charts/service-config/templates/image-pull-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secretsibmcloud.enabled }} 2 | --- 3 | kind: Secret 4 | apiVersion: v1 5 | metadata: 6 | name: {{ .Values.secretsibmcloud.imagePullSecrets }} 7 | namespace: "{{ .Values.service.K8s_namespace }}" 8 | data: 9 | ".dockerconfigjson": eyJhdXRocyI6eyJyZWdpc3RyeS5uZy5ibHVlbWl4Lm5ldCI6eyJ1c2VybmFtZSI6InRva2VuIiwicGFzc3dvcmQiOiJleUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKcWRHa2lPaUk1TkRNNE9XWXhaaTFoWm1KbUxUVmxNRFl0T1RaaFppMWhNemd5T1RFME5UUXlZakFpTENKcGMzTWlPaUp5WldkcGMzUnllUzV1Wnk1aWJIVmxiV2w0TG01bGRDSjkuU2NoVjVZUGV5QWdxZHZJdWVtSVU0MUJrek1TWWtLSUpRakttRDlwbEw0MCIsImVtYWlsIjoiYUBiLmMiLCJhdXRoIjoiZEc5clpXNDZaWGxLYUdKSFkybFBhVXBKVlhwSk1VNXBTWE5KYmxJMVkwTkpOa2xyY0ZoV1EwbzVMbVY1U25Ga1IydHBUMmxKTlU1RVRUUlBWMWw0V21reGFGcHRTbTFNVkZac1RVUlpkRTlVV21oYWFURm9UWHBuZVU5VVJUQk9WRkY1V1dwQmFVeERTbkJqTTAxcFQybEtlVnBYWkhCak0xSjVaVk0xZFZwNU5XbGlTRlpzWWxkc05FeHROV3hrUTBvNUxsTmphRlkxV1ZCbGVVRm5jV1IyU1hWbGJVbFZOREZDYTNwTlUxbHJTMGxLVVdwTGJVUTVjR3hNTkRBPSJ9fX0= 10 | type: kubernetes.io/dockerconfigjson 11 | 12 | --- 13 | kind: Secret 14 | apiVersion: v1 15 | metadata: 16 | name: bluemix-default-secret-regional 17 | namespace: "{{ .Values.service.K8s_namespace }}" 18 | data: 19 | ".dockerconfigjson": eyJhdXRocyI6eyJyZWdpc3RyeS5uZy5ibHVlbWl4Lm5ldCI6eyJ1c2VybmFtZSI6InRva2VuIiwicGFzc3dvcmQiOiJleUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKcWRHa2lPaUpsTURZM09EUXdPQzFtTkRSakxUVTBPRFl0WW1NME1DMWhNelE1WkRoa1pESmlPVFVpTENKcGMzTWlPaUp5WldkcGMzUnllUzV1Wnk1aWJIVmxiV2w0TG01bGRDSjkudVppczBSeDhGcnlNbkJxTzhjNkVUcHVUQ1pnMmdrRnNXbWE1ZURpd2RGVSIsImVtYWlsIjoidG9rZW4iLCJhdXRoIjoiZEc5clpXNDZaWGxLYUdKSFkybFBhVXBKVlhwSk1VNXBTWE5KYmxJMVkwTkpOa2xyY0ZoV1EwbzVMbVY1U25Ga1IydHBUMmxLYkUxRVdUTlBSRkYzVDBNeGJVNUVVbXBNVkZVd1QwUlpkRmx0VFRCTlF6Rm9UWHBSTlZwRWFHdGFSRXBwVDFSVmFVeERTbkJqTTAxcFQybEtlVnBYWkhCak0xSjVaVk0xZFZwNU5XbGlTRlpzWWxkc05FeHROV3hrUTBvNUxuVmFhWE13VW5nNFJuSjVUVzVDY1U4NFl6WkZWSEIxVkVOYVp6Sm5hMFp6VjIxaE5XVkVhWGRrUmxVPSJ9fX0= 20 | type: kubernetes.io/dockerconfigjson 21 | 22 | --- 23 | kind: Secret 24 | apiVersion: v1 25 | metadata: 26 | name: bluemix-default-secret-international 27 | namespace: "{{ .Values.service.K8s_namespace }}" 28 | data: 29 | ".dockerconfigjson": eyJhdXRocyI6eyJyZWdpc3RyeS5ibHVlbWl4Lm5ldCI6eyJ1c2VybmFtZSI6InRva2VuIiwicGFzc3dvcmQiOiJleUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKcWRHa2lPaUpsTldFeVptTTRZeTB3WWpFMUxUVXhZbUl0T1RrMll5MDNaVGhsTVRnM1pHRXlNMlVpTENKcGMzTWlPaUp5WldkcGMzUnllUzVpYkhWbGJXbDRMbTVsZENKOS5sMjd0dW1ZU2tPeXgwZXZvc0ZvamRfTmRTM1dsUjlGOEs0enNUajJPOTdJIiwiZW1haWwiOiJ0b2tlbiIsImF1dGgiOiJkRzlyWlc0NlpYbEthR0pIWTJsUGFVcEpWWHBKTVU1cFNYTkpibEkxWTBOSk5rbHJjRmhXUTBvNUxtVjVTbkZrUjJ0cFQybEtiRTVYUlhsYWJVMDBXWGt3ZDFscVJURk1WRlY0V1cxSmRFOVVhekpaZVRBeldsUm9iRTFVWnpOYVIwVjVUVEpWYVV4RFNuQmpNMDFwVDJsS2VWcFhaSEJqTTFKNVpWTTFhV0pJVm14aVYydzBURzAxYkdSRFNqa3ViREkzZEhWdFdWTnJUM2w0TUdWMmIzTkdiMnBrWDA1a1V6TlhiRkk1UmpoTE5IcHpWR295VHprM1NRPT0ifX19 30 | type: kubernetes.io/dockerconfigjson 31 | 32 | --- 33 | kind: Secret 34 | apiVersion: v1 35 | metadata: 36 | name: bluemix-default-secret 37 | namespace: "{{ .Values.service.K8s_namespace }}" 38 | data: 39 | ".dockerconfigjson": eyJhdXRocyI6eyJyZWdpc3RyeS5uZy5ibHVlbWl4Lm5ldCI6eyJ1c2VybmFtZSI6InRva2VuIiwicGFzc3dvcmQiOiJleUpoYkdjaU9pSklVekkxTmlJc0luUjVjQ0k2SWtwWFZDSjkuZXlKcWRHa2lPaUpsTURZM09EUXdPQzFtTkRSakxUVTBPRFl0WW1NME1DMWhNelE1WkRoa1pESmlPVFVpTENKcGMzTWlPaUp5WldkcGMzUnllUzV1Wnk1aWJIVmxiV2w0TG01bGRDSjkudVppczBSeDhGcnlNbkJxTzhjNkVUcHVUQ1pnMmdrRnNXbWE1ZURpd2RGVSIsImVtYWlsIjoidG9rZW4iLCJhdXRoIjoiZEc5clpXNDZaWGxLYUdKSFkybFBhVXBKVlhwSk1VNXBTWE5KYmxJMVkwTkpOa2xyY0ZoV1EwbzVMbVY1U25Ga1IydHBUMmxLYkUxRVdUTlBSRkYzVDBNeGJVNUVVbXBNVkZVd1QwUlpkRmx0VFRCTlF6Rm9UWHBSTlZwRWFHdGFSRXBwVDFSVmFVeERTbkJqTTAxcFQybEtlVnBYWkhCak0xSjVaVk0xZFZwNU5XbGlTRlpzWWxkc05FeHROV3hrUTBvNUxuVmFhWE13VW5nNFJuSjVUVzVDY1U4NFl6WkZWSEIxVkVOYVp6Sm5hMFp6VjIxaE5XVkVhWGRrUmxVPSJ9fX0= 40 | type: kubernetes.io/dockerconfigjson 41 | 42 | 43 | {{- end }} 44 | -------------------------------------------------------------------------------- /src/gitutil/git_utility_test.go: -------------------------------------------------------------------------------- 1 | package gitutil 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | conf "config-data-util" 7 | userutil "config-data-util/user" 8 | "gopkg.in/src-d/go-billy.v4" 9 | "gopkg.in/src-d/go-git.v4" 10 | "io/ioutil" 11 | // "io/ioutil" 12 | "os" 13 | "path/filepath" 14 | "testing" 15 | ) 16 | 17 | type config_git struct { 18 | name_value string 19 | api_token_value string 20 | } 21 | 22 | type Testwhitelist struct { 23 | Whitelist []string `json:"whitelist"` 24 | Admin []string `json:"admin"` 25 | } 26 | 27 | /* 28 | * Grab the jhub-whitelist.json from git repo config-data 29 | * compare that with the sample sandbox_whitelist.json which is the same file as jhub-whitelist.json 30 | * Expect the two to be the same 31 | */ 32 | 33 | // TODO: Rewrrite this test because it fails now due to repo change 34 | // - Should use the users.json possibly or some file for testing 35 | func TestGetGitRepoConfigFile(t *testing.T) { 36 | // read file that is the exact same as one coming from repository whitelist.json 37 | absPath, _ := filepath.Abs("./test_data/sandbox_whitelist.json") 38 | jsonFile, err := os.Open(absPath) 39 | if err != nil { 40 | fmt.Println("Issue with the file being read") 41 | print(err) 42 | t.Fail() 43 | } 44 | jsonByteValue, _ := ioutil.ReadAll(jsonFile) 45 | 46 | var GitCredentials GitCredentials 47 | credentialsFile, _ := ioutil.ReadFile("../credentials.json") 48 | json.Unmarshal(credentialsFile, &GitCredentials) 49 | fmt.Println(GitCredentials) 50 | var whiteList Testwhitelist 51 | json.Unmarshal(jsonByteValue, &whiteList) 52 | fmt.Println(whiteList) 53 | fs, _, _ := GetRepoFromGit(GitCredentials.Account, GitCredentials.ApiToken, GitCredentials.RepoName, "sandbox") 54 | 55 | //// TODO: change this to work from the test branch, since the following code doesn't exist any more 56 | contents, _ := GetFileFromRepo(fs, "jhub-whitelist.json") 57 | var gitWhiteList Testwhitelist 58 | json.Unmarshal(contents, &gitWhiteList) 59 | fmt.Println(gitWhiteList) 60 | if gitWhiteList.Whitelist[0] != whiteList.Whitelist[0] { 61 | t.Errorf("The white lists do not equal to one another") 62 | } 63 | if len(gitWhiteList.Whitelist) != len(whiteList.Whitelist) { 64 | t.Errorf("The white lists do not equal to one another") 65 | } 66 | if len(gitWhiteList.Admin) != len(whiteList.Admin) { 67 | t.Errorf("The white lists do not equal to one another") 68 | } 69 | if gitWhiteList.Admin[2] != whiteList.Admin[2] { 70 | t.Errorf("The white lists do not equal to one another") 71 | } 72 | } 73 | 74 | func CreateTestingEnvironment(repo *git.Repository, fs billy.Filesystem, filePath string) map[string]*conf.Environment { 75 | absPath, _ := filepath.Abs(filePath) 76 | jsonFile, err := os.Open(absPath) 77 | if err != nil { 78 | fmt.Println("Issue with the file being read") 79 | print(err) 80 | } 81 | json_byte_value, _ := ioutil.ReadAll(jsonFile) 82 | users := userutil.CreateAllUsers(json_byte_value, nil) 83 | testEnvironment := make(map[string]*conf.Environment) 84 | testEnvironment["test"] = &conf.Environment{ 85 | Name: "test", 86 | FileSystem: fs, 87 | Repository: repo, 88 | Users: users, 89 | } 90 | 91 | return testEnvironment 92 | } 93 | 94 | func TestUpdateFileOnGitRepo(t *testing.T) { 95 | var GitCredentials GitCredentials 96 | credentialsFile, _ := ioutil.ReadFile("../credentials.json") 97 | err := json.Unmarshal(credentialsFile, &GitCredentials) 98 | if err != nil { 99 | _ = fmt.Errorf("some issue with unmarshalling credentials") 100 | } 101 | fs, repo, _ := GetRepoFromGit(GitCredentials.Account, GitCredentials.ApiToken, GitCredentials.RepoName, "test") 102 | testEnvironment := CreateTestingEnvironment(repo, fs, "./test_data/test_users2.json") 103 | fmt.Println(len(testEnvironment["test"].Users)) 104 | newUser := conf.User{ 105 | Email: "newuser@ibm.com", 106 | UserMetadata: conf.UserMetaData{ 107 | IsAdmin: true, 108 | ListOfTeamBucketMapping: nil, 109 | UserBucket: "", 110 | }, 111 | } 112 | testEnvironment["test"].Users = append(testEnvironment["test"].Users, newUser) 113 | fmt.Println(len(testEnvironment["test"].Users)) 114 | 115 | err = UpdateFileOnGitRepo(testEnvironment["test"], "users.json") 116 | if err != nil { 117 | _ = fmt.Errorf("some issue with my udate") 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/handlers/userHandler.go: -------------------------------------------------------------------------------- 1 | // Package handlers holds all of the handlers used by the service-config-data service 2 | package handlers 3 | 4 | import ( 5 | "encoding/json" 6 | "fmt" 7 | "github.com/gorilla/mux" 8 | log "github.com/oleggorj/service-common-lib/common/logging" 9 | conf "config-data-util" 10 | "config-data-util/user" 11 | "config-data-util/memfilesystem" 12 | "gitutil" 13 | "net/http" 14 | "strings" 15 | ) 16 | 17 | type UsersHandler struct { 18 | Environments conf.MappingToEnv 19 | } 20 | 21 | type UserHandler struct { 22 | Environments conf.MappingToEnv 23 | } 24 | 25 | type UserRequestResponse struct { 26 | Data interface{} `json:"data"` 27 | Err string `json:"err"` 28 | } 29 | 30 | func (u *UsersHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 31 | 32 | rw.Header().Set("Content-Type", "application/json") 33 | 34 | envValue := strings.ToLower(mux.Vars(req)["environment"]) 35 | environment := u.Environments[envValue] 36 | 37 | resp := UserRequestResponse{} 38 | 39 | if environment == nil { 40 | resp.Err = fmt.Sprintf("Environment '%s' not found", envValue) 41 | _ = json.NewEncoder(rw).Encode(resp) 42 | rw.WriteHeader(http.StatusBadRequest) 43 | return 44 | } 45 | 46 | resp.Data = environment.Users 47 | _ = json.NewEncoder(rw).Encode(resp) 48 | 49 | rw.WriteHeader(http.StatusOK) 50 | 51 | } 52 | 53 | // TODO: Notice some redundancies in how error responses are being thrown, need to fix that 54 | 55 | func (u *UserHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 56 | 57 | // TODO: Some repeated code here might want to refactor this later 58 | // - Env checking is going to be common between all handlers 59 | rw.Header().Set("Content-Type", "application/json") 60 | 61 | envValue := strings.ToLower(mux.Vars(req)["environment"]) 62 | environment := u.Environments[envValue] 63 | 64 | resp := UserRequestResponse{} 65 | 66 | if environment == nil { 67 | resp.Err = fmt.Sprintf("Environment '%s' not found", envValue) 68 | _ = json.NewEncoder(rw).Encode(resp) 69 | rw.WriteHeader(http.StatusBadRequest) 70 | return 71 | } 72 | 73 | userEmail := strings.ToLower(mux.Vars(req)["email"]) 74 | log.Info("User: ", userEmail, " was Requested") 75 | 76 | users := &environment.Users 77 | fs := &environment.FileSystem 78 | repo := environment.Repository 79 | 80 | bytes, _ := memfilesystem.ReadFile(*fs, "users.json") 81 | fmt.Println("AFTER ", string(bytes)) 82 | 83 | indexOfUser, err := users.Read(userEmail) 84 | 85 | // Check if user requested exists if not a POST request 86 | if req.Method != http.MethodPost && err != nil { 87 | resp.Err = fmt.Sprintf("User '%s' not found", userEmail) 88 | _ = json.NewEncoder(rw).Encode(resp) 89 | rw.WriteHeader(http.StatusBadRequest) 90 | return 91 | } 92 | 93 | if req.Method == http.MethodGet { 94 | // read a user 95 | 96 | resp.Data = environment.Users[indexOfUser] 97 | _ = json.NewEncoder(rw).Encode(resp) 98 | rw.WriteHeader(http.StatusOK) 99 | 100 | } else { 101 | 102 | switch req.Method { 103 | 104 | // TODO: Some redundancies in code for put and post need to write method to fix that 105 | case http.MethodPost: 106 | // create a user 107 | // Questions: Do we want to create a user with just the new email. Or should we accept the whole body? 108 | 109 | // Check if already created 110 | 111 | newUser := user.User{} 112 | _ = json.NewDecoder(req.Body).Decode(&newUser) 113 | 114 | if newUser.Email == "" || newUser.Email != userEmail{ 115 | resp.Err = fmt.Sprintf("User '%s' in route does not match '%s' in body", newUser.Email, userEmail) 116 | _ = json.NewEncoder(rw).Encode(resp) 117 | rw.WriteHeader(http.StatusBadRequest) 118 | return 119 | } 120 | 121 | newUserBytes, _ := json.Marshal(newUser) 122 | err := users.Create(fs, newUserBytes) 123 | 124 | if err != nil { 125 | resp.Err = fmt.Sprintf("Error: %s", err) 126 | _ = json.NewEncoder(rw).Encode(resp) 127 | rw.WriteHeader(http.StatusInternalServerError) 128 | return 129 | } 130 | 131 | resp.Data = fmt.Sprintf("Successfully created '%s'", userEmail) 132 | 133 | case http.MethodPut: 134 | 135 | updatedUser := user.User{} 136 | _ = json.NewDecoder(req.Body).Decode(&updatedUser) 137 | 138 | // Check that the email in route matches body of request 139 | if updatedUser.Email != userEmail { 140 | resp.Err = fmt.Sprintf("User '%s' in route does not match '%s' in body", updatedUser.Email, userEmail) 141 | _ = json.NewEncoder(rw).Encode(resp) 142 | rw.WriteHeader(http.StatusBadRequest) 143 | return 144 | } 145 | 146 | updatedUserBytes, _ := json.Marshal(updatedUser) 147 | err := users.Update(fs, userEmail, updatedUserBytes) 148 | 149 | if err != nil { 150 | resp.Err = fmt.Sprintf("Something went wrong updating user") 151 | _ = json.NewEncoder(rw).Encode(resp) 152 | rw.WriteHeader(http.StatusInternalServerError) 153 | return 154 | } 155 | 156 | resp.Data = fmt.Sprintf("Successfully updated '%s'", userEmail) 157 | 158 | case http.MethodDelete: 159 | 160 | err := users.Delete(fs, userEmail) 161 | 162 | if err != nil { 163 | resp.Err = fmt.Sprintf("There was an internal error") 164 | _ = json.NewEncoder(rw).Encode(resp) 165 | rw.WriteHeader(http.StatusInternalServerError) 166 | return 167 | } 168 | 169 | resp.Data = fmt.Sprintf("Successfully deleted '%s'", userEmail) 170 | } 171 | 172 | // TODO: Fix git push issue 173 | err = gitutil.UpdateFileOnGitRepo(*repo, *fs, "users.json") 174 | if err != nil { 175 | resp.Data = fmt.Sprint(resp.Data) + fmt.Sprintf("but %s", err) 176 | } 177 | _ = json.NewEncoder(rw).Encode(resp) 178 | rw.WriteHeader(http.StatusOK) 179 | 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/handlers/userHandler_test.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | //"bytes" 5 | "encoding/json" 6 | "fmt" 7 | conf "config-data-util" 8 | "config-data-util/environment" 9 | "config-data-util/user" 10 | "gotest.tools/assert" 11 | "github.com/gorilla/mux" 12 | "gopkg.in/src-d/go-billy.v4/memfs" 13 | "io/ioutil" 14 | "net/http" 15 | "net/http/httptest" 16 | "os" 17 | "testing" 18 | is "gotest.tools/assert/cmp" 19 | ) 20 | 21 | 22 | 23 | func generateUsers() *environment.Environment{ 24 | 25 | path := "users.json" 26 | 27 | fs := memfs.New() 28 | f, _ := fs.Create(path) 29 | 30 | jsonFile, err := os.Open("../gitutil/test_data/test_users1.json") 31 | 32 | if err != nil { 33 | fmt.Println("Issue with the file being read") 34 | fmt.Println(err) 35 | return nil 36 | } 37 | 38 | jsonBytes, _ := ioutil.ReadAll(jsonFile) 39 | listOfUsers := user.Users{} 40 | 41 | err = (&listOfUsers).CreateAllUsers(jsonBytes) 42 | 43 | if err != nil { 44 | fmt.Println(err) 45 | } 46 | _, err = f.Write(jsonBytes) 47 | 48 | if err != nil { 49 | fmt.Println(err) 50 | } 51 | 52 | testEnv := &environment.Environment{ 53 | FileSystem: fs, 54 | Users: listOfUsers, 55 | } 56 | return testEnv 57 | } 58 | 59 | 60 | 61 | func TestUsersHandler_ServeHTTP(t *testing.T) { 62 | 63 | sampleEnvironmentMapping := make(conf.MappingToEnv) 64 | sampleEnvironmentMapping["test"] = generateUsers() 65 | 66 | req, err := http.NewRequest("GET", "/api/v1/configs/test/users", nil) 67 | 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | rr := httptest.NewRecorder() 73 | handler := http.Handler(&UsersHandler{Environments: sampleEnvironmentMapping}) 74 | 75 | router := mux.NewRouter() 76 | router.Handle("/api/v1/configs/{environment}/users", handler).Methods("GET") 77 | router.ServeHTTP(rr, req) 78 | 79 | assert.Equal(t, rr.Code, http.StatusOK) 80 | 81 | 82 | var response UserRequestResponse 83 | err = json.Unmarshal(rr.Body.Bytes(), &response) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | userBytes, _ := json.Marshal(response.Data) 89 | var usersArray user.Users 90 | _ = json.Unmarshal(userBytes, &usersArray) 91 | 92 | assert.Assert(t, is.Len(usersArray, 3)) 93 | } 94 | 95 | //func TestUserHandlerGet_ServeHTTP(t *testing.T) { 96 | // 97 | // sampleEnvironmentMapping := usersEnvGenHelper() 98 | // 99 | // req, err := http.NewRequest("GET", "/api/v1/configs/test/users/abdullah@ibm.com", nil) 100 | // 101 | // if err != nil { 102 | // t.Fatal(err) 103 | // } 104 | // 105 | // rr := httptest.NewRecorder() 106 | // handler := http.Handler(&UserHandler{Environments: sampleEnvironmentMapping}) 107 | // 108 | // router := mux.NewRouter() 109 | // router.Handle("/api/v1/configs/{environment}/users/{email}", handler).Methods("GET") 110 | // router.ServeHTTP(rr, req) 111 | // 112 | // var usersArray conf.User 113 | // err = json.Unmarshal(rr.Body.Bytes(), &usersArray) 114 | // 115 | // 116 | // assert.Equal(t, usersArray.Email, sampleEnvironmentMapping["test"].Users[0].Email) 117 | // 118 | // 119 | // if err != nil { 120 | // t.Fatal(err) 121 | // } 122 | // 123 | // fmt.Println(usersArray) 124 | // fmt.Println(rr.Body) 125 | // 126 | //} 127 | 128 | //func TestUserHandlerPost_ServeHTTP(t *testing.T) { 129 | // 130 | // reqType := "POST" 131 | // 132 | // userToUpdate := []byte(`{"email":"abdullah@ibm.com","metadata":{"is_admin":false,"list_of_team_bucket_mapping":[{"cos_instance":"client2_instance","team_name":"fun_team_2","bucket_name":"sample_bucket_2"}],"user_bucket":"abdullas_bucket"}}`) 133 | // 134 | // 135 | // req, err := http.NewRequest(reqType, "/api/v1/configs/test/users/abdullah@ibm.com", bytes.NewBuffer(userToUpdate)) 136 | // if err != nil { 137 | // t.Fatal(err) 138 | // } 139 | // req.Header.Set("Content-Type", "application/json") 140 | // 141 | // sampleEnvironmentMapping := usersEnvGenHelper() 142 | // 143 | // rr := httptest.NewRecorder() 144 | // handler := http.Handler(&UserHandler{Environments: sampleEnvironmentMapping}) 145 | // 146 | // router := mux.NewRouter() 147 | // router.Handle("/api/v1/configs/{environment}/users/{email}", handler).Methods(reqType) 148 | // router.ServeHTTP(rr, req) 149 | // 150 | // fmt.Println("TEST ", rr.Body) 151 | // var usersArray conf.User 152 | // err = json.Unmarshal(rr.Body.Bytes(), &usersArray) 153 | // if err != nil { 154 | // t.Fatal(err) 155 | // } 156 | // 157 | // fmt.Println(usersArray) 158 | // fmt.Println(rr.Body) 159 | // 160 | //testEnv := generateUsers() 161 | 162 | //newUser = conf.User{ 163 | // Email: "bobby@ibm.com", 164 | // UserMetadata: conf.UserMetaData{ 165 | // IsAdmin: false, 166 | // UserBucket: "", 167 | // }, 168 | //} 169 | //userBytes, _ := json.Marshal(user) 170 | 171 | // Have to add user to array as well 172 | //testEnv.Users = append(testEnv.Users, newUser) 173 | // 174 | //usersBytes, _ := json.Marshal(testEnv.Users) 175 | //_ = testEnv.FileSystem.Remove("users.json") 176 | //f, _ := testEnv.FileSystem.Create("users.json") 177 | //_, _ = f.Write(usersBytes) 178 | // 179 | //f, _ = testEnv.FileSystem.Open("users.json") 180 | //contents, _ := ioutil.ReadAll(f) 181 | //var usersStruct []conf.User 182 | //json.Unmarshal(contents, &usersStruct) 183 | //fmt.Println(usersStruct) 184 | 185 | // Must also has them to the filesystem so that a git commit can occur 186 | 187 | 188 | //req, err := http.NewRequest("GET", "/api/v1/configs/test/users/abdullah@ibm.com", nil) 189 | // 190 | //if err != nil { 191 | // t.Fatal(err) 192 | //} 193 | // 194 | //rr := httptest.NewRecorder() 195 | //handler := http.Handler(&UserHandler{Environments: sampleEnvironmentMapping}) 196 | // 197 | //router := mux.NewRouter() 198 | //router.Handle("/api/v1/configs/{environment}/users/{email}", handler).Methods("GET") 199 | //router.ServeHTTP(rr, req) 200 | //var usersArray conf.User 201 | //err = json.Unmarshal(rr.Body.Bytes(), &usersArray) 202 | //if err != nil { 203 | // t.Fatal(err) 204 | //} 205 | // 206 | //fmt.Println(usersArray) 207 | //fmt.Println(rr.Body) 208 | 209 | //} 210 | 211 | // 212 | //func usersEnvGenHelper() map[string]*conf.Environment { 213 | // userFile := helpers.ReadFromFileToBytes("../gitutil/test_data/test_users1.json") 214 | // sampleEnvironmentMapping := make(map[string]*conf.Environment) 215 | // sampleEnvironmentMapping["test"] = &conf.Environment{ 216 | // Name: "test", 217 | // Users: userutil.CreateAllUsers(userFile, nil), 218 | // } 219 | // return sampleEnvironmentMapping 220 | //} 221 | -------------------------------------------------------------------------------- /service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/oleggorj/service-common-lib/common/config" 7 | log "github.com/oleggorj/service-common-lib/common/logging" 8 | "github.com/oleggorj/service-common-lib/common/util" 9 | "github.com/oleggorj/service-common-lib/service" 10 | "config-data-util/environment" 11 | "handlers" 12 | conf "config-data-util" 13 | "gitutil" 14 | 15 | "github.com/spf13/viper" 16 | "github.com/gorilla/mux" 17 | "io/ioutil" 18 | "net/http" 19 | "os" 20 | "strings" 21 | 22 | "github.com/google/go-github/github" 23 | ) 24 | 25 | var ( 26 | githubRepoName string 27 | githubAccount string 28 | githubApiToken string 29 | servicePort string 30 | serviceApiVersion string 31 | serviceAppName string 32 | // legacy 33 | serviceDebugFlag bool 34 | githubConfigFile string 35 | githubBranch string 36 | configFormat string 37 | 38 | ConfMappingOfEnvs conf.MappingToEnv 39 | confEnvNames []string 40 | ) 41 | 42 | func init() { 43 | 44 | viper.SetConfigName("config") 45 | viper.AddConfigPath(".") ; viper.AddConfigPath("/") 46 | 47 | err := viper.ReadInConfig() 48 | if err != nil { 49 | panic(fmt.Errorf("Fatal error config file: %s \n", err)) 50 | } 51 | 52 | serviceAppName = viper.GetString("service.name") 53 | log.SetAppName(serviceAppName) 54 | args := os.Args 55 | if len(args) == 1 { 56 | 57 | githubRepoName = viper.GetString("service.backend.repo") 58 | if githubRepoName == "" { 59 | log.Fatal("ERROR: REPO name is required") 60 | } 61 | githubAccount = viper.GetString("service.backend.account") 62 | if githubAccount == "" { 63 | log.Warn("warning: GITACCOUNT is required") 64 | } 65 | githubApiToken = viper.GetString("service.backend.token") 66 | if githubApiToken == "" { 67 | log.Warn("warning: git APITOKEN is required") 68 | } 69 | serviceApiVersion = viper.GetString("service.apiver") 70 | if serviceApiVersion == "" { 71 | log.Warn("warning: service APIVER is required") 72 | } 73 | 74 | // init list of branches 75 | confEnvNames = []string{} 76 | err := viper.UnmarshalKey("service.backend.branches", &confEnvNames) 77 | log.Debug("Branches from config: %v, %#v", err, confEnvNames) 78 | 79 | } else if args[1] == "dev" { 80 | 81 | serviceApiVersion = "test" 82 | 83 | var GitCredentials gitutil.GitCredentials 84 | jsonFile, _ := ioutil.ReadFile("credentials.json") 85 | err := json.Unmarshal(jsonFile, &GitCredentials) 86 | 87 | if err != nil { 88 | log.Info("No credentials file found") 89 | log.Fatal(err) 90 | } 91 | 92 | githubRepoName = GitCredentials.RepoName 93 | githubAccount = GitCredentials.Account 94 | githubApiToken = GitCredentials.ApiToken 95 | 96 | confEnvNames = []string{"test"} 97 | 98 | } else { 99 | log.Fatal("ERROR: Invalid arguments passed") 100 | } 101 | 102 | servicePort = viper.GetString("service.port") 103 | if servicePort == "" { 104 | servicePort = "8000" 105 | } 106 | 107 | githubBranch = util.GetENV("GITBRANCH") ; if githubBranch == "" { githubBranch = "sandbox" } 108 | configFormat = util.GetENV("FORMAT") ; if configFormat == "" { configFormat = "json" } 109 | configFile := util.GetENV("CONFIGFILE") ; if configFile == "" { configFile = "services" } 110 | 111 | ConfMappingOfEnvs = make(conf.MappingToEnv) 112 | initializeEnvironment() 113 | } 114 | 115 | 116 | func main() { 117 | 118 | log.Info("Starting service '" + serviceAppName + "'...") 119 | log.Info("INFO: Starting config data service for '", githubRepoName, "' environment.. ") 120 | 121 | // legacy handlers - have to clean up 122 | service.RegisterHandlerFunction("/api/v1/{app}/{env}/{key}", "GET", KeyHandler) 123 | service.RegisterHandlerFunction("/api/v1/{app}/{env}/{key}/{debug}", "GET", KeyHandler) 124 | 125 | // 126 | // v2 handlers 127 | // 128 | service.RegisterHandlerFunction("/api", "GET", ApiHandler) 129 | service.RegisterHandlerFunction("/api/v2/reload", "GET", ApiHandlerReload) 130 | service.RegisterHandler("/api/v2/configs/{environment}/users", "GET", &handlers.UsersHandler{Environments: ConfMappingOfEnvs}) 131 | service.RegisterHandler("/api/v2/configs/{environment}/users/{email}", "GET", &handlers.UserHandler{Environments: ConfMappingOfEnvs}) 132 | service.RegisterHandler("/api/v2/configs/{environment}/users/{email}", "DELETE", &handlers.UserHandler{Environments: ConfMappingOfEnvs}) 133 | service.RegisterHandler("/api/v2/configs/{environment}/users/{email}", "POST", &handlers.UserHandler{Environments: ConfMappingOfEnvs}) 134 | service.RegisterHandler("/api/v2/configs/{environment}/users/{email}", "PUT", &handlers.UserHandler{Environments: ConfMappingOfEnvs}) 135 | service.RegisterHandler("/api/v1/kernels/{environment}", "GET", &handlers.KernelHandler{Environments: ConfMappingOfEnvs}) 136 | 137 | // keys endpoints 138 | service.RegisterHandler("/api/v2/{app}/{env}/{key}", "GET", &handlers.KeyHandler{Environments: ConfMappingOfEnvs}) 139 | service.RegisterHandler("/api/v2/{app}/{env}/{key}/debug", "GET", &handlers.KeyHandler{Environments: ConfMappingOfEnvs}) 140 | 141 | // TODO add endpoints for configmaps 142 | 143 | // endpoint for webhooks 144 | service.RegisterHandlerFunction( viper.GetString("service.backend.webhook_path"), "POST", ApiHandlerWebhooksV2 ) 145 | 146 | service.StartServer(servicePort) 147 | 148 | // TODO: Need to add side-cart container to stream the logs out 149 | // 150 | // TODO: Have better way to log and handle errors to avoid excessive prints 151 | // - https://hackernoon.com/golang-handling-errors-gracefully-8e27f1db729f 152 | } 153 | 154 | func initializeEnvironment() { 155 | 156 | for _, envName := range confEnvNames { 157 | fs, repo, err := gitutil.GetRepoFromGit(githubAccount, githubApiToken, githubRepoName, envName) 158 | if err != nil { 159 | log.Error("-- Branch ", envName, " not intialized. Does it exist?") 160 | log.Error(err) 161 | continue 162 | } else { 163 | log.Info("-- Branch ", envName, " is intialized.") 164 | } 165 | ConfMappingOfEnvs[envName] = &environment.Environment{ 166 | FileSystem: fs, 167 | Repository: repo, 168 | } 169 | } 170 | } 171 | 172 | func ApiHandlerWebhooksV2(w http.ResponseWriter, r *http.Request) { 173 | temp := viper.New() ; temp.SetConfigName("creds") ; temp.AddConfigPath(".") ; viper.AddConfigPath("/") 174 | 175 | payload, err := github.ValidatePayload(r, []byte( temp.GetString("git.git-hook-secret") )) 176 | if err != nil { 177 | log.Error("error reading request body: err=%s\n", err) 178 | return 179 | } 180 | defer r.Body.Close() 181 | //log.Debug("payload: %s", string(payload)) 182 | event, err := github.ParseWebHook( github.WebHookType(r), payload ) 183 | if err != nil { 184 | log.Error("could not parse webhook: err=%s\n", err) 185 | return 186 | } 187 | 188 | switch e := event.(type) { 189 | case *github.PushEvent: // this is a commit push 190 | log.Info("------ User "+ *e.Sender.Login + " made commit to " + *e.Repo.FullName ) 191 | log.Info("------ PushEvent triggered - releading configs from backend") 192 | initializeEnvironment() 193 | w.WriteHeader(http.StatusOK) 194 | 195 | case *github.PullRequestEvent: // this is a pull request 196 | log.Info("PullRequestEvent") 197 | 198 | default: 199 | log.Info("unknown event type %s\n", github.WebHookType(r)) 200 | return 201 | } 202 | 203 | } 204 | 205 | // ---------------------------------------------------------------------------------- 206 | // --- Depricated --- 207 | // Legacy code - needs to be cleaned up 208 | // 209 | func ApiHandler(rw http.ResponseWriter, req *http.Request) { 210 | _, err := rw.Write([]byte(serviceApiVersion)) 211 | if err != nil { 212 | log.Error("ERROR: Variable is not defined properly") 213 | } 214 | rw.WriteHeader(http.StatusOK) 215 | } 216 | 217 | func ApiHandlerReload(rw http.ResponseWriter, req *http.Request) { 218 | 219 | initializeEnvironment() 220 | 221 | _, err := rw.Write([]byte(serviceApiVersion)) 222 | if err != nil { 223 | log.Error("ERROR: Variable is not defined properly") 224 | } 225 | rw.WriteHeader(http.StatusOK) 226 | } 227 | 228 | // 229 | func KeyHandler(rw http.ResponseWriter, req *http.Request) { 230 | serviceDebugFlag = false 231 | rw.Header().Set("Content-Type", "application/json") 232 | 233 | the_app := strings.ToLower(mux.Vars(req)["app"]) 234 | the_env := strings.ToLower(mux.Vars(req)["env"]) 235 | the_key := mux.Vars(req)["key"] 236 | 237 | log.Info("the_app " + the_app) 238 | 239 | if the_app == "" || the_env == "" || the_key == "" { 240 | log.Error("ERROR: , or can not be empty.\n") 241 | rw.WriteHeader(http.StatusNotAcceptable) 242 | return 243 | } 244 | if debug := strings.ToLower(mux.Vars(req)["debug"]); debug == "debug" { serviceDebugFlag = true } 245 | 246 | if serviceDebugFlag == true { 247 | _, _ = rw.Write( []byte( fmt.Sprintf("param `app` = %s\n", the_app) ) ) 248 | _, _ = rw.Write( []byte( fmt.Sprintf("param `env` = %s\n", the_env) ) ) 249 | _, _ = rw.Write( []byte( fmt.Sprintf("param `key` = %s\n", the_key) ) ) 250 | log.Debug(fmt.Sprintf("param `env` = %s, param `key` = %s", the_env, the_key)) 251 | } 252 | 253 | // set config file 254 | githubConfigFile = the_app + ".json" 255 | val, err := getValue(the_key) 256 | if err != nil { rw.WriteHeader(http.StatusInternalServerError) ; return } 257 | _, _ = rw.Write( []byte( val )) 258 | 259 | rw.WriteHeader(http.StatusOK) 260 | } 261 | //@params 262 | // 263 | //@return 264 | // 265 | func getValue(key string) (string, error){ 266 | if serviceDebugFlag == true { 267 | log.Debug( githubAccount," ", githubApiToken, " ",githubRepoName, " ",githubBranch, " ",githubConfigFile) 268 | } 269 | configFile, err := config.GetGitRepoConfigFile(githubAccount, githubApiToken, githubRepoName, githubBranch, githubConfigFile) 270 | if err != nil { return "", fmt.Errorf("ERROR: error retriving configuration: %v", err) } 271 | if configFile == "" { return "", fmt.Errorf("Can not resolve temp file name.") } 272 | 273 | // reading config file into Viper interface 274 | v, err := config.ReadConfig(configFile) 275 | if err != nil { return "", fmt.Errorf("Error when reading config: %v\n", err) } 276 | // debug part of endpoint 277 | if serviceDebugFlag == true { 278 | c := v.AllKeys() 279 | for i := 0; i < len(c) ; i++ { 280 | log.Debug( c[i] + " -> " + fmt.Sprintf("%s", v.Get(c[i])) ) 281 | } 282 | } 283 | 284 | // look up the key and return value 285 | return fmt.Sprintf("%s", v.Get(key)), nil 286 | 287 | } 288 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Service to manage configuration data (service-config-data) 2 | 3 | [![GitHub release](https://img.shields.io/github/release/OlegGorj/service-config-data.svg)](https://github.com/OlegGorj/service-config-data/releases) 4 | [![GitHub issues](https://img.shields.io/github/issues/OlegGorj/service-config-data.svg)](https://github.com/OlegGorj/service-config-data/issues) 5 | [![GitHub commits](https://img.shields.io/github/commits-since/OlegGorj/service-config-data/0.0.1.svg)](https://github.com/OlegGorj/service-config-data) 6 | 7 | [![Build Status](https://travis-ci.org/OlegGorj/service-config-data.svg?branch=master)](https://travis-ci.org/OlegGorj/service-config-data) 8 | [![GitHub Issues](https://img.shields.io/github/issues/OlegGorJ/service-config-data.svg)](https://github.com/OlegGorJ/service-config-data/issues) 9 | [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/OlegGorJ/service-config-data.svg)](http://isitmaintained.com/project/OlegGorJ/service-config-data "Average time to resolve an issue") 10 | [![Percentage of issues still open](http://isitmaintained.com/badge/open/OlegGorJ/service-config-data.svg)](http://isitmaintained.com/project/OlegGorJ/service-config-data "Percentage of issues still open") 11 | [![Docker Stars](https://img.shields.io/docker/stars/oleggorj/service-config-data.svg)](https://hub.docker.com/r/oleggorj/service-config-data/) 12 | [![Docker Pulls](https://img.shields.io/docker/pulls/oleggorj/service-config-data.svg)](https://hub.docker.com/r/oleggorj/service-config-data/) 13 | 14 | [![Docker Build Status](https://img.shields.io/docker/build/oleggorj/service-config-data)](https://hub.docker.com/r/oleggorj/service-config-data/builds/) 15 | 16 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/1818748c6ba745ce97bb43ab6dbbfd2c)](https://www.codacy.com/app/OlegGorj/service-config-data?utm_source=github.com&utm_medium=referral&utm_content=OlegGorj/service-config-data&utm_campaign=Badge_Grade) 17 | 18 | 19 | The `service-config-data` service designed to provide configuration management capabilities across all components. This common service can be used by any application, component or module within permitted boundaries. In it's present implementation, the service allows retrieval of values based on provided key and specified environment (`dev`, `stg`, `prod`, etc.). 20 | 21 | --- 22 | 23 | ## Directory structure 24 | 25 | This is how complete directory structure should look like after installing all repos and resolving dependancies. 26 | 27 | ``` 28 | ├── config-data 29 | │   ├── README.md 30 | │   ├── cassandra.json 31 | │   ├── jhub-whitelist.json 32 | │   ├── jhub_kernels.json 33 | │   ├── k8s-cluster.json 34 | │   ├── service-config-data.json 35 | │   ├── service-scada-load.json 36 | │   ├── services.json 37 | │   └── test.json 38 | ├── pkg 39 | │   ├── darwin_amd64 40 | │   │   └── github.com 41 | │   │   └── tidwall 42 | │   │   └── gjson.a 43 | │   └── dep 44 | │   └── sources 45 | └── service-config-data 46 | ├── JenkinsPod.yaml 47 | ├── Jenkinsfile 48 | ├── Makefile 49 | ├── README.md 50 | ├── SECURITY.md 51 | ├── charts 52 | │   └── service-config 53 | │   ├── Chart.yaml 54 | │   ├── requirements.yaml 55 | │   ├── templates 56 | │   │   ├── deployment.yaml 57 | │   │   ├── image-pull-secret.yaml 58 | │   │   └── service.yaml 59 | │   └── values-template.yaml 60 | ├── config.json 61 | ├── glide.lock 62 | ├── glide.yaml 63 | ├── service.go 64 | ├── skaffold.yaml 65 | ├── src 66 | │   ├── config-data-util 67 | │   │   ├── config_structs.go 68 | │   │   ├── data_utility.go 69 | │   │   ├── environment 70 | │   │   │   └── environment.go 71 | │   │   ├── kernel 72 | │   │   │   ├── kernel.go 73 | │   │   │   └── kernel_test.go 74 | │   │   ├── key 75 | │   │   │   └── key.go 76 | │   │   ├── memfilesystem 77 | │   │   │   └── memfilesystem.go 78 | │   │   └── user 79 | │   │   ├── user.go 80 | │   │   └── user_test.go 81 | │   ├── github.com 82 | │   │   ├── oleggorj 83 | │   │   │   └── service-common-lib 84 | │   │   │   ├── LICENSE 85 | │   │   │   ├── Makefile 86 | │   │   │   ├── README.md 87 | │   │   │   ├── common 88 | │   │   │   │   ├── circuitbreaker 89 | │   │   │   │   │   └── circuitbreaker.go 90 | │   │   │   │   ├── config 91 | │   │   │   │   │   ├── config.go 92 | │   │   │   │   │   └── git.go 93 | │   │   │   │   ├── logging 94 | │   │   │   │   │   └── logging.go 95 | │   │   │   │   └── util 96 | │   │   │   │   └── util.go 97 | │   │   │   ├── docker 98 | │   │   │   │   └── Dockerfile.goservice.template 99 | │   │   │   └── service 100 | │   │   │   ├── router.go 101 | │   │   │   ├── routes.go 102 | │   │   │   └── webserver.go 103 | │   │   ├── subosito 104 | │   │   │   └── gotenv 105 | │   │   │   ├── CHANGELOG.md 106 | │   │   │   ├── LICENSE 107 | │   │   │   ├── README.md 108 | │   │   │   ├── fixtures 109 | │   │   │   │   ├── bom.env 110 | │   │   │   │   ├── exported.env 111 | │   │   │   │   ├── plain.env 112 | │   │   │   │   ├── quoted.env 113 | │   │   │   │   └── yaml.env 114 | │   │   │   ├── go.mod 115 | │   │   │   ├── go.sum 116 | │   │   │   ├── gotenv.go 117 | │   │   │   └── gotenv_test.go 118 | │   │   └── tidwall 119 | │   │   ├── gjson 120 | │   │   │   ├── LICENSE 121 | │   │   │   ├── README.md 122 | │   │   │   ├── SYNTAX.md 123 | │   │   │   ├── gjson.go 124 | │   │   │   ├── gjson_gae.go 125 | │   │   │   ├── gjson_ngae.go 126 | │   │   │   ├── gjson_test.go 127 | │   │   │   ├── go.mod 128 | │   │   │   ├── go.sum 129 | │   │   │   └── logo.png 130 | │   │   ├── match 131 | │   │   │   ├── LICENSE 132 | │   │   │   ├── README.md 133 | │   │   │   ├── match.go 134 | │   │   │   └── match_test.go 135 | │   │   └── pretty 136 | │   │   ├── LICENSE 137 | │   │   ├── README.md 138 | │   │   ├── pretty.go 139 | │   │   └── pretty_test.go 140 | │   ├── gitutil 141 | │   │   ├── git_utility.go 142 | │   │   ├── git_utility_test.go 143 | │   │   └── test_data 144 | │   │   ├── kernels 145 | │   │   │   └── spark-kernel-12CPU-24GB 146 | │   │   ├── sandbox_whitelist.json 147 | │   │   ├── test_object_storage.json 148 | │   │   ├── test_users1.json 149 | │   │   ├── test_users2.json 150 | │   │   └── user.json 151 | │   ├── handlers 152 | │   │   ├── confEnvHandler.go 153 | │   │   ├── gitHandler.go 154 | │   │   ├── kernelHandler.go 155 | │   │   ├── keyHandler.go 156 | │   │   ├── userHandler.go 157 | │   │   └── userHandler_test.go 158 | │   └── helpers 159 | │   └── helpers.go 160 | ├── vars-gcp.mk 161 | ├── vars.mk 162 | └── watch.sh 163 | ``` 164 | --- 165 | 166 | ## How to deploy Config Service 167 | 168 | ### Setup configuration backend Git repo 169 | 170 | The very first part of setting up and deploying configuration data service is to setup git-based backend to host actual configurations. 171 | 172 | - create Git repository `config-data` (as an example https://github.com/OlegGorj/config-data.git ) 173 | (whether you make it private or public, is entirely up to you) 174 | - create branch called `sandbox` 175 | - check into branch `sandbox` file with the name `test.json` 176 | - edit `test.json` as follows 177 | 178 | ``` 179 | { 180 | "hello": "world" 181 | } 182 | ``` 183 | 184 | Note - do not merge the branches! 185 | 186 | ### Deployment on local laptop: 187 | 188 | Install main package and dependancies : 189 | 190 | ``` 191 | git clone https://github.com/OlegGorj/service-config-data.git 192 | cd service-config-data 193 | ``` 194 | 195 | Setup GOPATH: 196 | 197 | ``` 198 | echo $GOPATH 199 | export GOPATH=$GOPATH:$PWD 200 | echo $GOPATH 201 | ``` 202 | 203 | Make sure all dependancies are installed 204 | 205 | ``` 206 | make deps 207 | ``` 208 | 209 | Build `service-config-data` service binaries: 210 | 211 | ``` 212 | make build 213 | ``` 214 | 215 | To get service to run on your local machine (make sure docker is running): 216 | 217 | ``` 218 | make run 219 | ``` 220 | 221 | Now, test if the service runs correctly - open new terminal window and run: 222 | 223 | ``` 224 | curl http://localhost:8000/api/v2/test/sandbox/hello 225 | ``` 226 | 227 | Service API allows optional parameter specifying the output format. For instance, to get config data in JSON format, run: 228 | 229 | ``` 230 | curl http://localhost:8000/api/v2/test/sandbox/hello?out=json 231 | ``` 232 | 233 | 234 | 235 | 236 | ### Deployment of GCP: 237 | 238 | The following environment variables must be set as part of `vars-gcp.mk` in order for service to compile and run properly: 239 | 240 | `vars-gcp.mk` file 241 | 242 | ``` 243 | REPO?=github.com/OlegGorJ/config-data 244 | GITACCOUNT?= 245 | APITOKEN?= 246 | CONFIGFILE?=services 247 | REGISTRY?=oleggorj 248 | ``` 249 | 250 | Initialize GCP environment: 251 | 252 | ``` 253 | gcloud auth login 254 | ``` 255 | 256 | List available kubernetes clusters: 257 | 258 | ``` 259 | gcloud container clusters list 260 | ``` 261 | 262 | Example of output: 263 | 264 | ``` 265 | NAME LOCATION MASTER_VERSION MASTER_IP MACHINE_TYPE NODE_VERSION NUM_NODES STATUS 266 | cluster-services-sandbox us-central1-a 1.13.7-gke.19 00.000.000.00 n1-standard-1 1.13.7-gke.19 3 RUNNING 267 | ``` 268 | 269 | Than, create `tiller` service account and cluster binding: 270 | 271 | ``` 272 | kubectl create serviceaccount --namespace kube-system tiller 273 | kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller 274 | ``` 275 | 276 | And, finally, initialize Helm: 277 | 278 | ``` 279 | helm init --service-account tiller --upgrade 280 | ``` 281 | 282 | Last step to build service binaries, create image, push image to docker repo and deploy Helm chart: 283 | 284 | ``` 285 | cd service-config-data 286 | make push 287 | make deploy 288 | ``` 289 | 290 | Once, above steps are done, run following command to list all k8s services deployed in `default` namespace: 291 | 292 | ``` 293 | $ kubectl get services 294 | NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE 295 | kubernetes ClusterIP 10.12.0.1 443/TCP 4d16h 296 | service-config-data LoadBalancer 10.12.14.8 xxx.xxx.xxx.xx 8000:30956/TCP 10h 297 | ``` 298 | 299 | You should see public IP for service `service-config-data` - `xxx.xxx.xxx.xx`. 300 | 301 | And, finally, to test service deployment, run quick test curl call: 302 | 303 | ``` 304 | $ curl http://xxx.xxx.xxx.xx/api/v2/test/sandbox/hello 305 | 306 | world 307 | ``` 308 | 309 | Clean up: 310 | 311 | ``` 312 | make deployclean 313 | ``` 314 | 315 | --- 316 | ## How to use config service 317 | 318 | 319 | ### 1. `config-data` repository 320 | 321 | Consider an example of configuration file `config-data/k8s-cluster.json` in branch `sandbox`, that looks like this 322 | 323 | ``` 324 | { 325 | "name" : "jx-sandbox", 326 | "k8s-name" : "k8s-sandbox", 327 | "apps": [ 328 | { 329 | "name" : "jenkins", 330 | "app-type" : "devops" 331 | }, 332 | { 333 | "name" : "grafana", 334 | "app-type" : "devops" 335 | }, 336 | { 337 | "name" : "cassandra", 338 | "app-type" : "storage" 339 | } 340 | ], 341 | "env" : "sandbox", 342 | "region" : "us-east", 343 | "git-org" : "", 344 | "kubectl-ver" : "v1.13.0", 345 | "helm-ver": "v2.1.3", 346 | "docker-ver" : "1.12.6", 347 | "cloud-provider":{ 348 | "name": "ibmcloud", 349 | "api-endpoint" : "https://api.us-east.bluemix.net" 350 | } 351 | } 352 | ``` 353 | 354 | Assuming running config service container locally (http://localhost:8000) 355 | 356 | 357 | ### 2. A few use cases of `service-config-data` service 358 | 359 | Hope these examples are self explanatory. 360 | 361 | ``` 362 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/region 363 | 364 | us-east 365 | ``` 366 | 367 | ``` 368 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps 369 | 370 | [ 371 | { 372 | "name" : "jenkins", 373 | "app-type" : "devops" 374 | }, 375 | { 376 | "name" : "grafana", 377 | "app-type" : "devops" 378 | }, 379 | { 380 | "name" : "cassandra", 381 | "app-type" : "storage" 382 | } 383 | ] 384 | ``` 385 | 386 | ``` 387 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.@ 388 | 389 | 3 390 | ``` 391 | 392 | ``` 393 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.2 394 | 395 | { "name" : "cassandra", "app-type" : "storage" } 396 | ``` 397 | 398 | ``` 399 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.@.name 400 | 401 | ["jenkins","grafana","cassandra"] 402 | ``` 403 | 404 | ``` 405 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.1.name 406 | 407 | ["grafana"] 408 | ``` 409 | 410 | ``` 411 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.@(name=="cassandra").app-type 412 | 413 | storage 414 | ``` 415 | 416 | ``` 417 | $ curl http://localhost:8000/api/v2/k8s-cluster/sandbox/apps.@(app-type=="devops")@.name 418 | 419 | ["jenkins","grafana"] 420 | ``` 421 | 422 | 423 | --- 424 | 425 | ## Manual step-by-step deployment 426 | 427 | ### Build service runtime from Go code 428 | 429 | ``` 430 | make build 431 | ``` 432 | 433 | ### Build image and push to registry 434 | 435 | ``` 436 | make push 437 | ``` 438 | 439 | ### Deploy service 440 | 441 | ``` 442 | make deploy 443 | ``` 444 | 445 | ### Clean up 446 | 447 | ``` 448 | make deployclean 449 | ``` 450 | 451 | 452 | # Deploy service on Kubernetes cluster using Helm chart 453 | 454 | Make sure values file `./service-config/values.yaml` is populated based on template `../service-common-lib/docker/values-template.yaml` 455 | 456 | If you're having trouble understanding what values should be in `values.yaml`, refer to `Makefile` - it generates `values.yaml` in section `deploy` 457 | 458 | Deploy service by executing the following 459 | 460 | ``` 461 | cd service-config-data/charts 462 | helm upgrade --install config-service --values ./service-config/values.yaml --namespace default ./service-config/ 463 | ``` 464 | 465 | ...and clean up: 466 | 467 | ``` 468 | helm del --purge config-service 469 | ``` 470 | 471 | 472 | --- 473 | --------------------------------------------------------------------------------