├── keda-prometheus.jpg ├── Dockerfile ├── go.mod ├── keda-prometheus-scaledobject.yaml ├── go-app.yaml ├── README.md ├── main.go ├── prometheus.yaml └── go.sum /keda-prometheus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhirockzz/kubernetes-keda-prometheus/HEAD/keda-prometheus.jpg -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang as build-stage 2 | COPY go.mod / 3 | COPY go.sum / 4 | COPY main.go / 5 | RUN cd / && CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o go-prom-app 6 | 7 | FROM alpine 8 | COPY --from=build-stage /go-prom-app / 9 | EXPOSE 8080 10 | CMD ["/go-prom-app"] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module goprom 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/go-redis/redis v6.15.6+incompatible 7 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 8 | github.com/modern-go/reflect2 v1.0.1 // indirect 9 | github.com/prometheus/client_golang v1.2.0 10 | ) 11 | -------------------------------------------------------------------------------- /keda-prometheus-scaledobject.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: keda.k8s.io/v1alpha1 2 | kind: ScaledObject 3 | metadata: 4 | name: prometheus-scaledobject 5 | namespace: default 6 | labels: 7 | deploymentName: go-prom-app 8 | spec: 9 | scaleTargetRef: 10 | deploymentName: go-prom-app 11 | pollingInterval: 15 # Optional. Default: 30 seconds 12 | cooldownPeriod: 30 # Optional. Default: 300 seconds 13 | minReplicaCount: 1 # Optional. Default: 0 14 | maxReplicaCount: 10 # Optional. Default: 100 15 | triggers: 16 | - type: prometheus 17 | metadata: 18 | # Required 19 | serverAddress: http://prometheus-service.default.svc.cluster.local:9090 20 | metricName: access_frequency 21 | threshold: '3' 22 | query: sum(rate(http_requests[2m])) -------------------------------------------------------------------------------- /go-app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: go-prom-app 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: go-prom-app 10 | template: 11 | metadata: 12 | labels: 13 | app: go-prom-app 14 | spec: 15 | containers: 16 | - name: go-prom-app 17 | image: abhirockzz/go-prom-app 18 | env: 19 | - name: REDIS_HOST 20 | value: redis-server-master.default.svc.cluster.local 21 | - name: REDIS_PORT 22 | value: "6379" 23 | imagePullPolicy: Always 24 | ports: 25 | - containerPort: 8080 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: go-prom-app-service 31 | labels: 32 | run: go-prom-app-service 33 | spec: 34 | ports: 35 | - port: 8080 36 | protocol: TCP 37 | selector: 38 | app: go-prom-app -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Kubernetes app auto-scaling with Prometheus and KEDA 2 | 3 | ![](keda-prometheus.jpg) 4 | 5 | Scalability is a key requirement for cloud native applications. With [Kubernetes](https://kubernetes.io), scaling your application is as simple as increasing the number of replicas for the corresponding `Deployment` or `ReplicaSet` - but, this is a manual process. Kubernetes makes it possible to automatically scale your applications (i.e. `Pod`s in a `Deployment` or `ReplicaSet`) in a declarative manner using the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.16/#horizontalpodautoscaler-v1-autoscaling) specification. 6 | 7 | [This blog post](https://dev.to/azure/how-to-auto-scale-your-kubernetes-apps-with-prometheus-and-keda-39km) demonstrates how you can external metrics to auto-scale a Kubernetes application. For demonstration purposes, we will use HTTP access request metrics that are exposed using [Prometheus](https://prometheus.io). Instead of using the `Horizontal Pod Autoscaler` directly, we will leverage [**Kubernetes Event Driven Autoscaling** aka **KEDA**](https://github.com/kedacore/keda) - an open source Kubernetes operator which integrates natively with the `Horizontal Pod Autoscaler` to provide fine grained autoscaling (including to/from zero) for event-driven workloads. -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/go-redis/redis" 12 | 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promauto" 15 | 16 | "github.com/prometheus/client_golang/prometheus/promhttp" 17 | ) 18 | 19 | var httpRequestsCounter = promauto.NewCounter(prometheus.CounterOpts{ 20 | Name: "http_requests", 21 | Help: "number of http requests", 22 | }) 23 | 24 | const redisCounterName = "access_count" 25 | 26 | var once sync.Once 27 | var client *redis.Client 28 | 29 | func init() { 30 | host := os.Getenv("REDIS_HOST") 31 | if host == "" { 32 | fmt.Printf("REDIS_HOST env var absent!") 33 | os.Exit(1) 34 | } 35 | 36 | port := os.Getenv("REDIS_PORT") 37 | if port == "" { 38 | fmt.Printf("REDIS_PORT env var absent!") 39 | os.Exit(1) 40 | } 41 | client = redis.NewClient(&redis.Options{Addr: host + ":" + port, PoolSize: 500}) 42 | err := client.Ping().Err() 43 | if err != nil { 44 | fmt.Printf("Unable to connect to Redis at %s:%s", host, port) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func main() { 50 | http.Handle("/metrics", promhttp.Handler()) 51 | http.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) { 52 | defer httpRequestsCounter.Inc() 53 | count, err := client.Incr(redisCounterName).Result() 54 | if err != nil { 55 | fmt.Println("Unable to increment redis counter", err) 56 | os.Exit(1) 57 | } 58 | resp := "Accessed on " + time.Now().String() + "\nAccess count " + strconv.Itoa(int(count)) 59 | w.Write([]byte(resp)) 60 | }) 61 | 62 | http.ListenAndServe(":8080", nil) 63 | } 64 | -------------------------------------------------------------------------------- /prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | name: prometheus 5 | rules: 6 | - apiGroups: [""] 7 | resources: 8 | - services 9 | verbs: ["get", "list", "watch"] 10 | - nonResourceURLs: ["/metrics"] 11 | verbs: ["get"] 12 | --- 13 | apiVersion: v1 14 | kind: ServiceAccount 15 | metadata: 16 | name: default 17 | --- 18 | apiVersion: rbac.authorization.k8s.io/v1beta1 19 | kind: ClusterRoleBinding 20 | metadata: 21 | name: prometheus 22 | roleRef: 23 | apiGroup: rbac.authorization.k8s.io 24 | kind: ClusterRole 25 | name: prometheus 26 | subjects: 27 | - kind: ServiceAccount 28 | name: default 29 | namespace: default 30 | --- 31 | apiVersion: v1 32 | kind: ConfigMap 33 | metadata: 34 | name: prom-conf 35 | labels: 36 | name: prom-conf 37 | data: 38 | prometheus.yml: |- 39 | global: 40 | scrape_interval: 5s 41 | evaluation_interval: 5s 42 | scrape_configs: 43 | - job_name: 'go-prom-job' 44 | 45 | kubernetes_sd_configs: 46 | - role: service 47 | relabel_configs: 48 | - source_labels: [__meta_kubernetes_service_label_run] 49 | regex: go-prom-app-service 50 | action: keep 51 | --- 52 | apiVersion: extensions/v1beta1 53 | kind: Deployment 54 | metadata: 55 | name: prometheus-deployment 56 | spec: 57 | replicas: 1 58 | template: 59 | metadata: 60 | labels: 61 | app: prometheus-server 62 | spec: 63 | containers: 64 | - name: prometheus 65 | image: prom/prometheus 66 | args: 67 | - "--config.file=/etc/prometheus/prometheus.yml" 68 | - "--storage.tsdb.path=/prometheus/" 69 | ports: 70 | - containerPort: 9090 71 | volumeMounts: 72 | - name: prometheus-config-volume 73 | mountPath: /etc/prometheus/ 74 | - name: prometheus-storage-volume 75 | mountPath: /prometheus/ 76 | volumes: 77 | - name: prometheus-config-volume 78 | configMap: 79 | defaultMode: 420 80 | name: prom-conf 81 | 82 | - name: prometheus-storage-volume 83 | emptyDir: {} 84 | --- 85 | apiVersion: v1 86 | kind: Service 87 | metadata: 88 | name: prometheus-service 89 | spec: 90 | ports: 91 | - port: 9090 92 | protocol: TCP 93 | selector: 94 | app: prometheus-server -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 2 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 4 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 5 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 6 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= 10 | github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 14 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 15 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 16 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 17 | github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg= 18 | github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 19 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 20 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 21 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 22 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 23 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 24 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 26 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 27 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 28 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 29 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 30 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 31 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 32 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 33 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 34 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 36 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 37 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 38 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 39 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 40 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 41 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 42 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 43 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 44 | github.com/prometheus/client_golang v1.2.0 h1:g4yo/h/me4ZL9o0SVHNRdS2jn5SY8GDmMgkhQ8Mz70s= 45 | github.com/prometheus/client_golang v1.2.0/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= 46 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 47 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 48 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= 49 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 50 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 51 | github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= 52 | github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= 53 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 54 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 55 | github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= 56 | github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= 57 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 58 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 59 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 60 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 61 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 62 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 63 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 64 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 65 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 66 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 67 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 68 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 70 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 72 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 73 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 74 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 75 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 76 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 77 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 78 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 79 | --------------------------------------------------------------------------------