├── services ├── user │ ├── lib │ │ ├── __init__.py │ │ └── kafka.py │ ├── Dockerfile │ └── app.py ├── operation │ ├── lib │ │ ├── __init__.py │ │ └── kafka.py │ ├── Dockerfile │ └── app.py └── userApproval │ ├── Dockerfile │ └── main.go ├── run ├── kiali.sh ├── grafana.sh ├── destroy.sh └── install.sh ├── infrastructure ├── terraform │ ├── variables.tf │ ├── modules │ │ ├── azure-cosmos-db │ │ │ ├── variables.tf │ │ │ ├── output.tf │ │ │ └── main.tf │ │ ├── azure-container-registry │ │ │ ├── variables.tf │ │ │ ├── output.tf │ │ │ └── main.tf │ │ ├── azure-k8s-cluster │ │ │ ├── output.tf │ │ │ ├── variables.tf │ │ │ └── main.tf │ │ └── ccloud-kafka-cluster │ │ │ └── main.tf │ ├── main.tf │ └── output.tf ├── k8s │ ├── namespaces.yaml │ ├── kiali.yaml │ ├── priority.yaml │ ├── placeholder.yaml │ ├── user-service.yaml │ ├── rest-requester.yaml │ ├── operation-service.yaml │ ├── istio.yaml │ └── user-approval-service.yaml ├── helm │ ├── helmfile.yaml │ └── config │ │ └── prometheus-adapter.yaml └── grafana │ └── dashboard.json ├── .env ├── .gitignore └── README.md /services/user/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /services/operation/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /run/kiali.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | kubectl port-forward -n istio-system service/kiali 20001:20001 3 | -------------------------------------------------------------------------------- /run/grafana.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | kubectl port-forward -n istio-system service/grafana 3000:3000 3 | -------------------------------------------------------------------------------- /infrastructure/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "azure_client_id" {} 2 | variable "azure_client_secret" {} 3 | -------------------------------------------------------------------------------- /infrastructure/k8s/namespaces.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: default 5 | labels: 6 | istio-injection: enabled 7 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | # you Azure IAM credentials 2 | TF_VAR_azure_client_id="" 3 | TD_VAR_azure_client_secret="" 4 | 5 | # your Confluent Cloud login credentials 6 | CONFLUENT_CLOUD_USERNAME="" 7 | CONFLUENT_CLOUD_PASSWORD="" 8 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-cosmos-db/variables.tf: -------------------------------------------------------------------------------- 1 | variable resource_group_name { 2 | default = "scalable_microservice" 3 | } 4 | 5 | variable resource_group_location { 6 | default = "West Europe" 7 | } 8 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-container-registry/variables.tf: -------------------------------------------------------------------------------- 1 | variable resource_group_name { 2 | default = "scalable_microservice" 3 | } 4 | 5 | variable resource_group_location { 6 | default = "West Europe" 7 | } 8 | -------------------------------------------------------------------------------- /infrastructure/k8s/kiali.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: kiali 5 | namespace: istio-system 6 | labels: 7 | app: kiali 8 | type: Opaque 9 | data: 10 | username: YWRtaW4= 11 | passphrase: YWRtaW4= 12 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-cosmos-db/output.tf: -------------------------------------------------------------------------------- 1 | output "endpoint" { 2 | value = azurerm_cosmosdb_account.db.endpoint 3 | } 4 | 5 | output "connection_strings" { 6 | value = azurerm_cosmosdb_account.db.connection_strings 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /.env.* 3 | /infrastructure/terraform/kube_config 4 | /infrastructure/terraform/.terraform 5 | /infrastructure/terraform/.terraform/* 6 | /COMMANDS.md 7 | /kafka.config.properties 8 | /auth 9 | infrastructure/terraform/terraform.tfstate 10 | infrastructure/terraform/terraform.tfstate.backup 11 | .DS_Store 12 | -------------------------------------------------------------------------------- /services/user/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.1-alpine 2 | RUN mkdir -p /usr/src/app 3 | WORKDIR /usr/src/app 4 | 5 | RUN apk add --no-cache gcc musl-dev librdkafka-dev \ 6 | && pip install -U pymongo==3.10.1 PyPubSub==4.0.3 avro-python3==1.9.2.1 crcmod requests==2.22.0 confluent_kafka==1.3.0 7 | 8 | COPY . /usr/src/app 9 | 10 | ENTRYPOINT ["python3", "-u", "app.py"] 11 | -------------------------------------------------------------------------------- /infrastructure/k8s/priority.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.k8s.io/v1 2 | kind: PriorityClass 3 | metadata: 4 | name: placeholder 5 | value: 0 6 | preemptionPolicy: Never 7 | globalDefault: false 8 | description: 'placeholder' 9 | --- 10 | apiVersion: scheduling.k8s.io/v1 11 | kind: PriorityClass 12 | metadata: 13 | name: normal 14 | value: 1 15 | preemptionPolicy: Never 16 | globalDefault: true 17 | description: 'normal' 18 | -------------------------------------------------------------------------------- /infrastructure/helm/helmfile.yaml: -------------------------------------------------------------------------------- 1 | helmDefaults: 2 | tillerless: true 3 | atomic: false 4 | verify: false 5 | wait: true 6 | timeout: 1200 7 | 8 | repositories: 9 | - name: stable 10 | url: https://kubernetes-charts.storage.googleapis.com 11 | 12 | releases: 13 | - name: prometheus-adapter 14 | chart: stable/prometheus-adapter 15 | version: 2.1.2 16 | namespace: default 17 | values: 18 | - "./config/prometheus-adapter.yaml" 19 | -------------------------------------------------------------------------------- /services/operation/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.1-alpine 2 | RUN mkdir -p /usr/src/app 3 | WORKDIR /usr/src/app 4 | 5 | RUN apk add --no-cache gcc musl-dev librdkafka-dev &&\ 6 | pip install -U pymongo==3.10.1 PyPubSub==4.0.3 avro-python3==1.9.2.1 crcmod requests==2.22.0 confluent_kafka==1.3.0 \ 7 | flask==1.1.1 flask-cors==3.0.8 requests==2.22.0 8 | 9 | COPY . /usr/src/app 10 | 11 | EXPOSE 5000 12 | ENTRYPOINT ["python3", "-u", "app.py"] 13 | -------------------------------------------------------------------------------- /run/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ####### credentials ####### 4 | FILE=.env.local 5 | if [ ! -f "$FILE" ]; then 6 | echo "$FILE does not exist. Copy .env to .env.local and fill in credentials" 7 | exit 8 | fi 9 | source .env.local 10 | export TF_VAR_azure_client_id TF_VAR_azure_client_secret CONFLUENT_CLOUD_USERNAME CONFLUENT_CLOUD_PASSWORD 11 | 12 | 13 | ####### terraform ####### 14 | cd infrastructure/terraform 15 | terraform destroy -auto-approve 16 | 17 | rm -rf auth/* 18 | -------------------------------------------------------------------------------- /infrastructure/helm/config/prometheus-adapter.yaml: -------------------------------------------------------------------------------- 1 | prometheus: 2 | port: 9090 3 | url: http://prometheus.istio-system 4 | rules: 5 | default: false 6 | resource: {} 7 | custom: 8 | - seriesQuery: 'kafka_consumergroup_lag' 9 | resources: 10 | overrides: 11 | namespace: {resource: "namespace"} 12 | pod_name: {resource: "pod"} 13 | name: 14 | matches: "kafka_consumergroup_lag" 15 | as: "kafka_consumergroup_lag" 16 | metricsQuery: 'avg_over_time(kafka_consumergroup_lag{topic="user-approve",consumergroup="user-approval-service"}[1m])' 17 | -------------------------------------------------------------------------------- /infrastructure/k8s/placeholder.yaml: -------------------------------------------------------------------------------- 1 | # ensures that always two more nodes are reserved for faster scaling 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: placeholder 6 | spec: 7 | replicas: 3 8 | selector: 9 | matchLabels: 10 | app: placeholder 11 | template: 12 | metadata: 13 | labels: 14 | app: placeholder 15 | spec: 16 | terminationGracePeriodSeconds: 0 17 | priorityClassName: placeholder 18 | containers: 19 | - image: nginx 20 | name: placeholder 21 | resources: 22 | requests: 23 | cpu: 1500m 24 | memory: 1000Mi 25 | -------------------------------------------------------------------------------- /infrastructure/terraform/main.tf: -------------------------------------------------------------------------------- 1 | provider "azurerm" { 2 | version = "~>1.32" 3 | } 4 | 5 | module "azure-k8s-cluster" { 6 | source = "./modules/azure-k8s-cluster" 7 | client_id = var.azure_client_id 8 | client_secret = var.azure_client_secret 9 | } 10 | 11 | module "azure-container-registry" { 12 | source = "./modules/azure-container-registry" 13 | } 14 | 15 | module "azure-cosmos-db" { 16 | source = "./modules/azure-cosmos-db" 17 | } 18 | 19 | module "ccloud-kafka-cluster" { 20 | source = "./modules/ccloud-kafka-cluster" 21 | cluster_name = "scalable_microservice_cluster" 22 | cluster_cloud_provider = "aws" 23 | environment_name = "scalable_microservice_env" 24 | } 25 | -------------------------------------------------------------------------------- /infrastructure/k8s/user-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: user-service 5 | spec: 6 | replicas: 6 7 | selector: 8 | matchLabels: 9 | app: user-service 10 | template: 11 | metadata: 12 | labels: 13 | app: user-service 14 | spec: 15 | containers: 16 | - image: containerrregistryscalablemicroservice.azurecr.io/user_service 17 | name: user-service 18 | imagePullPolicy: Always 19 | envFrom: 20 | - secretRef: 21 | name: kafka 22 | - secretRef: 23 | name: mongodb 24 | resources: 25 | requests: 26 | cpu: 50m 27 | memory: 50Mi 28 | limits: 29 | cpu: 100m 30 | memory: 100Mi 31 | -------------------------------------------------------------------------------- /services/userApproval/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.12.4 as builder 2 | ENV LIBRDKAFKA_VERSION 1.3.0 3 | 4 | RUN apt-get -y update \ 5 | && apt-get install -y --no-install-recommends upx-ucl zip libssl-dev \ 6 | && apt-get clean \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | 10 | RUN curl -Lk -o /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz && \ 11 | tar -xzf /root/librdkafka-${LIBRDKAFKA_VERSION}.tar.gz -C /root && \ 12 | cd /root/librdkafka-${LIBRDKAFKA_VERSION} && \ 13 | ./configure --prefix /usr && make && make install && make clean && ./configure --clean 14 | 15 | 16 | WORKDIR /app/ 17 | COPY . . 18 | ENV GO111MODULE=on 19 | RUN go build -o app main.go 20 | ENTRYPOINT ["/bin/sh", "-c", "./app"] 21 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-k8s-cluster/output.tf: -------------------------------------------------------------------------------- 1 | output "client_key" { 2 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_key 3 | } 4 | 5 | output "client_certificate" { 6 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate 7 | } 8 | 9 | output "cluster_ca_certificate" { 10 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate 11 | } 12 | 13 | output "cluster_username" { 14 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.username 15 | } 16 | 17 | output "cluster_password" { 18 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.password 19 | } 20 | 21 | output "kube_config" { 22 | value = azurerm_kubernetes_cluster.k8s.kube_config_raw 23 | } 24 | 25 | output "host" { 26 | value = azurerm_kubernetes_cluster.k8s.kube_config.0.host 27 | } 28 | -------------------------------------------------------------------------------- /infrastructure/k8s/rest-requester.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: rest-requester 5 | spec: 6 | replicas: 0 7 | selector: 8 | matchLabels: 9 | app: rest-requester 10 | template: 11 | metadata: 12 | labels: 13 | app: rest-requester 14 | spec: 15 | containers: 16 | - image: byrnedo/alpine-curl:0.1.8 17 | name: rest-requester 18 | command: 19 | - "sh" 20 | - "-c" 21 | - "while true; sleep 0.01; do curl --resolve 'operation-service.scalable-microservice:80:13.80.69.134' http://operation-service.scalable-microservice/operation/create/user-create -X POST -H 'Content-Type: application/json' -d '{\"name\": \"hans\"}'; echo; done" 22 | resources: 23 | requests: 24 | cpu: 50m 25 | memory: 50Mi 26 | limits: 27 | cpu: 100m 28 | memory: 100Mi 29 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-container-registry/output.tf: -------------------------------------------------------------------------------- 1 | output "server" { 2 | value = azurerm_container_registry.acr.login_server 3 | } 4 | 5 | output "docker_login" { 6 | value = "docker login ${azurerm_container_registry.acr.login_server} -u ${azurerm_azuread_service_principal.acr-sp.application_id} -p ${azurerm_azuread_service_principal_password.acr-sp-pass.value}" 7 | } 8 | 9 | output "kubernetes_secret" { 10 | value = "kubectl create secret docker-registry docker-rep-pull --docker-server=${azurerm_container_registry.acr.login_server} --docker-username='${azurerm_azuread_service_principal.acr-sp.application_id}' --docker-password='${azurerm_azuread_service_principal_password.acr-sp-pass.value}'" 11 | } 12 | 13 | output "username" { 14 | value = azurerm_azuread_service_principal.acr-sp.application_id 15 | } 16 | 17 | output "password" { 18 | value = azurerm_azuread_service_principal_password.acr-sp-pass.value 19 | } 20 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-cosmos-db/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "scalable_microservice" { 2 | name = var.resource_group_name 3 | location = var.resource_group_location 4 | } 5 | 6 | resource "random_integer" "ri" { 7 | min = 10000 8 | max = 99999 9 | } 10 | 11 | resource "azurerm_cosmosdb_account" "db" { 12 | name = "scalable-microservice-demo-${random_integer.ri.result}" 13 | location = var.resource_group_location 14 | resource_group_name = var.resource_group_name 15 | offer_type = "Standard" 16 | kind = "MongoDB" 17 | 18 | enable_automatic_failover = true 19 | 20 | consistency_policy { 21 | consistency_level = "BoundedStaleness" 22 | max_interval_in_seconds = 10 23 | max_staleness_prefix = 200 24 | } 25 | 26 | geo_location { 27 | prefix = "scalable-microservice-demo-${random_integer.ri.result}-customid" 28 | location = var.resource_group_location 29 | failover_priority = 0 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /infrastructure/k8s/operation-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: operation-service 5 | spec: 6 | replicas: 5 7 | selector: 8 | matchLabels: 9 | app: operation-service 10 | template: 11 | metadata: 12 | labels: 13 | app: operation-service 14 | spec: 15 | containers: 16 | - image: containerrregistryscalablemicroservice.azurecr.io/operation_service 17 | name: operation-service 18 | imagePullPolicy: Always 19 | envFrom: 20 | - secretRef: 21 | name: kafka 22 | - secretRef: 23 | name: mongodb 24 | resources: 25 | requests: 26 | cpu: 50m 27 | memory: 50Mi 28 | limits: 29 | cpu: 100m 30 | memory: 100Mi 31 | --- 32 | apiVersion: v1 33 | kind: Service 34 | metadata: 35 | name: operation-service 36 | spec: 37 | ports: 38 | - name: http 39 | port: 80 40 | protocol: TCP 41 | targetPort: 5000 42 | selector: 43 | app: operation-service 44 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-k8s-cluster/variables.tf: -------------------------------------------------------------------------------- 1 | variable "client_id" {} 2 | variable "client_secret" {} 3 | 4 | variable "kubernetes_version" { 5 | // az aks get-versions --location "West Europe" --output table 6 | default = "1.17.3" 7 | } 8 | 9 | variable "agent_count" { 10 | default = 3 11 | } 12 | 13 | variable "ssh_public_key" { 14 | default = "~/.ssh/id_rsa.pub" 15 | } 16 | 17 | variable "dns_prefix" { 18 | default = "k8s" 19 | } 20 | 21 | variable cluster_name { 22 | default = "scalable_ms_prod" 23 | } 24 | 25 | variable resource_group_name { 26 | default = "scalable_microservice" 27 | } 28 | 29 | variable resource_group_location { 30 | default = "West Europe" 31 | } 32 | 33 | variable log_analytics_workspace_name { 34 | default = "testLogAnalyticsWorkspaceName" 35 | } 36 | 37 | # refer https://azure.microsoft.com/global-infrastructure/services/?products=monitor for log analytics available regions 38 | variable log_analytics_workspace_location { 39 | default = "eastus" 40 | } 41 | 42 | # refer https://azure.microsoft.com/pricing/details/monitor/ for log analytics pricing 43 | variable log_analytics_workspace_sku { 44 | default = "PerGB2018" 45 | } 46 | -------------------------------------------------------------------------------- /infrastructure/k8s/istio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1alpha3 2 | kind: Gateway 3 | metadata: 4 | name: ingressgateway 5 | labels: 6 | release: istio 7 | namespace: istio-system 8 | spec: 9 | selector: 10 | istio: ingressgateway 11 | servers: 12 | - port: 13 | number: 80 14 | name: http 15 | protocol: HTTP 16 | hosts: 17 | - "operation-service.scalable-microservice" 18 | --- 19 | apiVersion: networking.istio.io/v1alpha3 20 | kind: VirtualService 21 | metadata: 22 | name: operation-service 23 | namespace: default 24 | spec: 25 | gateways: 26 | - istio-system/ingressgateway 27 | hosts: 28 | - "operation-service.scalable-microservice" 29 | http: 30 | - match: 31 | - {} 32 | route: 33 | - destination: 34 | host: operation-service.default.svc.cluster.local 35 | port: 36 | number: 80 37 | weight: 100 38 | #--- 39 | #apiVersion: networking.istio.io/v1alpha3 40 | #kind: ServiceEntry 41 | #metadata: 42 | # name: kafka 43 | #spec: 44 | # hosts: 45 | # - "*.confluent.cloud" 46 | # - "*.amazonaws.com" 47 | # ports: 48 | # - number: 9092 49 | # name: kafka 50 | # protocol: tcp 51 | # resolution: NONE 52 | # location: MESH_EXTERNAL 53 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-container-registry/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "scalable_microservice" { 2 | name = var.resource_group_name 3 | location = var.resource_group_location 4 | } 5 | 6 | //resource "azurerm_container_registry" "acr" { 7 | // name = "containerRregistryScalableMicroservice" 8 | // resource_group_name = azurerm_resource_group.scalable_microservice.name 9 | // location = azurerm_resource_group.scalable_microservice.location 10 | // sku = "Standard" 11 | // admin_enabled = true 12 | // network_rule_set = [] 13 | //} 14 | 15 | resource "azurerm_container_registry" "acr" { 16 | name = "containerRregistryScalableMicroservice" 17 | resource_group_name = azurerm_resource_group.scalable_microservice.name 18 | location = azurerm_resource_group.scalable_microservice.location 19 | sku = "standard" 20 | } 21 | 22 | resource "azurerm_azuread_application" "acr-app" { 23 | name = "acr-app" 24 | } 25 | 26 | resource "azurerm_azuread_service_principal" "acr-sp" { 27 | application_id = azurerm_azuread_application.acr-app.application_id 28 | } 29 | 30 | resource "azurerm_azuread_service_principal_password" "acr-sp-pass" { 31 | service_principal_id = azurerm_azuread_service_principal.acr-sp.id 32 | value = "Password666" 33 | end_date = "2030-01-01T01:02:03Z" 34 | } 35 | 36 | resource "azurerm_role_assignment" "acr-assignment" { 37 | scope = azurerm_container_registry.acr.id 38 | role_definition_name = "Contributor" 39 | principal_id = azurerm_azuread_service_principal_password.acr-sp-pass.service_principal_id 40 | } 41 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/ccloud-kafka-cluster/main.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" {} 2 | variable "cluster_cloud_provider" {} 3 | variable "environment_name" {} 4 | 5 | provider "confluentcloud" {} 6 | 7 | resource "confluentcloud_environment" "test" { 8 | name = var.environment_name 9 | } 10 | 11 | resource "confluentcloud_kafka_cluster" "test" { 12 | name = var.cluster_name 13 | environment_id = confluentcloud_environment.test.id 14 | service_provider = var.cluster_cloud_provider 15 | region = "eu-west-1" 16 | availability = "LOW" 17 | } 18 | 19 | resource "confluentcloud_api_key" "provider_test" { 20 | cluster_id = confluentcloud_kafka_cluster.test.id 21 | environment_id = confluentcloud_environment.test.id 22 | } 23 | 24 | //provider "kafka" { 25 | // bootstrap_servers = [replace(confluentcloud_kafka_cluster.test.bootstrap_servers, "SASL_SSL://", "")] 26 | // 27 | // tls_enabled = true 28 | // sasl_username = confluentcloud_api_key.provider_test.key 29 | // sasl_password = confluentcloud_api_key.provider_test.secret 30 | // sasl_mechanism = "plain" 31 | //} 32 | 33 | //resource "kafka_topic" "user-create" { 34 | // name = "user-create" 35 | // replication_factor = 3 36 | // partitions = 6 37 | //} 38 | 39 | //resource "kafka_topic" "user-create-response" { 40 | // name = "user-create-response" 41 | // replication_factor = 3 42 | // partitions = 6 43 | //} 44 | 45 | output "kafka_url" { 46 | value = replace(confluentcloud_kafka_cluster.test.bootstrap_servers, "SASL_SSL://", "") 47 | } 48 | 49 | output "key" { 50 | value = confluentcloud_api_key.provider_test.key 51 | } 52 | 53 | output "secret" { 54 | value = confluentcloud_api_key.provider_test.secret 55 | } 56 | -------------------------------------------------------------------------------- /infrastructure/k8s/user-approval-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: user-approval-service 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: user-approval-service 10 | template: 11 | metadata: 12 | labels: 13 | app: user-approval-service 14 | annotations: 15 | sidecar.istio.io/inject: "false" 16 | prometheus.io/scrape: 'true' 17 | prometheus.io/path: '/metrics' 18 | prometheus.io/port: '9308' 19 | spec: 20 | containers: 21 | - image: danielqsj/kafka-exporter 22 | name: kafka-exporter 23 | command: 24 | - "sh" 25 | - "-c" 26 | - "/bin/kafka_exporter --kafka.server $KAFKA_BOOTSTRAP_SERVERS --sasl.enabled --sasl.username $KAFKA_SASL_USERNAME --sasl.password $KAFKA_SASL_PASSWORD --tls.insecure-skip-tls-verify --tls.enabled" 27 | envFrom: 28 | - secretRef: 29 | name: kafka 30 | ports: 31 | - name: telemetry 32 | containerPort: 9308 33 | - image: containerrregistryscalablemicroservice.azurecr.io/user_approval_service 34 | name: user-approval-service 35 | imagePullPolicy: Always 36 | envFrom: 37 | - secretRef: 38 | name: kafka 39 | resources: 40 | requests: 41 | cpu: 50m 42 | memory: 50Mi 43 | limits: 44 | cpu: 100m 45 | memory: 100Mi 46 | --- 47 | apiVersion: autoscaling/v2beta1 48 | kind: HorizontalPodAutoscaler 49 | metadata: 50 | name: user-approval-service 51 | spec: 52 | maxReplicas: 10000 53 | minReplicas: 1 54 | scaleTargetRef: 55 | apiVersion: apps/v1 56 | kind: Deployment 57 | name: user-approval-service 58 | metrics: 59 | - type: Pods 60 | pods: 61 | metricName: "kafka_consumergroup_lag" 62 | targetAverageValue: 5 63 | -------------------------------------------------------------------------------- /infrastructure/terraform/output.tf: -------------------------------------------------------------------------------- 1 | # azure k8s cluster 2 | output "azure-k8s-cluster_client_key" { 3 | value = module.azure-k8s-cluster.client_key 4 | } 5 | output "azure-k8s-cluster_client_certificate" { 6 | value = module.azure-k8s-cluster.client_certificate 7 | } 8 | output "azure-k8s-cluster_cluster_ca_certificate" { 9 | value = module.azure-k8s-cluster.cluster_ca_certificate 10 | } 11 | output "azure-k8s-cluster_cluster_username" { 12 | value = module.azure-k8s-cluster.cluster_username 13 | } 14 | output "azure-k8s-cluster_cluster_password" { 15 | value = module.azure-k8s-cluster.cluster_password 16 | } 17 | output "azure-k8s-cluster_kube_config" { 18 | value = module.azure-k8s-cluster.kube_config 19 | } 20 | output "azure-k8s-cluster_host" { 21 | value = module.azure-k8s-cluster.host 22 | } 23 | 24 | 25 | # azure cosmos db 26 | output "azure-cosmos_db-endpoint" { 27 | value = module.azure-cosmos-db.endpoint 28 | } 29 | output "azure-cosmos_db-connection_strings" { 30 | value = module.azure-cosmos-db.connection_strings 31 | } 32 | output "azure-cosmos_db-connection_string_one" { 33 | value = module.azure-cosmos-db.connection_strings[0] 34 | } 35 | 36 | 37 | # azure container registry 38 | output "azure-container-registry-login_server" { 39 | value = module.azure-container-registry.server 40 | } 41 | output "azure-container-registry-docker_login" { 42 | value = module.azure-container-registry.docker_login 43 | } 44 | output "azure-container-registry-kubernetes_secret" { 45 | value = module.azure-container-registry.kubernetes_secret 46 | } 47 | output "azure-container-registry-username" { 48 | value = module.azure-container-registry.username 49 | } 50 | output "azure-container-registry-password" { 51 | value = module.azure-container-registry.password 52 | } 53 | 54 | 55 | # kafka 56 | output "ccloud-kafka-cluster_kafka_url" { 57 | value = module.ccloud-kafka-cluster.kafka_url 58 | } 59 | output "ccloud-kafka-cluster_key" { 60 | value = module.ccloud-kafka-cluster.key 61 | } 62 | output "ccloud-kafka-cluster_secret" { 63 | value = module.ccloud-kafka-cluster.secret 64 | } 65 | -------------------------------------------------------------------------------- /services/user/app.py: -------------------------------------------------------------------------------- 1 | from lib.kafka import Kafka 2 | import json 3 | import logging 4 | from datetime import datetime 5 | import uuid as uuid_lib 6 | import sys 7 | import pymongo 8 | import os 9 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 10 | 11 | 12 | topic_user_create = 'user-create' 13 | topic_user_create_response = 'user-create-response' 14 | topic_user_approve = 'user-approve' 15 | topic_user_approve_response = 'user-approve-response' 16 | kafka = Kafka('user-service', [topic_user_create, topic_user_approve_response], [topic_user_approve, topic_user_approve_response]) 17 | 18 | mongodb_connection_string = os.environ['MONGODB_CONNECTION_STRING'] 19 | mongodb_client = pymongo.MongoClient(mongodb_connection_string) 20 | mongodb_database = mongodb_client["user_service"] 21 | mongodb_collection = mongodb_database["user"] 22 | 23 | 24 | def user_create(msg): 25 | logging.info(f'creating user for msg:{msg}') 26 | 27 | user = msg['data'] 28 | user['createdAt'] = str(datetime.now()) 29 | user['uuid'] = str(uuid_lib.uuid4()) 30 | user['approved'] = 'pending' 31 | 32 | result = mongodb_collection.insert_one(user.copy()) 33 | logging.info(f'MongoDB insert: {result.inserted_id}') 34 | 35 | msg['data'] = user 36 | 37 | kafka.create_message(topic_user_approve, msg) 38 | logging.info(f'send user approval for msg:{msg}') 39 | 40 | 41 | def user_approve(msg): 42 | logging.info(f'approving user for msg:{msg}') 43 | 44 | user = msg['data'] 45 | 46 | user_mongo = user.copy() 47 | mongodb_collection.replace_one( 48 | {'uuid': user_mongo['uuid']}, 49 | user_mongo 50 | ) 51 | logging.info(f'MongoDB user updated') 52 | 53 | msg['data'] = user 54 | 55 | if user['approved'] == 'true': 56 | msg['response']['state'] = 'completed' 57 | elif user['approved'] == 'false': 58 | msg['response']['state'] = 'failed' 59 | msg['response']['error'] = 'approval failed' 60 | 61 | kafka.create_message(topic_user_create_response, msg) 62 | logging.info(f'updated user for msg:{msg}') 63 | 64 | 65 | def new_message_listener(msg): 66 | topic = msg.topic() 67 | data = json.loads(msg.value()) 68 | if topic == topic_user_create: 69 | user_create(data) 70 | elif topic == topic_user_approve_response: 71 | user_approve(data) 72 | 73 | 74 | kafka.subscribe(new_message_listener, 'kafka_new_message') 75 | -------------------------------------------------------------------------------- /infrastructure/terraform/modules/azure-k8s-cluster/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "scalable_microservice" { 2 | name = var.resource_group_name 3 | location = var.resource_group_location 4 | } 5 | 6 | resource "random_id" "log_analytics_workspace_name_suffix" { 7 | byte_length = 8 8 | } 9 | 10 | resource "azurerm_log_analytics_workspace" "test" { 11 | # The WorkSpace name has to be unique across the whole of azure, not just the current subscription/tenant. 12 | name = "${var.log_analytics_workspace_name}-${random_id.log_analytics_workspace_name_suffix.dec}" 13 | location = var.log_analytics_workspace_location 14 | resource_group_name = azurerm_resource_group.scalable_microservice.name 15 | sku = var.log_analytics_workspace_sku 16 | } 17 | 18 | resource "azurerm_log_analytics_solution" "test" { 19 | solution_name = "ContainerInsights" 20 | location = azurerm_log_analytics_workspace.test.location 21 | resource_group_name = azurerm_resource_group.scalable_microservice.name 22 | workspace_resource_id = azurerm_log_analytics_workspace.test.id 23 | workspace_name = azurerm_log_analytics_workspace.test.name 24 | 25 | plan { 26 | publisher = "Microsoft" 27 | product = "OMSGallery/ContainerInsights" 28 | } 29 | } 30 | 31 | resource "azurerm_kubernetes_cluster" "k8s" { 32 | name = var.cluster_name 33 | location = azurerm_resource_group.scalable_microservice.location 34 | resource_group_name = azurerm_resource_group.scalable_microservice.name 35 | dns_prefix = var.dns_prefix 36 | kubernetes_version = var.kubernetes_version 37 | 38 | linux_profile { 39 | admin_username = "ubuntu" 40 | 41 | ssh_key { 42 | key_data = file(var.ssh_public_key) 43 | } 44 | } 45 | 46 | default_node_pool { 47 | name = "agentpool" 48 | node_count = var.agent_count 49 | max_count = 30 50 | min_count = var.agent_count 51 | vm_size = "Standard_DS2_v2" 52 | enable_auto_scaling = true 53 | } 54 | 55 | service_principal { 56 | client_id = var.client_id 57 | client_secret = var.client_secret 58 | } 59 | 60 | addon_profile { 61 | oms_agent { 62 | enabled = true 63 | log_analytics_workspace_id = azurerm_log_analytics_workspace.test.id 64 | } 65 | } 66 | 67 | tags = { 68 | Environment = "Development" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /services/operation/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask_cors import CORS 3 | from flask import request, Response 4 | import json 5 | import uuid as uuid_lib 6 | from lib.kafka import Kafka 7 | import logging 8 | import pymongo 9 | import sys 10 | import os 11 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 12 | 13 | app = Flask(__name__) 14 | CORS(app) 15 | 16 | topic_user_create = 'user-create' 17 | topic_user_create_response = 'user-create-response' 18 | kafka = Kafka('operation-service', [topic_user_create_response], [topic_user_create, topic_user_create_response]) 19 | 20 | allowed_operations = [topic_user_create] 21 | 22 | mongodb_connection_string = os.environ['MONGODB_CONNECTION_STRING'] 23 | mongodb_client = pymongo.MongoClient(mongodb_connection_string) 24 | mongodb_database = mongodb_client["operation_service"] 25 | mongodb_collection = mongodb_database["operation"] 26 | 27 | 28 | @app.route('/operation/create/', methods=['POST']) 29 | def action_operation_create(operation_name): 30 | uuid = uuid_lib.uuid4() 31 | content = request.json 32 | msg = { 33 | 'uuid': f'{uuid}', 34 | 'type': operation_name, 35 | 'state': 'pending', 36 | 'data': content, 37 | 'response': { 38 | 'state': 'pending' 39 | } 40 | } 41 | if operation_name in allowed_operations: 42 | kafka.create_message(operation_name, msg) 43 | else: 44 | msg['state'] = 'failed' 45 | msg['error'] = 'operation not allowed' 46 | 47 | result = mongodb_collection.insert_one(msg.copy()) 48 | logging.info(f'MongoDB insert: {result.inserted_id}') 49 | 50 | return Response(json.dumps(msg)) 51 | 52 | 53 | @app.route('/operation/get/', methods=['GET']) 54 | def action_operation_get(uuid): 55 | result = mongodb_collection.find_one({'uuid': uuid}) 56 | if result and '_id' in result: 57 | del result['_id'] 58 | logging.info(f'MongoDB find: {result}') 59 | 60 | return Response(json.dumps(result)) 61 | else: 62 | return Response(json.dumps({})) 63 | 64 | 65 | def update_operation_response(msg): 66 | if msg['response']['state'] == 'completed': 67 | msg['state'] = 'completed' 68 | else: 69 | msg['state'] = 'failed' 70 | msg['error'] = msg['response']['error'] 71 | msg_mongo = msg.copy() 72 | mongodb_collection.replace_one( 73 | {'uuid': msg_mongo['uuid']}, 74 | msg_mongo 75 | ) 76 | logging.info(f'MongoDB update done') 77 | 78 | 79 | def new_message_listener(msg): 80 | topic = msg.topic() 81 | data = json.loads(msg.value()) 82 | if topic == topic_user_create_response: 83 | update_operation_response(data) 84 | 85 | 86 | kafka.subscribe(new_message_listener, 'kafka_new_message') 87 | 88 | 89 | if __name__ == '__main__': 90 | app.run(debug=True, host='0.0.0.0', use_reloader=False) 91 | -------------------------------------------------------------------------------- /services/userApproval/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/confluentinc/confluent-kafka-go/kafka" 7 | "os" 8 | "os/signal" 9 | "runtime" 10 | "syscall" 11 | "time" 12 | ) 13 | 14 | type UserApproveEvent struct { 15 | Uuid string `json:"uuid"` 16 | Type string `json:"type"` 17 | State string `json:"state"` 18 | Data struct { 19 | Name string `json:"name"` 20 | CreatedAt string `json:"createdAt"` 21 | Uuid string `json:"uuid"` 22 | Approved string `json:"approved"` 23 | } `json:"data"` 24 | Response struct { 25 | State string `json:"state"` 26 | } `json:"response"` 27 | } 28 | 29 | func main() { 30 | bootstrapServers := os.Getenv("KAFKA_BOOTSTRAP_SERVERS") 31 | saslUsername := os.Getenv("KAFKA_SASL_USERNAME") 32 | saslPassword := os.Getenv("KAFKA_SASL_PASSWORD") 33 | 34 | topicUserApprove := "user-approve" 35 | topicUserApproveResponse := "user-approve-response" 36 | 37 | // Create Consumer instance 38 | c, err := kafka.NewConsumer(&kafka.ConfigMap{ 39 | "bootstrap.servers": bootstrapServers, 40 | "sasl.mechanisms": "PLAIN", 41 | "security.protocol": "SASL_SSL", 42 | "sasl.username": saslUsername, 43 | "sasl.password": saslPassword, 44 | "group.id": "user-approval-service", 45 | "auto.offset.reset": "earliest"}) 46 | if err != nil { 47 | fmt.Printf("Failed to create consumer: %s", err) 48 | os.Exit(1) 49 | } 50 | 51 | // Create Producer instance 52 | p, err := kafka.NewProducer(&kafka.ConfigMap{ 53 | "bootstrap.servers": bootstrapServers, 54 | "sasl.mechanisms": "PLAIN", 55 | "security.protocol": "SASL_SSL", 56 | "sasl.username": saslUsername, 57 | "sasl.password": saslPassword}) 58 | if err != nil { 59 | fmt.Printf("Failed to create producer: %s", err) 60 | os.Exit(1) 61 | } 62 | 63 | // Subscribe to topicUserApprove 64 | err = c.SubscribeTopics([]string{topicUserApprove}, nil) 65 | // Set up a channel for handling Ctrl-C, etc 66 | sigchan := make(chan os.Signal, 1) 67 | signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) 68 | 69 | // Process messages 70 | run := true 71 | for run == true { 72 | select { 73 | case sig := <-sigchan: 74 | fmt.Printf("Caught signal %v: terminating\n", sig) 75 | run = false 76 | default: 77 | msg, err := c.ReadMessage(100 * time.Millisecond) 78 | if err != nil { 79 | // Errors are informational and automatically handled by the consumer 80 | continue 81 | } 82 | recordKey := string(msg.Key) 83 | recordValue := msg.Value 84 | 85 | var data UserApproveEvent 86 | 87 | err = json.Unmarshal(recordValue, &data) 88 | if err != nil { 89 | fmt.Printf("Failed to decode JSON at offset %d: %v", msg.TopicPartition.Offset, err) 90 | continue 91 | } 92 | 93 | fmt.Printf("Consumed record with key %s and value %s\n", recordKey, recordValue) 94 | 95 | //fmt.Printf("Going to use CPU for: %d ms\n", milliSeconds) 96 | //milliSeconds := rand.Intn(1000) 97 | //cpuUsage(milliSeconds) 98 | 99 | milliSeconds := 200 // + rand.Intn(300) 100 | fmt.Printf("Going to sleep for: %d ms\n", milliSeconds) 101 | time.Sleep(time.Duration(milliSeconds) * time.Millisecond) 102 | 103 | data.Data.Approved = "false" 104 | recordValueSend, _ := json.Marshal(data) 105 | 106 | fmt.Printf("Preparing to produce record: %s\t%s\n", recordKey, recordValueSend) 107 | 108 | _ = p.Produce(&kafka.Message{ 109 | TopicPartition: kafka.TopicPartition{Topic: &topicUserApproveResponse, Partition: kafka.PartitionAny}, 110 | Key: []byte(recordKey), 111 | Value: []byte(recordValueSend), 112 | }, nil) 113 | 114 | fmt.Printf("Produced record with key %s and value %s\n", recordKey, recordValueSend) 115 | } 116 | } 117 | 118 | fmt.Printf("Closing\n") 119 | _ = c.Close() 120 | p.Close() 121 | } 122 | 123 | func cpuUsage(milliSeconds int) { 124 | n := runtime.NumCPU() 125 | runtime.GOMAXPROCS(n) 126 | 127 | quit := make(chan bool) 128 | 129 | for i := 0; i < n; i++ { 130 | go func() { 131 | for { 132 | select { 133 | case <-quit: 134 | return 135 | default: 136 | } 137 | } 138 | }() 139 | } 140 | 141 | time.Sleep(time.Duration(milliSeconds) * time.Millisecond) 142 | for i := 0; i < n; i++ { 143 | quit <- true 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /run/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | ####### credentials ####### 6 | FILE=.env.local 7 | if [[ ! -f "$FILE" ]]; then 8 | echo "$FILE does not exist. Copy .env to .env.local and fill in credentials" 9 | exit 10 | fi 11 | source .env.local 12 | export TF_VAR_azure_client_id TF_VAR_azure_client_secret CONFLUENT_CLOUD_USERNAME CONFLUENT_CLOUD_PASSWORD 13 | 14 | 15 | ####### terraform plugins ####### 16 | # install plugins... 17 | 18 | 19 | ####### terraform ####### 20 | cd infrastructure/terraform 21 | terraform init 22 | terraform apply -auto-approve 23 | #terraform refresh 24 | 25 | 26 | mkdir -p ../../auth 27 | echo "MONGODB_CONNECTION_STRING=$(terraform output azure-cosmos_db-connection_string_one)" > ../../auth/mongodb.env 28 | echo "KAFKA_BOOTSTRAP_SERVERS=$(terraform output ccloud-kafka-cluster_kafka_url)" > ../../auth/kafka.env 29 | echo "KAFKA_SASL_USERNAME=$(terraform output ccloud-kafka-cluster_key)" >> ../../auth/kafka.env 30 | echo "KAFKA_SASL_PASSWORD=$(terraform output ccloud-kafka-cluster_secret)" >> ../../auth/kafka.env 31 | terraform output azure-k8s-cluster_kube_config > ../../auth/kube_config 32 | AZURE_REGISTRY=$(terraform output azure-container-registry-login_server) 33 | DOCKER_LOGIN_COMMAND=$(terraform output azure-container-registry-docker_login) 34 | KUBERNETES_SECRET_CREATE_COMMAND=$(terraform output azure-container-registry-kubernetes_secret) 35 | echo ${DOCKER_LOGIN_COMMAND} 36 | echo ${KUBERNETES_SECRET_CREATE_COMMAND} 37 | 38 | ####### build & push containers ####### 39 | cd ../../ 40 | 41 | eval ${DOCKER_LOGIN_COMMAND} 42 | 43 | docker build -t ${AZURE_REGISTRY}/operation_service ./services/operation 44 | docker push ${AZURE_REGISTRY}/operation_service 45 | 46 | docker build -t ${AZURE_REGISTRY}/user_service ./services/user 47 | docker push ${AZURE_REGISTRY}/user_service 48 | 49 | docker build -t ${AZURE_REGISTRY}/user_approval_service ./services/userApproval 50 | docker push ${AZURE_REGISTRY}/user_approval_service 51 | 52 | 53 | ####### k8s ####### 54 | export KUBECONFIG=$(PWD)/auth/kube_config 55 | kubectl get node 56 | 57 | kubectl create secret generic kafka --from-env-file=./auth/kafka.env --dry-run -o yaml | kubectl apply -f - 58 | kubectl create secret generic mongodb --from-env-file=./auth/mongodb.env --dry-run -o yaml | kubectl apply -f - 59 | eval "${KUBERNETES_SECRET_CREATE_COMMAND} --dry-run -o yaml" | kubectl apply -f - 60 | kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "docker-rep-pull"}]}' 61 | 62 | kubectl apply -f infrastructure/k8s 63 | 64 | 65 | ####### helm ####### 66 | #kubectl create ns prometheus-adapter 67 | cd infrastructure/helm 68 | helmfile sync 69 | #helmfile --helm-binary "/usr/local/opt/helm@3/bin/helm" sync 70 | cd ../.. 71 | 72 | 73 | ####### output connection info ####### 74 | echo 75 | echo 76 | echo "##### SETUP DONE #####" 77 | echo "export KUBECONFIG=$(pwd)/auth/kube_config" 78 | export KUBECONFIG=$(pwd)/auth/kube_config 79 | 80 | ISTIO_INGRESS_IP=$(kubectl -n istio-system get svc istio-ingressgateway -o jsonpath={".status.loadBalancer.ingress[0].ip"}) 81 | 82 | if [[ -z "$ISTIO_INGRESS_IP" ]] 83 | then 84 | echo 85 | echo "Run the following to check for the Istio LoadBalancer IP:" 86 | echo 'kubectl -n istio-system get svc istio-ingressgateway -o jsonpath={".status.loadBalancer.ingress[0].ip"}' 87 | echo 88 | echo "Run the following to create a new operation to create a new user:" 89 | echo "curl --resolve 'operation-service.scalable-microservice:80:ISTIO_INGRESS_IP' http://operation-service.scalable-microservice/operation/create/user-create -X POST -H 'Content-Type: application/json' -d '{\"name\": \"hans\"}'" 90 | echo 91 | echo "Run the following to get an operation status:" 92 | echo "curl --resolve 'operation-service.scalable-microservice:80:ISTIO_INGRESS_IP' http://operation-service.scalable-microservice/operation/get/f591a1e4-fb9e-459b-baa2-19ccd82fc84f" 93 | echo 94 | else 95 | echo 96 | echo "Run the following to create a new operation to create a new user:" 97 | echo "curl --resolve 'operation-service.scalable-microservice:80:${ISTIO_INGRESS_IP}' http://operation-service.scalable-microservice/operation/create/user-create -X POST -H 'Content-Type: application/json' -d '{\"name\": \"hans\"}'" 98 | echo 99 | echo "Run the following to get an operation status:" 100 | echo "curl --resolve 'operation-service.scalable-microservice:80:${ISTIO_INGRESS_IP}' http://operation-service.scalable-microservice/operation/get/f591a1e4-fb9e-459b-baa2-19ccd82fc84f" 101 | echo 102 | fi 103 | -------------------------------------------------------------------------------- /services/user/lib/kafka.py: -------------------------------------------------------------------------------- 1 | from confluent_kafka import Producer, Consumer 2 | from confluent_kafka.admin import AdminClient, NewTopic 3 | import json 4 | import os 5 | import threading 6 | import logging 7 | import sys 8 | from pubsub import pub 9 | 10 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 11 | 12 | bootstrap_servers = os.environ['KAFKA_BOOTSTRAP_SERVERS'] 13 | sasl_username = os.environ['KAFKA_SASL_USERNAME'] 14 | sasl_password = os.environ['KAFKA_SASL_PASSWORD'] 15 | 16 | 17 | class Kafka: 18 | __p = None 19 | __c = None 20 | __a = None 21 | __poll_thread = None 22 | __topics_consume = [] 23 | __topics_create = [] 24 | __messages = {} 25 | 26 | def __init__(self, group_id, topics_consume, topics_create=None): 27 | self.__topics_consume = topics_consume 28 | self.__topics_create = topics_create 29 | 30 | self.__p = Producer({ 31 | 'bootstrap.servers': bootstrap_servers, 32 | 'sasl.mechanisms': 'PLAIN', 33 | 'security.protocol': 'SASL_SSL', 34 | 'sasl.username': sasl_username, 35 | 'sasl.password': sasl_password, 36 | }) 37 | 38 | self.__c = Consumer({ 39 | 'bootstrap.servers': bootstrap_servers, 40 | 'sasl.mechanisms': 'PLAIN', 41 | 'security.protocol': 'SASL_SSL', 42 | 'sasl.username': sasl_username, 43 | 'sasl.password': sasl_password, 44 | 'group.id': group_id, 45 | 'auto.offset.reset': 'earliest' 46 | }) 47 | 48 | self.__a = AdminClient({ 49 | 'bootstrap.servers': bootstrap_servers, 50 | 'sasl.mechanisms': 'PLAIN', 51 | 'security.protocol': 'SASL_SSL', 52 | 'sasl.username': sasl_username, 53 | 'sasl.password': sasl_password, 54 | 'group.id': group_id, 55 | 'auto.offset.reset': 'earliest' 56 | }) 57 | 58 | if topics_create: 59 | self.__create_topics(topics_create) 60 | 61 | self.__poll_thread = threading.Thread(target=self.__thread_consume) 62 | self.__poll_thread.start() 63 | self.subscribe(self.__new_message_listener, 'kafka_new_message') 64 | 65 | @staticmethod 66 | def subscribe(listener, topic_name): 67 | pub.subscribe(listener, topic_name) 68 | 69 | def create_message(self, topic, value): 70 | """ 71 | create a new Kafka message 72 | """ 73 | logging.info(f'Producing record: {value}') 74 | self.__p.produce(topic, value=json.dumps(value), on_delivery=self.__create_message_acked) 75 | self.__p.poll(0) 76 | self.__p.flush() 77 | 78 | def __create_topics(self, topics): 79 | """ Create topics """ 80 | new_topics = [NewTopic(topic, num_partitions=6, replication_factor=3) for topic in topics] 81 | # Call create_topics to asynchronously create topics, a dict 82 | # of is returned. 83 | fs = self.__a.create_topics(new_topics) 84 | 85 | # Wait for operation to finish. 86 | # Timeouts are preferably controlled by passing request_timeout=15.0 87 | # to the create_topics() call. 88 | # All futures will finish at the same time. 89 | for topic, f in fs.items(): 90 | try: 91 | f.result() # The result itself is None 92 | print("Topic {} created".format(topic)) 93 | except Exception as e: 94 | print("Failed to create topic {}: {}".format(topic, e)) 95 | 96 | def __thread_consume(self): 97 | """ 98 | endless loop polling Kafka for new messages 99 | """ 100 | self.__c.subscribe(self.__topics_consume) 101 | try: 102 | while True: 103 | msg = self.__c.poll(0.1) 104 | if msg is None: 105 | # logging.info('poll') 106 | continue 107 | elif msg.error(): 108 | logging.error('error: {}'.format(msg.error())) 109 | continue 110 | else: 111 | pub.sendMessage('kafka_new_message', msg=msg) 112 | except KeyboardInterrupt: 113 | self.__c.close() 114 | return None 115 | 116 | def __new_message_listener(self, msg): 117 | topic = msg.topic() 118 | data = json.loads(msg.value()) 119 | logging.info(f'Consumed record with topic:{topic} data:{data}') 120 | uuid = data['uuid'] 121 | messages_key = f'{topic}_{uuid}' 122 | if messages_key in self.__messages and self.__messages[messages_key] == 'awaiting': 123 | self.__messages[messages_key] = data 124 | 125 | @staticmethod 126 | def __create_message_acked(err, msg): 127 | if err is not None: 128 | logging.error('Failed to deliver message: {}'.format(err)) 129 | else: 130 | logging.info('Produced record to topic {} partition [{}] @ offset {}'.format(msg.topic(), msg.partition(), 131 | msg.offset())) 132 | -------------------------------------------------------------------------------- /services/operation/lib/kafka.py: -------------------------------------------------------------------------------- 1 | from confluent_kafka import Producer, Consumer 2 | from confluent_kafka.admin import AdminClient, NewTopic 3 | import json 4 | import os 5 | import threading 6 | import logging 7 | import sys 8 | from pubsub import pub 9 | 10 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 11 | 12 | bootstrap_servers = os.environ['KAFKA_BOOTSTRAP_SERVERS'] 13 | sasl_username = os.environ['KAFKA_SASL_USERNAME'] 14 | sasl_password = os.environ['KAFKA_SASL_PASSWORD'] 15 | 16 | 17 | class Kafka: 18 | __p = None 19 | __c = None 20 | __a = None 21 | __poll_thread = None 22 | __topics_consume = [] 23 | __topics_create = [] 24 | __messages = {} 25 | 26 | def __init__(self, group_id, topics_consume, topics_create=None): 27 | self.__topics_consume = topics_consume 28 | self.__topics_create = topics_create 29 | 30 | self.__p = Producer({ 31 | 'bootstrap.servers': bootstrap_servers, 32 | 'sasl.mechanisms': 'PLAIN', 33 | 'security.protocol': 'SASL_SSL', 34 | 'sasl.username': sasl_username, 35 | 'sasl.password': sasl_password, 36 | }) 37 | 38 | self.__c = Consumer({ 39 | 'bootstrap.servers': bootstrap_servers, 40 | 'sasl.mechanisms': 'PLAIN', 41 | 'security.protocol': 'SASL_SSL', 42 | 'sasl.username': sasl_username, 43 | 'sasl.password': sasl_password, 44 | 'group.id': group_id, 45 | 'auto.offset.reset': 'earliest' 46 | }) 47 | 48 | self.__a = AdminClient({ 49 | 'bootstrap.servers': bootstrap_servers, 50 | 'sasl.mechanisms': 'PLAIN', 51 | 'security.protocol': 'SASL_SSL', 52 | 'sasl.username': sasl_username, 53 | 'sasl.password': sasl_password, 54 | 'group.id': group_id, 55 | 'auto.offset.reset': 'earliest' 56 | }) 57 | 58 | if topics_create: 59 | self.__create_topics(topics_create) 60 | 61 | self.__poll_thread = threading.Thread(target=self.__thread_consume) 62 | self.__poll_thread.start() 63 | self.subscribe(self.__new_message_listener, 'kafka_new_message') 64 | 65 | @staticmethod 66 | def subscribe(listener, topic_name): 67 | pub.subscribe(listener, topic_name) 68 | 69 | def create_message(self, topic, value): 70 | """ 71 | create a new Kafka message 72 | """ 73 | logging.info(f'Producing record: {value}') 74 | self.__p.produce(topic, value=json.dumps(value), on_delivery=self.__create_message_acked) 75 | self.__p.poll(0) 76 | self.__p.flush() 77 | 78 | def __create_topics(self, topics): 79 | """ Create topics """ 80 | new_topics = [NewTopic(topic, num_partitions=6, replication_factor=3) for topic in topics] 81 | # Call create_topics to asynchronously create topics, a dict 82 | # of is returned. 83 | fs = self.__a.create_topics(new_topics) 84 | 85 | # Wait for operation to finish. 86 | # Timeouts are preferably controlled by passing request_timeout=15.0 87 | # to the create_topics() call. 88 | # All futures will finish at the same time. 89 | for topic, f in fs.items(): 90 | try: 91 | f.result() # The result itself is None 92 | print("Topic {} created".format(topic)) 93 | except Exception as e: 94 | print("Failed to create topic {}: {}".format(topic, e)) 95 | 96 | def __thread_consume(self): 97 | """ 98 | endless loop polling Kafka for new messages 99 | """ 100 | self.__c.subscribe(self.__topics_consume) 101 | try: 102 | while True: 103 | msg = self.__c.poll(0.1) 104 | if msg is None: 105 | # logging.info('poll') 106 | continue 107 | elif msg.error(): 108 | logging.error('error: {}'.format(msg.error())) 109 | continue 110 | else: 111 | pub.sendMessage('kafka_new_message', msg=msg) 112 | except KeyboardInterrupt: 113 | self.__c.close() 114 | return None 115 | 116 | def __new_message_listener(self, msg): 117 | topic = msg.topic() 118 | data = json.loads(msg.value()) 119 | logging.info(f'Consumed record with topic:{topic} data:{data}') 120 | uuid = data['uuid'] 121 | messages_key = f'{topic}_{uuid}' 122 | if messages_key in self.__messages and self.__messages[messages_key] == 'awaiting': 123 | self.__messages[messages_key] = data 124 | 125 | @staticmethod 126 | def __create_message_acked(err, msg): 127 | if err is not None: 128 | logging.error('Failed to deliver message: {}'.format(err)) 129 | else: 130 | logging.info('Produced record to topic {} partition [{}] @ offset {}'.format(msg.topic(), msg.partition(), 131 | msg.offset())) 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scalable Microservice Infrastructure Example with K8s Istio Kafka 2 | 3 | Medium article: https://medium.com/@wuestkamp/scalable-microservice-demo-k8s-istio-kafka-344a2610eba3?sk=7404e77f2a42d21261707794afaed58d 4 | 5 | 6 | ## Setup project 7 | Install: 8 | * terraform 9 | * https://github.com/Mongey/terraform-provider-confluent-cloud 10 | * kubectl 11 | * docker 12 | * helm 13 | * helmfile 14 | 15 | Then: 16 | ``` 17 | cp .env .env.local 18 | # fill .env.local with correct values 19 | 20 | ./run/install.sh 21 | ``` 22 | 23 | ## Run 24 | Edit `infrastructure/k8s/rest-request.yaml` deployment to fetch the correct Istio Gateway IP. Then slowly 25 | raise its replicas and watch the metrics. 26 | 27 | 28 | 29 | ## Monitor Cluster 30 | 31 | ### Grafana 32 | Run Grafana and then import the `./infrastructure/grafana/dashboard.json`. 33 | 34 | ``` 35 | ./run/grafana.sh # admin:admin 36 | ``` 37 | 38 | ### Kiali 39 | ``` 40 | ./run/kiali.sh # admin:admin 41 | ``` 42 | 43 | 44 | 45 | ## Use/build Services manually 46 | 47 | ### Operation Service 48 | 49 | #### build and run 50 | ``` 51 | cd services/operation 52 | 53 | # pass env file 54 | docker build -t operation-service . && \ 55 | docker run --env-file ../../auth/kafka.env \ 56 | --env-file ../../auth/mongodb.env \ 57 | -p 80:5000 \ 58 | operation-service 59 | 60 | 61 | # build and pass all parameters manually 62 | docker build -t operation-service . && \ 63 | docker run -e "KAFKA_BOOTSTRAP_SERVERS=server:9092" \ 64 | -e "KAFKA_SASL_USERNAME=XXX" \ 65 | -e "KAFKA_SASL_PASSWORD=XXX" \ 66 | -e "MONGODB_CONNECTION_STRING=mongodb://" \ 67 | -p 80:5000 \ 68 | operation-service 69 | ``` 70 | 71 | #### call and use 72 | ``` 73 | # add user 74 | curl -X POST \ 75 | -H 'Content-Type: application/json' \ 76 | -d '{"name": "hans"}' \ 77 | "http://localhost:80/operation/create/user-create" 78 | 79 | curl -X POST -H 'Content-Type: application/json' -d '{"name": "hans"}' "http://localhost:80/operation/create/user-create" 80 | 81 | # get operation status 82 | curl "http://localhost:80/operation/get/6d232092-dceb-419f-bff1-2d686eace56c" 83 | ``` 84 | 85 | 86 | ### User Service 87 | 88 | #### build and run 89 | ``` 90 | cd services/user 91 | 92 | docker build -t user-service . && \ 93 | docker run --env-file ../../auth/kafka.env \ 94 | --env-file ../../auth/mongodb.env \ 95 | user-service 96 | 97 | # build and pass all env variables manually 98 | docker build -t user-service . && \ 99 | docker run -e "KAFKA_BOOTSTRAP_SERVERS=server:9092" \ 100 | -e "KAFKA_SASL_USERNAME=XXX" \ 101 | -e "KAFKA_SASL_PASSWORD=XXX" \ 102 | -e "MONGODB_CONNECTION_STRING=mongodb://" \ 103 | user-service 104 | ``` 105 | 106 | 107 | ### User Approval Service 108 | 109 | #### build and run 110 | ``` 111 | docker build -t user-approval-service . && \ 112 | docker run --env-file ../../auth/kafka.env \ 113 | user-approval-service 114 | ``` 115 | 116 | 117 | 118 | 119 | ## Setup project step by step 120 | 121 | ### Terraform 122 | https://docs.microsoft.com/en-us/azure/terraform/terraform-create-k8s-cluster-with-tf-and-aks 123 | 124 | #### Plugins 125 | 126 | install Terraform ConfluenceCloud plugin 127 | ``` 128 | TF_PLUGIN_VERSION=0.0.1 129 | mkdir ~/.terraform.d/plugins/darwin_amd64 130 | cd ~/tmp 131 | wget "https://github.com/Mongey/terraform-provider-confluent-cloud/releases/download/v${TF_PLUGIN_VERSION}/terraform-provider-confluent-cloud_${TF_PLUGIN_VERSION}_darwin_amd64.tar.gz" 132 | tar xzf terraform-provider-confluent-cloud_${TF_PLUGIN_VERSION}_darwin_amd64.tar.gz 133 | mv terraform-provider-confluent-cloud_v${TF_PLUGIN_VERSION} ~/.terraform.d/plugins/darwin_amd64/ 134 | ``` 135 | 136 | #### Init 137 | ``` 138 | terraform init 139 | ``` 140 | 141 | #### Plan & Apply 142 | ``` 143 | cp .env .env.local 144 | # fill .env.local with correct values 145 | 146 | source .env.local 147 | export TF_VAR_azure_client_id TF_VAR_azure_client_secret CONFLUENT_CLOUD_USERNAME CONFLUENT_CLOUD_PASSWORD 148 | 149 | cd infrastructure/terraform 150 | terraform apply 151 | ``` 152 | 153 | #### Connect 154 | ``` 155 | echo "$(terraform output azure-k8s-cluster_kube_config)" > ./kube_config 156 | export KUBECONFIG=$(PWD)/kube_config 157 | kubectl get node 158 | ``` 159 | 160 | 161 | ### Istio 162 | Istio is already persisted at `infrastructure/k8s/istio-system.yaml`. It was generated using: 163 | ``` 164 | istioctl manifest generate --set values.kiali.enabled=true --set values.tracing.enabled=true --set values.grafana.enabled=true --set values.prometheus.enabled=true 165 | ``` 166 | 167 | 168 | ### Apply K8s Resources 169 | ``` 170 | kubectl apply -f infrastructure/k8s 171 | ``` 172 | 173 | 174 | ### Install Helm Charts 175 | If your k8s provider doesn't install metrics-server you need to enable in it `infrastructure/helm/helmfile.yaml` 176 | and the namespace in `infrastructure/k8s/namespaces.yaml`. 177 | 178 | ``` 179 | cd infrastructure/helm 180 | 181 | # as of now I had to specify the path to helm3 manually because I also had helm2 installed 182 | helmfile --helm-binary "/usr/local/opt/helm@3/bin/helm" diff 183 | helmfile --helm-binary "/usr/local/opt/helm@3/bin/helm" sync 184 | ``` 185 | 186 | 187 | ## view Terraform credentials 188 | ``` 189 | terraform refresh 190 | ``` 191 | 192 | 193 | ## Kafka commands 194 | 195 | ### Apache Kafka CLI 196 | Use Apache Kafka CLI tools with ConfluentCloud: 197 | https://www.confluent.io/blog/using-apache-kafka-command-line-tools-confluent-cloud 198 | 199 | First create the file `kafka.config.properties` and add the API key and token. 200 | 201 | ``` 202 | kafka-console-producer --broker-list pkc-e8mp5.eu-west-1.aws.confluent.cloud:9092 --producer.config kafka.config.properties --topic user-create 203 | kafka-console-consumer --bootstrap-server pkc-e8mp5.eu-west-1.aws.confluent.cloud:9092 --consumer.config kafka.config.properties --topic user-create 204 | ``` 205 | 206 | 207 | 208 | ### Confluent Cloud CLI 209 | ``` 210 | ccloud api-key create --resource lkc-1j98z 211 | ccloud api-key use XXX --resource lkc-1j98z 212 | 213 | ccloud kafka topic create test-topic 214 | ccloud kafka topic produce test-topic 215 | ccloud kafka topic consume test-topic 216 | ``` 217 | -------------------------------------------------------------------------------- /infrastructure/grafana/dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS", 5 | "label": "Prometheus", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "grafana", 15 | "id": "grafana", 16 | "name": "Grafana", 17 | "version": "6.4.3" 18 | }, 19 | { 20 | "type": "panel", 21 | "id": "graph", 22 | "name": "Graph", 23 | "version": "" 24 | }, 25 | { 26 | "type": "datasource", 27 | "id": "prometheus", 28 | "name": "Prometheus", 29 | "version": "1.0.0" 30 | }, 31 | { 32 | "type": "panel", 33 | "id": "singlestat", 34 | "name": "Singlestat", 35 | "version": "" 36 | } 37 | ], 38 | "annotations": { 39 | "list": [ 40 | { 41 | "builtIn": 1, 42 | "datasource": "-- Grafana --", 43 | "enable": true, 44 | "hide": true, 45 | "iconColor": "rgba(0, 211, 255, 1)", 46 | "name": "Annotations & Alerts", 47 | "type": "dashboard" 48 | } 49 | ] 50 | }, 51 | "editable": true, 52 | "gnetId": null, 53 | "graphTooltip": 0, 54 | "id": null, 55 | "links": [], 56 | "panels": [ 57 | { 58 | "cacheTimeout": null, 59 | "colorBackground": true, 60 | "colorValue": false, 61 | "colors": [ 62 | "#1F60C4", 63 | "#F2495C", 64 | "#C0D8FF" 65 | ], 66 | "datasource": "${DS_PROMETHEUS}", 67 | "decimals": null, 68 | "format": "none", 69 | "gauge": { 70 | "maxValue": 100, 71 | "minValue": 0, 72 | "show": false, 73 | "thresholdLabels": false, 74 | "thresholdMarkers": true 75 | }, 76 | "gridPos": { 77 | "h": 8, 78 | "w": 6, 79 | "x": 0, 80 | "y": 0 81 | }, 82 | "id": 37, 83 | "interval": null, 84 | "links": [], 85 | "mappingType": 1, 86 | "mappingTypes": [ 87 | { 88 | "name": "value to text", 89 | "value": 1 90 | }, 91 | { 92 | "name": "range to text", 93 | "value": 2 94 | } 95 | ], 96 | "maxDataPoints": 100, 97 | "nullPointMode": "connected", 98 | "nullText": null, 99 | "options": {}, 100 | "postfix": " e/s", 101 | "postfixFontSize": "50%", 102 | "prefix": "", 103 | "prefixFontSize": "50%", 104 | "rangeMaps": [ 105 | { 106 | "from": "null", 107 | "text": "N/A", 108 | "to": "null" 109 | } 110 | ], 111 | "sparkline": { 112 | "fillColor": "rgba(31, 118, 189, 0.18)", 113 | "full": false, 114 | "lineColor": "rgb(31, 120, 193)", 115 | "show": false, 116 | "ymax": null, 117 | "ymin": null 118 | }, 119 | "tableColumn": "", 120 | "targets": [ 121 | { 122 | "expr": "sum(rate(kafka_topic_partition_current_offset[1m]))", 123 | "refId": "A" 124 | } 125 | ], 126 | "thresholds": "", 127 | "timeFrom": null, 128 | "timeShift": null, 129 | "title": "Events/Sec", 130 | "type": "singlestat", 131 | "valueFontSize": "80%", 132 | "valueMaps": [ 133 | { 134 | "op": "=", 135 | "text": "N/A", 136 | "value": "null" 137 | } 138 | ], 139 | "valueName": "current" 140 | }, 141 | { 142 | "aliasColors": { 143 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\",consumergroup=\"user-approval-service\"}[1m]))": "red", 144 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\"}[1m]))": "red" 145 | }, 146 | "bars": false, 147 | "dashLength": 10, 148 | "dashes": false, 149 | "datasource": "${DS_PROMETHEUS}", 150 | "fill": 1, 151 | "fillGradient": 0, 152 | "gridPos": { 153 | "h": 8, 154 | "w": 6, 155 | "x": 6, 156 | "y": 0 157 | }, 158 | "id": 29, 159 | "legend": { 160 | "avg": false, 161 | "current": false, 162 | "max": false, 163 | "min": false, 164 | "show": true, 165 | "total": false, 166 | "values": false 167 | }, 168 | "lines": true, 169 | "linewidth": 1, 170 | "nullPointMode": "null", 171 | "options": { 172 | "dataLinks": [] 173 | }, 174 | "percentage": false, 175 | "pointradius": 2, 176 | "points": false, 177 | "renderer": "flot", 178 | "seriesOverrides": [], 179 | "spaceLength": 10, 180 | "stack": false, 181 | "steppedLine": false, 182 | "targets": [ 183 | { 184 | "expr": "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\",consumergroup=\"user-approval-service\"}[1m]))", 185 | "hide": false, 186 | "refId": "A" 187 | }, 188 | { 189 | "expr": "kafka_consumergroup_lag", 190 | "hide": true, 191 | "refId": "B" 192 | } 193 | ], 194 | "thresholds": [], 195 | "timeFrom": null, 196 | "timeRegions": [], 197 | "timeShift": null, 198 | "title": "Topic user-approve lag", 199 | "tooltip": { 200 | "shared": true, 201 | "sort": 0, 202 | "value_type": "individual" 203 | }, 204 | "type": "graph", 205 | "xaxis": { 206 | "buckets": null, 207 | "mode": "time", 208 | "name": null, 209 | "show": true, 210 | "values": [] 211 | }, 212 | "yaxes": [ 213 | { 214 | "format": "none", 215 | "label": null, 216 | "logBase": 1, 217 | "max": null, 218 | "min": null, 219 | "show": true 220 | }, 221 | { 222 | "format": "short", 223 | "label": null, 224 | "logBase": 1, 225 | "max": null, 226 | "min": null, 227 | "show": true 228 | } 229 | ], 230 | "yaxis": { 231 | "align": false, 232 | "alignLevel": null 233 | } 234 | }, 235 | { 236 | "aliasColors": {}, 237 | "bars": false, 238 | "dashLength": 10, 239 | "dashes": false, 240 | "datasource": "${DS_PROMETHEUS}", 241 | "fill": 1, 242 | "fillGradient": 0, 243 | "gridPos": { 244 | "h": 8, 245 | "w": 6, 246 | "x": 12, 247 | "y": 0 248 | }, 249 | "id": 33, 250 | "legend": { 251 | "avg": false, 252 | "current": false, 253 | "max": false, 254 | "min": false, 255 | "show": true, 256 | "total": false, 257 | "values": false 258 | }, 259 | "lines": true, 260 | "linewidth": 1, 261 | "nullPointMode": "null", 262 | "options": { 263 | "dataLinks": [] 264 | }, 265 | "percentage": false, 266 | "pointradius": 2, 267 | "points": false, 268 | "renderer": "flot", 269 | "seriesOverrides": [], 270 | "spaceLength": 10, 271 | "stack": false, 272 | "steppedLine": false, 273 | "targets": [ 274 | { 275 | "expr": "count(irate(process_cpu_seconds_total{pod_name=~\"user-approval-service-.*\"}[30s]))", 276 | "refId": "C" 277 | } 278 | ], 279 | "thresholds": [], 280 | "timeFrom": null, 281 | "timeRegions": [], 282 | "timeShift": null, 283 | "title": "user-approval-service Pods", 284 | "tooltip": { 285 | "shared": true, 286 | "sort": 0, 287 | "value_type": "individual" 288 | }, 289 | "type": "graph", 290 | "xaxis": { 291 | "buckets": null, 292 | "mode": "time", 293 | "name": null, 294 | "show": true, 295 | "values": [] 296 | }, 297 | "yaxes": [ 298 | { 299 | "format": "none", 300 | "label": null, 301 | "logBase": 1, 302 | "max": null, 303 | "min": null, 304 | "show": true 305 | }, 306 | { 307 | "format": "short", 308 | "label": null, 309 | "logBase": 1, 310 | "max": null, 311 | "min": null, 312 | "show": true 313 | } 314 | ], 315 | "yaxis": { 316 | "align": false, 317 | "alignLevel": null 318 | } 319 | }, 320 | { 321 | "aliasColors": {}, 322 | "bars": false, 323 | "dashLength": 10, 324 | "dashes": false, 325 | "datasource": "${DS_PROMETHEUS}", 326 | "fill": 1, 327 | "fillGradient": 0, 328 | "gridPos": { 329 | "h": 8, 330 | "w": 6, 331 | "x": 18, 332 | "y": 0 333 | }, 334 | "id": 34, 335 | "legend": { 336 | "avg": false, 337 | "current": false, 338 | "max": false, 339 | "min": false, 340 | "show": true, 341 | "total": false, 342 | "values": false 343 | }, 344 | "lines": true, 345 | "linewidth": 1, 346 | "nullPointMode": "null", 347 | "options": { 348 | "dataLinks": [] 349 | }, 350 | "percentage": false, 351 | "pointradius": 2, 352 | "points": false, 353 | "renderer": "flot", 354 | "seriesOverrides": [], 355 | "spaceLength": 10, 356 | "stack": false, 357 | "steppedLine": false, 358 | "targets": [ 359 | { 360 | "expr": "sum(machine_cpu_cores) / 2", 361 | "refId": "A" 362 | } 363 | ], 364 | "thresholds": [], 365 | "timeFrom": null, 366 | "timeRegions": [], 367 | "timeShift": null, 368 | "title": "Nodes", 369 | "tooltip": { 370 | "shared": true, 371 | "sort": 0, 372 | "value_type": "individual" 373 | }, 374 | "type": "graph", 375 | "xaxis": { 376 | "buckets": null, 377 | "mode": "time", 378 | "name": null, 379 | "show": true, 380 | "values": [] 381 | }, 382 | "yaxes": [ 383 | { 384 | "format": "none", 385 | "label": null, 386 | "logBase": 1, 387 | "max": null, 388 | "min": null, 389 | "show": true 390 | }, 391 | { 392 | "format": "short", 393 | "label": null, 394 | "logBase": 1, 395 | "max": null, 396 | "min": null, 397 | "show": true 398 | } 399 | ], 400 | "yaxis": { 401 | "align": false, 402 | "alignLevel": null 403 | } 404 | }, 405 | { 406 | "aliasColors": { 407 | "sum(rate(istio_requests_total{destination_service_name=\"operation-service\", reporter=\"source\"}[1m]))": "purple", 408 | "sum(rate(istio_requests_total{destination_service_name=\"operation-service\"}[1m]))": "purple" 409 | }, 410 | "bars": false, 411 | "dashLength": 10, 412 | "dashes": false, 413 | "datasource": "${DS_PROMETHEUS}", 414 | "fill": 1, 415 | "fillGradient": 0, 416 | "gridPos": { 417 | "h": 8, 418 | "w": 24, 419 | "x": 0, 420 | "y": 8 421 | }, 422 | "id": 28, 423 | "legend": { 424 | "avg": false, 425 | "current": false, 426 | "max": false, 427 | "min": false, 428 | "show": true, 429 | "total": false, 430 | "values": false 431 | }, 432 | "lines": true, 433 | "linewidth": 1, 434 | "nullPointMode": "null", 435 | "options": { 436 | "dataLinks": [] 437 | }, 438 | "percentage": false, 439 | "pointradius": 2, 440 | "points": false, 441 | "renderer": "flot", 442 | "seriesOverrides": [], 443 | "spaceLength": 10, 444 | "stack": false, 445 | "steppedLine": false, 446 | "targets": [ 447 | { 448 | "expr": "sum(rate(istio_requests_total{destination_service_name=\"operation-service\", reporter=\"source\"}[1m]))", 449 | "hide": false, 450 | "refId": "A" 451 | }, 452 | { 453 | "expr": "istio_requests_total{destination_service_name=\"operation-service\", reporter=\"source\"}", 454 | "hide": true, 455 | "refId": "B" 456 | }, 457 | { 458 | "expr": "rate(istio_requests_total{destination_service_name=\"operation-service\", reporter=\"destination\"}[1m])", 459 | "hide": true, 460 | "refId": "C" 461 | } 462 | ], 463 | "thresholds": [], 464 | "timeFrom": null, 465 | "timeRegions": [], 466 | "timeShift": null, 467 | "title": "Istio Requests", 468 | "tooltip": { 469 | "shared": true, 470 | "sort": 0, 471 | "value_type": "individual" 472 | }, 473 | "type": "graph", 474 | "xaxis": { 475 | "buckets": null, 476 | "mode": "time", 477 | "name": null, 478 | "show": true, 479 | "values": [] 480 | }, 481 | "yaxes": [ 482 | { 483 | "format": "short", 484 | "label": null, 485 | "logBase": 1, 486 | "max": null, 487 | "min": null, 488 | "show": true 489 | }, 490 | { 491 | "format": "short", 492 | "label": null, 493 | "logBase": 1, 494 | "max": null, 495 | "min": null, 496 | "show": true 497 | } 498 | ], 499 | "yaxis": { 500 | "align": false, 501 | "alignLevel": null 502 | } 503 | }, 504 | { 505 | "aliasColors": {}, 506 | "bars": false, 507 | "dashLength": 10, 508 | "dashes": false, 509 | "datasource": "${DS_PROMETHEUS}", 510 | "fill": 1, 511 | "fillGradient": 0, 512 | "gridPos": { 513 | "h": 8, 514 | "w": 6, 515 | "x": 0, 516 | "y": 16 517 | }, 518 | "id": 20, 519 | "legend": { 520 | "avg": false, 521 | "current": false, 522 | "max": false, 523 | "min": false, 524 | "show": true, 525 | "total": false, 526 | "values": false 527 | }, 528 | "lines": true, 529 | "linewidth": 1, 530 | "nullPointMode": "null", 531 | "options": { 532 | "dataLinks": [] 533 | }, 534 | "percentage": false, 535 | "pointradius": 2, 536 | "points": false, 537 | "renderer": "flot", 538 | "seriesOverrides": [], 539 | "spaceLength": 10, 540 | "stack": false, 541 | "steppedLine": false, 542 | "targets": [ 543 | { 544 | "expr": "sum(machine_cpu_cores) / 2", 545 | "refId": "A" 546 | } 547 | ], 548 | "thresholds": [], 549 | "timeFrom": null, 550 | "timeRegions": [], 551 | "timeShift": null, 552 | "title": "Nodes", 553 | "tooltip": { 554 | "shared": true, 555 | "sort": 0, 556 | "value_type": "individual" 557 | }, 558 | "type": "graph", 559 | "xaxis": { 560 | "buckets": null, 561 | "mode": "time", 562 | "name": null, 563 | "show": true, 564 | "values": [] 565 | }, 566 | "yaxes": [ 567 | { 568 | "format": "none", 569 | "label": null, 570 | "logBase": 1, 571 | "max": null, 572 | "min": null, 573 | "show": true 574 | }, 575 | { 576 | "format": "short", 577 | "label": null, 578 | "logBase": 1, 579 | "max": null, 580 | "min": null, 581 | "show": true 582 | } 583 | ], 584 | "yaxis": { 585 | "align": false, 586 | "alignLevel": null 587 | } 588 | }, 589 | { 590 | "aliasColors": {}, 591 | "bars": false, 592 | "dashLength": 10, 593 | "dashes": false, 594 | "datasource": "${DS_PROMETHEUS}", 595 | "fill": 1, 596 | "fillGradient": 0, 597 | "gridPos": { 598 | "h": 8, 599 | "w": 6, 600 | "x": 6, 601 | "y": 16 602 | }, 603 | "id": 26, 604 | "legend": { 605 | "avg": false, 606 | "current": false, 607 | "max": false, 608 | "min": false, 609 | "show": true, 610 | "total": false, 611 | "values": false 612 | }, 613 | "lines": true, 614 | "linewidth": 1, 615 | "nullPointMode": "null", 616 | "options": { 617 | "dataLinks": [] 618 | }, 619 | "percentage": false, 620 | "pointradius": 2, 621 | "points": false, 622 | "renderer": "flot", 623 | "seriesOverrides": [], 624 | "spaceLength": 10, 625 | "stack": false, 626 | "steppedLine": false, 627 | "targets": [ 628 | { 629 | "expr": "count(irate(process_cpu_seconds_total{pod_name=~\"user-approval-service-.*\"}[30s]))", 630 | "refId": "C" 631 | } 632 | ], 633 | "thresholds": [], 634 | "timeFrom": null, 635 | "timeRegions": [], 636 | "timeShift": null, 637 | "title": "user-approval-service Pods", 638 | "tooltip": { 639 | "shared": true, 640 | "sort": 0, 641 | "value_type": "individual" 642 | }, 643 | "type": "graph", 644 | "xaxis": { 645 | "buckets": null, 646 | "mode": "time", 647 | "name": null, 648 | "show": true, 649 | "values": [] 650 | }, 651 | "yaxes": [ 652 | { 653 | "format": "none", 654 | "label": null, 655 | "logBase": 1, 656 | "max": null, 657 | "min": null, 658 | "show": true 659 | }, 660 | { 661 | "format": "short", 662 | "label": null, 663 | "logBase": 1, 664 | "max": null, 665 | "min": null, 666 | "show": true 667 | } 668 | ], 669 | "yaxis": { 670 | "align": false, 671 | "alignLevel": null 672 | } 673 | }, 674 | { 675 | "aliasColors": { 676 | "sum(rate(kafka_topic_partition_current_offset[1m]))": "purple", 677 | "sum(rate(kafka_topic_partition_current_offset{topic=\"user-create\"}[1m]))": "purple" 678 | }, 679 | "bars": false, 680 | "dashLength": 10, 681 | "dashes": false, 682 | "datasource": "${DS_PROMETHEUS}", 683 | "fill": 1, 684 | "fillGradient": 0, 685 | "gridPos": { 686 | "h": 8, 687 | "w": 12, 688 | "x": 12, 689 | "y": 16 690 | }, 691 | "id": 2, 692 | "legend": { 693 | "avg": false, 694 | "current": false, 695 | "max": false, 696 | "min": false, 697 | "show": true, 698 | "total": false, 699 | "values": false 700 | }, 701 | "lines": true, 702 | "linewidth": 1, 703 | "nullPointMode": "null", 704 | "options": { 705 | "dataLinks": [] 706 | }, 707 | "percentage": false, 708 | "pointradius": 2, 709 | "points": false, 710 | "renderer": "flot", 711 | "seriesOverrides": [], 712 | "spaceLength": 10, 713 | "stack": false, 714 | "steppedLine": false, 715 | "targets": [ 716 | { 717 | "expr": "sum(rate(kafka_topic_partition_current_offset{topic=\"user-create\"}[1m]))", 718 | "hide": false, 719 | "refId": "A" 720 | } 721 | ], 722 | "thresholds": [], 723 | "timeFrom": null, 724 | "timeRegions": [], 725 | "timeShift": null, 726 | "title": "New create-user Topics / Second", 727 | "tooltip": { 728 | "shared": true, 729 | "sort": 0, 730 | "value_type": "individual" 731 | }, 732 | "type": "graph", 733 | "xaxis": { 734 | "buckets": null, 735 | "mode": "time", 736 | "name": null, 737 | "show": true, 738 | "values": [] 739 | }, 740 | "yaxes": [ 741 | { 742 | "format": "short", 743 | "label": null, 744 | "logBase": 1, 745 | "max": null, 746 | "min": null, 747 | "show": true 748 | }, 749 | { 750 | "format": "short", 751 | "label": null, 752 | "logBase": 1, 753 | "max": null, 754 | "min": null, 755 | "show": true 756 | } 757 | ], 758 | "yaxis": { 759 | "align": false, 760 | "alignLevel": null 761 | } 762 | }, 763 | { 764 | "aliasColors": { 765 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create\",consumergroup=\"user-service\"}[1m]))": "red", 766 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create\"}[1m]))": "red" 767 | }, 768 | "bars": false, 769 | "dashLength": 10, 770 | "dashes": false, 771 | "datasource": "${DS_PROMETHEUS}", 772 | "fill": 1, 773 | "fillGradient": 0, 774 | "gridPos": { 775 | "h": 8, 776 | "w": 6, 777 | "x": 0, 778 | "y": 24 779 | }, 780 | "id": 22, 781 | "legend": { 782 | "avg": false, 783 | "current": false, 784 | "max": false, 785 | "min": false, 786 | "show": true, 787 | "total": false, 788 | "values": false 789 | }, 790 | "lines": true, 791 | "linewidth": 1, 792 | "nullPointMode": "null", 793 | "options": { 794 | "dataLinks": [] 795 | }, 796 | "percentage": false, 797 | "pointradius": 2, 798 | "points": false, 799 | "renderer": "flot", 800 | "seriesOverrides": [], 801 | "spaceLength": 10, 802 | "stack": false, 803 | "steppedLine": false, 804 | "targets": [ 805 | { 806 | "expr": "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create\",consumergroup=\"user-service\"}[1m]))", 807 | "hide": false, 808 | "refId": "A" 809 | } 810 | ], 811 | "thresholds": [], 812 | "timeFrom": null, 813 | "timeRegions": [], 814 | "timeShift": null, 815 | "title": "Topic user-create lag", 816 | "tooltip": { 817 | "shared": true, 818 | "sort": 0, 819 | "value_type": "individual" 820 | }, 821 | "type": "graph", 822 | "xaxis": { 823 | "buckets": null, 824 | "mode": "time", 825 | "name": null, 826 | "show": true, 827 | "values": [] 828 | }, 829 | "yaxes": [ 830 | { 831 | "format": "none", 832 | "label": null, 833 | "logBase": 1, 834 | "max": null, 835 | "min": null, 836 | "show": true 837 | }, 838 | { 839 | "format": "short", 840 | "label": null, 841 | "logBase": 1, 842 | "max": null, 843 | "min": null, 844 | "show": true 845 | } 846 | ], 847 | "yaxis": { 848 | "align": false, 849 | "alignLevel": null 850 | } 851 | }, 852 | { 853 | "aliasColors": { 854 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\",consumergroup=\"user-approval-service\"}[1m]))": "red", 855 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\"}[1m]))": "red" 856 | }, 857 | "bars": false, 858 | "dashLength": 10, 859 | "dashes": false, 860 | "datasource": "${DS_PROMETHEUS}", 861 | "fill": 1, 862 | "fillGradient": 0, 863 | "gridPos": { 864 | "h": 8, 865 | "w": 6, 866 | "x": 6, 867 | "y": 24 868 | }, 869 | "id": 25, 870 | "legend": { 871 | "avg": false, 872 | "current": false, 873 | "max": false, 874 | "min": false, 875 | "show": true, 876 | "total": false, 877 | "values": false 878 | }, 879 | "lines": true, 880 | "linewidth": 1, 881 | "nullPointMode": "null", 882 | "options": { 883 | "dataLinks": [] 884 | }, 885 | "percentage": false, 886 | "pointradius": 2, 887 | "points": false, 888 | "renderer": "flot", 889 | "seriesOverrides": [], 890 | "spaceLength": 10, 891 | "stack": false, 892 | "steppedLine": false, 893 | "targets": [ 894 | { 895 | "expr": "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve\",consumergroup=\"user-approval-service\"}[1m]))", 896 | "hide": false, 897 | "refId": "A" 898 | }, 899 | { 900 | "expr": "kafka_consumergroup_lag", 901 | "hide": true, 902 | "refId": "B" 903 | } 904 | ], 905 | "thresholds": [], 906 | "timeFrom": null, 907 | "timeRegions": [], 908 | "timeShift": null, 909 | "title": "Topic user-approve lag", 910 | "tooltip": { 911 | "shared": true, 912 | "sort": 0, 913 | "value_type": "individual" 914 | }, 915 | "type": "graph", 916 | "xaxis": { 917 | "buckets": null, 918 | "mode": "time", 919 | "name": null, 920 | "show": true, 921 | "values": [] 922 | }, 923 | "yaxes": [ 924 | { 925 | "format": "none", 926 | "label": null, 927 | "logBase": 1, 928 | "max": null, 929 | "min": null, 930 | "show": true 931 | }, 932 | { 933 | "format": "short", 934 | "label": null, 935 | "logBase": 1, 936 | "max": null, 937 | "min": null, 938 | "show": true 939 | } 940 | ], 941 | "yaxis": { 942 | "align": false, 943 | "alignLevel": null 944 | } 945 | }, 946 | { 947 | "aliasColors": { 948 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve-response\",consumergroup=\"user-service\"}[1m]))": "red", 949 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve-response\"}[1m]))": "red" 950 | }, 951 | "bars": false, 952 | "dashLength": 10, 953 | "dashes": false, 954 | "datasource": "${DS_PROMETHEUS}", 955 | "fill": 1, 956 | "fillGradient": 0, 957 | "gridPos": { 958 | "h": 8, 959 | "w": 6, 960 | "x": 12, 961 | "y": 24 962 | }, 963 | "id": 24, 964 | "legend": { 965 | "avg": false, 966 | "current": false, 967 | "max": false, 968 | "min": false, 969 | "show": true, 970 | "total": false, 971 | "values": false 972 | }, 973 | "lines": true, 974 | "linewidth": 1, 975 | "nullPointMode": "null", 976 | "options": { 977 | "dataLinks": [] 978 | }, 979 | "percentage": false, 980 | "pointradius": 2, 981 | "points": false, 982 | "renderer": "flot", 983 | "seriesOverrides": [], 984 | "spaceLength": 10, 985 | "stack": false, 986 | "steppedLine": false, 987 | "targets": [ 988 | { 989 | "expr": "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-approve-response\",consumergroup=\"user-service\"}[1m]))", 990 | "refId": "A" 991 | } 992 | ], 993 | "thresholds": [], 994 | "timeFrom": null, 995 | "timeRegions": [], 996 | "timeShift": null, 997 | "title": "Topic user-approve-response lag", 998 | "tooltip": { 999 | "shared": true, 1000 | "sort": 0, 1001 | "value_type": "individual" 1002 | }, 1003 | "type": "graph", 1004 | "xaxis": { 1005 | "buckets": null, 1006 | "mode": "time", 1007 | "name": null, 1008 | "show": true, 1009 | "values": [] 1010 | }, 1011 | "yaxes": [ 1012 | { 1013 | "format": "none", 1014 | "label": null, 1015 | "logBase": 1, 1016 | "max": null, 1017 | "min": null, 1018 | "show": true 1019 | }, 1020 | { 1021 | "format": "short", 1022 | "label": null, 1023 | "logBase": 1, 1024 | "max": null, 1025 | "min": null, 1026 | "show": true 1027 | } 1028 | ], 1029 | "yaxis": { 1030 | "align": false, 1031 | "alignLevel": null 1032 | } 1033 | }, 1034 | { 1035 | "aliasColors": { 1036 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create-response\",consumergroup=\"operation-service\"}[1m]))": "red", 1037 | "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create-response\"}[1m]))": "red" 1038 | }, 1039 | "bars": false, 1040 | "dashLength": 10, 1041 | "dashes": false, 1042 | "datasource": "${DS_PROMETHEUS}", 1043 | "fill": 1, 1044 | "fillGradient": 0, 1045 | "gridPos": { 1046 | "h": 8, 1047 | "w": 6, 1048 | "x": 18, 1049 | "y": 24 1050 | }, 1051 | "id": 23, 1052 | "legend": { 1053 | "avg": false, 1054 | "current": false, 1055 | "max": false, 1056 | "min": false, 1057 | "show": true, 1058 | "total": false, 1059 | "values": false 1060 | }, 1061 | "lines": true, 1062 | "linewidth": 1, 1063 | "nullPointMode": "null", 1064 | "options": { 1065 | "dataLinks": [] 1066 | }, 1067 | "percentage": false, 1068 | "pointradius": 2, 1069 | "points": false, 1070 | "renderer": "flot", 1071 | "seriesOverrides": [], 1072 | "spaceLength": 10, 1073 | "stack": false, 1074 | "steppedLine": false, 1075 | "targets": [ 1076 | { 1077 | "expr": "sum(avg_over_time(kafka_consumergroup_lag{topic=\"user-create-response\",consumergroup=\"operation-service\"}[1m]))", 1078 | "refId": "A" 1079 | } 1080 | ], 1081 | "thresholds": [], 1082 | "timeFrom": null, 1083 | "timeRegions": [], 1084 | "timeShift": null, 1085 | "title": "Topic user-create-response lag", 1086 | "tooltip": { 1087 | "shared": true, 1088 | "sort": 0, 1089 | "value_type": "individual" 1090 | }, 1091 | "type": "graph", 1092 | "xaxis": { 1093 | "buckets": null, 1094 | "mode": "time", 1095 | "name": null, 1096 | "show": true, 1097 | "values": [] 1098 | }, 1099 | "yaxes": [ 1100 | { 1101 | "format": "none", 1102 | "label": null, 1103 | "logBase": 1, 1104 | "max": null, 1105 | "min": null, 1106 | "show": true 1107 | }, 1108 | { 1109 | "format": "short", 1110 | "label": null, 1111 | "logBase": 1, 1112 | "max": null, 1113 | "min": null, 1114 | "show": true 1115 | } 1116 | ], 1117 | "yaxis": { 1118 | "align": false, 1119 | "alignLevel": null 1120 | } 1121 | }, 1122 | { 1123 | "aliasColors": { 1124 | "sum(kafka_topic_partition_current_offset{topic=\"user-create\"})": "blue" 1125 | }, 1126 | "bars": false, 1127 | "dashLength": 10, 1128 | "dashes": false, 1129 | "datasource": "${DS_PROMETHEUS}", 1130 | "fill": 1, 1131 | "fillGradient": 0, 1132 | "gridPos": { 1133 | "h": 8, 1134 | "w": 6, 1135 | "x": 0, 1136 | "y": 32 1137 | }, 1138 | "id": 4, 1139 | "legend": { 1140 | "avg": false, 1141 | "current": false, 1142 | "max": false, 1143 | "min": false, 1144 | "show": true, 1145 | "total": false, 1146 | "values": false 1147 | }, 1148 | "lines": true, 1149 | "linewidth": 1, 1150 | "nullPointMode": "null", 1151 | "options": { 1152 | "dataLinks": [] 1153 | }, 1154 | "percentage": false, 1155 | "pointradius": 2, 1156 | "points": false, 1157 | "renderer": "flot", 1158 | "seriesOverrides": [], 1159 | "spaceLength": 10, 1160 | "stack": false, 1161 | "steppedLine": false, 1162 | "targets": [ 1163 | { 1164 | "expr": "sum(kafka_topic_partition_current_offset{topic=\"user-create\"})", 1165 | "refId": "A" 1166 | } 1167 | ], 1168 | "thresholds": [], 1169 | "timeFrom": null, 1170 | "timeRegions": [], 1171 | "timeShift": null, 1172 | "title": "Topic user-create", 1173 | "tooltip": { 1174 | "shared": true, 1175 | "sort": 0, 1176 | "value_type": "individual" 1177 | }, 1178 | "type": "graph", 1179 | "xaxis": { 1180 | "buckets": null, 1181 | "mode": "time", 1182 | "name": null, 1183 | "show": true, 1184 | "values": [] 1185 | }, 1186 | "yaxes": [ 1187 | { 1188 | "format": "none", 1189 | "label": null, 1190 | "logBase": 1, 1191 | "max": null, 1192 | "min": null, 1193 | "show": true 1194 | }, 1195 | { 1196 | "format": "short", 1197 | "label": null, 1198 | "logBase": 1, 1199 | "max": null, 1200 | "min": null, 1201 | "show": true 1202 | } 1203 | ], 1204 | "yaxis": { 1205 | "align": false, 1206 | "alignLevel": null 1207 | } 1208 | }, 1209 | { 1210 | "aliasColors": { 1211 | "sum(kafka_topic_partition_current_offset{topic=\"user-approve\"})": "blue" 1212 | }, 1213 | "bars": false, 1214 | "dashLength": 10, 1215 | "dashes": false, 1216 | "datasource": "${DS_PROMETHEUS}", 1217 | "fill": 1, 1218 | "fillGradient": 0, 1219 | "gridPos": { 1220 | "h": 8, 1221 | "w": 6, 1222 | "x": 6, 1223 | "y": 32 1224 | }, 1225 | "id": 8, 1226 | "legend": { 1227 | "avg": false, 1228 | "current": false, 1229 | "max": false, 1230 | "min": false, 1231 | "show": true, 1232 | "total": false, 1233 | "values": false 1234 | }, 1235 | "lines": true, 1236 | "linewidth": 1, 1237 | "nullPointMode": "null", 1238 | "options": { 1239 | "dataLinks": [] 1240 | }, 1241 | "percentage": false, 1242 | "pointradius": 2, 1243 | "points": false, 1244 | "renderer": "flot", 1245 | "seriesOverrides": [], 1246 | "spaceLength": 10, 1247 | "stack": false, 1248 | "steppedLine": false, 1249 | "targets": [ 1250 | { 1251 | "expr": "sum(kafka_topic_partition_current_offset{topic=\"user-approve\"})", 1252 | "refId": "A" 1253 | } 1254 | ], 1255 | "thresholds": [], 1256 | "timeFrom": null, 1257 | "timeRegions": [], 1258 | "timeShift": null, 1259 | "title": "Topic user-approve", 1260 | "tooltip": { 1261 | "shared": true, 1262 | "sort": 0, 1263 | "value_type": "individual" 1264 | }, 1265 | "type": "graph", 1266 | "xaxis": { 1267 | "buckets": null, 1268 | "mode": "time", 1269 | "name": null, 1270 | "show": true, 1271 | "values": [] 1272 | }, 1273 | "yaxes": [ 1274 | { 1275 | "format": "none", 1276 | "label": null, 1277 | "logBase": 1, 1278 | "max": null, 1279 | "min": null, 1280 | "show": true 1281 | }, 1282 | { 1283 | "format": "short", 1284 | "label": null, 1285 | "logBase": 1, 1286 | "max": null, 1287 | "min": null, 1288 | "show": true 1289 | } 1290 | ], 1291 | "yaxis": { 1292 | "align": false, 1293 | "alignLevel": null 1294 | } 1295 | }, 1296 | { 1297 | "aliasColors": { 1298 | "sum(kafka_topic_partition_current_offset{topic=\"user-approve-response\"})": "blue" 1299 | }, 1300 | "bars": false, 1301 | "dashLength": 10, 1302 | "dashes": false, 1303 | "datasource": "${DS_PROMETHEUS}", 1304 | "fill": 1, 1305 | "fillGradient": 0, 1306 | "gridPos": { 1307 | "h": 8, 1308 | "w": 6, 1309 | "x": 12, 1310 | "y": 32 1311 | }, 1312 | "id": 10, 1313 | "legend": { 1314 | "avg": false, 1315 | "current": false, 1316 | "max": false, 1317 | "min": false, 1318 | "show": true, 1319 | "total": false, 1320 | "values": false 1321 | }, 1322 | "lines": true, 1323 | "linewidth": 1, 1324 | "nullPointMode": "null", 1325 | "options": { 1326 | "dataLinks": [] 1327 | }, 1328 | "percentage": false, 1329 | "pointradius": 2, 1330 | "points": false, 1331 | "renderer": "flot", 1332 | "seriesOverrides": [], 1333 | "spaceLength": 10, 1334 | "stack": false, 1335 | "steppedLine": false, 1336 | "targets": [ 1337 | { 1338 | "expr": "sum(kafka_topic_partition_current_offset{topic=\"user-approve-response\"})", 1339 | "refId": "A" 1340 | } 1341 | ], 1342 | "thresholds": [], 1343 | "timeFrom": null, 1344 | "timeRegions": [], 1345 | "timeShift": null, 1346 | "title": "Topic user-approve-response", 1347 | "tooltip": { 1348 | "shared": true, 1349 | "sort": 0, 1350 | "value_type": "individual" 1351 | }, 1352 | "type": "graph", 1353 | "xaxis": { 1354 | "buckets": null, 1355 | "mode": "time", 1356 | "name": null, 1357 | "show": true, 1358 | "values": [] 1359 | }, 1360 | "yaxes": [ 1361 | { 1362 | "format": "none", 1363 | "label": null, 1364 | "logBase": 1, 1365 | "max": null, 1366 | "min": null, 1367 | "show": true 1368 | }, 1369 | { 1370 | "format": "short", 1371 | "label": null, 1372 | "logBase": 1, 1373 | "max": null, 1374 | "min": null, 1375 | "show": true 1376 | } 1377 | ], 1378 | "yaxis": { 1379 | "align": false, 1380 | "alignLevel": null 1381 | } 1382 | }, 1383 | { 1384 | "aliasColors": { 1385 | "sum(kafka_topic_partition_current_offset{topic=\"user-create-response\"})": "blue" 1386 | }, 1387 | "bars": false, 1388 | "dashLength": 10, 1389 | "dashes": false, 1390 | "datasource": "${DS_PROMETHEUS}", 1391 | "fill": 1, 1392 | "fillGradient": 0, 1393 | "gridPos": { 1394 | "h": 8, 1395 | "w": 6, 1396 | "x": 18, 1397 | "y": 32 1398 | }, 1399 | "id": 6, 1400 | "legend": { 1401 | "avg": false, 1402 | "current": false, 1403 | "max": false, 1404 | "min": false, 1405 | "show": true, 1406 | "total": false, 1407 | "values": false 1408 | }, 1409 | "lines": true, 1410 | "linewidth": 1, 1411 | "nullPointMode": "null", 1412 | "options": { 1413 | "dataLinks": [] 1414 | }, 1415 | "percentage": false, 1416 | "pointradius": 2, 1417 | "points": false, 1418 | "renderer": "flot", 1419 | "seriesOverrides": [], 1420 | "spaceLength": 10, 1421 | "stack": false, 1422 | "steppedLine": false, 1423 | "targets": [ 1424 | { 1425 | "expr": "sum(kafka_topic_partition_current_offset{topic=\"user-create-response\"})", 1426 | "refId": "A" 1427 | } 1428 | ], 1429 | "thresholds": [], 1430 | "timeFrom": null, 1431 | "timeRegions": [], 1432 | "timeShift": null, 1433 | "title": "Topic user-create-response", 1434 | "tooltip": { 1435 | "shared": true, 1436 | "sort": 0, 1437 | "value_type": "individual" 1438 | }, 1439 | "type": "graph", 1440 | "xaxis": { 1441 | "buckets": null, 1442 | "mode": "time", 1443 | "name": null, 1444 | "show": true, 1445 | "values": [] 1446 | }, 1447 | "yaxes": [ 1448 | { 1449 | "format": "none", 1450 | "label": null, 1451 | "logBase": 1, 1452 | "max": null, 1453 | "min": null, 1454 | "show": true 1455 | }, 1456 | { 1457 | "format": "short", 1458 | "label": null, 1459 | "logBase": 1, 1460 | "max": null, 1461 | "min": null, 1462 | "show": true 1463 | } 1464 | ], 1465 | "yaxis": { 1466 | "align": false, 1467 | "alignLevel": null 1468 | } 1469 | }, 1470 | { 1471 | "aliasColors": {}, 1472 | "bars": false, 1473 | "dashLength": 10, 1474 | "dashes": false, 1475 | "datasource": "${DS_PROMETHEUS}", 1476 | "fill": 1, 1477 | "fillGradient": 0, 1478 | "gridPos": { 1479 | "h": 9, 1480 | "w": 24, 1481 | "x": 0, 1482 | "y": 40 1483 | }, 1484 | "id": 18, 1485 | "legend": { 1486 | "avg": false, 1487 | "current": false, 1488 | "max": false, 1489 | "min": false, 1490 | "show": true, 1491 | "total": false, 1492 | "values": false 1493 | }, 1494 | "lines": true, 1495 | "linewidth": 1, 1496 | "nullPointMode": "null", 1497 | "options": { 1498 | "dataLinks": [] 1499 | }, 1500 | "percentage": false, 1501 | "pointradius": 2, 1502 | "points": false, 1503 | "renderer": "flot", 1504 | "seriesOverrides": [], 1505 | "spaceLength": 10, 1506 | "stack": false, 1507 | "steppedLine": false, 1508 | "targets": [ 1509 | { 1510 | "expr": "sum(kafka_topic_partition_current_offset) by (topic)", 1511 | "refId": "A" 1512 | } 1513 | ], 1514 | "thresholds": [], 1515 | "timeFrom": null, 1516 | "timeRegions": [], 1517 | "timeShift": null, 1518 | "title": "Kafka Topics", 1519 | "tooltip": { 1520 | "shared": true, 1521 | "sort": 0, 1522 | "value_type": "individual" 1523 | }, 1524 | "type": "graph", 1525 | "xaxis": { 1526 | "buckets": null, 1527 | "mode": "time", 1528 | "name": null, 1529 | "show": true, 1530 | "values": [] 1531 | }, 1532 | "yaxes": [ 1533 | { 1534 | "format": "none", 1535 | "label": null, 1536 | "logBase": 1, 1537 | "max": null, 1538 | "min": null, 1539 | "show": true 1540 | }, 1541 | { 1542 | "format": "short", 1543 | "label": null, 1544 | "logBase": 1, 1545 | "max": null, 1546 | "min": null, 1547 | "show": true 1548 | } 1549 | ], 1550 | "yaxis": { 1551 | "align": false, 1552 | "alignLevel": null 1553 | } 1554 | } 1555 | ], 1556 | "refresh": "5s", 1557 | "schemaVersion": 20, 1558 | "style": "dark", 1559 | "tags": [], 1560 | "templating": { 1561 | "list": [] 1562 | }, 1563 | "time": { 1564 | "from": "now-15m", 1565 | "to": "now" 1566 | }, 1567 | "timepicker": { 1568 | "refresh_intervals": [ 1569 | "5s", 1570 | "10s", 1571 | "30s", 1572 | "1m", 1573 | "5m", 1574 | "15m", 1575 | "30m", 1576 | "1h", 1577 | "2h", 1578 | "1d" 1579 | ] 1580 | }, 1581 | "timezone": "", 1582 | "title": "Kafka Dashboard", 1583 | "uid": "vbXzw3QZk", 1584 | "version": 30 1585 | } --------------------------------------------------------------------------------