├── kustomize ├── env │ ├── team-a-namespace │ │ ├── namespace.yaml │ │ ├── operator-group.yaml │ │ ├── kustomization.yaml │ │ ├── networkpolicy.yaml │ │ └── access.yaml │ ├── team-b-namespace │ │ ├── namespace.yaml │ │ ├── operator-group.yaml │ │ ├── kustomization.yaml │ │ └── access.yaml │ ├── grafana-monitoring-operator │ │ ├── namespace.yaml │ │ ├── operator-group.yaml │ │ ├── kustomization.yaml │ │ └── grafana-cluster-monitoring-view.yaml │ ├── team-a-alerts │ │ ├── smtp-auth.yaml │ │ ├── kustomization.yaml │ │ └── alertmanagerconfig.yaml │ ├── team-b-alerts │ │ ├── kustomization.yaml │ │ └── alertmanagerconfig.yaml │ ├── openshift-monitoring │ │ ├── kustomization.yaml │ │ ├── cluster-monitoring-config.yaml │ │ └── user-workload-monitoring-config.yaml │ ├── team-a-operators │ │ ├── kustomization.yaml │ │ └── grafana-rolebinding.yaml │ ├── team-b-operators │ │ ├── kustomization.yaml │ │ └── grafana-rolebinding.yaml │ ├── openshift-monitoring-alerts │ │ ├── kustomization.yaml │ │ ├── cluster-monitoring-config.yaml │ │ └── user-workload-monitoring-config.yaml │ ├── grafana-monitoring │ │ └── kustomization.yaml │ ├── team-b-workload │ │ └── kustomization.yaml │ └── team-a-workload │ │ └── kustomization.yaml └── base │ ├── amq │ ├── operator │ │ ├── kustomization.yaml │ │ └── operator.yaml │ └── instance │ │ ├── kustomization.yaml │ │ ├── servicemonitor.yaml │ │ ├── broker.yaml │ │ └── alerts.yaml │ ├── sso │ ├── operator │ │ ├── kustomization.yaml │ │ └── sso-operator.yaml │ └── instance │ │ ├── kustomization.yaml │ │ ├── users-secret.yaml │ │ └── sso-instance.yaml │ ├── http-https-echo │ ├── kustomization.yaml │ ├── service.yaml │ └── deployment.yaml │ └── grafana │ ├── operator │ ├── kustomization.yaml │ ├── serviceaccount.yaml │ └── subscription.yaml │ ├── instance │ ├── ocp-injected-certs.yaml │ ├── session-secret.yaml │ ├── kustomization.yaml │ ├── datasource.yaml │ ├── grafana-readonly.yaml │ ├── grafana-editor.yaml │ └── grafana-admin.yaml │ ├── dashboards-openshift │ ├── kustomization.yaml │ ├── README.adoc │ └── grafana-dashboards-openshift.yaml │ └── dashboards │ ├── kustomization.yaml │ ├── grafana-dashboards.yaml │ ├── dashboard-sso.json │ └── dashboard-artemis.json ├── uninstall.sh ├── install.sh └── README.adoc /kustomize/env/team-a-namespace/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: team-a -------------------------------------------------------------------------------- /kustomize/env/team-b-namespace/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: team-b -------------------------------------------------------------------------------- /kustomize/env/grafana-monitoring-operator/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: grafana-monitoring -------------------------------------------------------------------------------- /kustomize/base/amq/operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - operator.yaml -------------------------------------------------------------------------------- /kustomize/base/sso/operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - sso-operator.yaml -------------------------------------------------------------------------------- /kustomize/env/team-a-alerts/smtp-auth.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: smtp-auth 5 | type: Opaque 6 | stringData: 7 | password: ... -------------------------------------------------------------------------------- /kustomize/base/http-https-echo/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | -------------------------------------------------------------------------------- /kustomize/base/sso/instance/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - users-secret.yaml 6 | - sso-instance.yaml -------------------------------------------------------------------------------- /kustomize/base/amq/instance/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - broker.yaml 6 | - servicemonitor.yaml 7 | - alerts.yaml -------------------------------------------------------------------------------- /kustomize/base/grafana/operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - subscription.yaml 6 | - serviceaccount.yaml 7 | -------------------------------------------------------------------------------- /kustomize/env/team-a-namespace/operator-group.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1 2 | kind: OperatorGroup 3 | metadata: 4 | name: self 5 | spec: 6 | targetNamespaces: 7 | - team-a 8 | -------------------------------------------------------------------------------- /kustomize/env/team-b-namespace/operator-group.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1 2 | kind: OperatorGroup 3 | metadata: 4 | name: self 5 | spec: 6 | targetNamespaces: 7 | - team-b 8 | -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/ocp-injected-certs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | labels: 5 | config.openshift.io/inject-trusted-cabundle: "true" 6 | name: ocp-injected-certs -------------------------------------------------------------------------------- /kustomize/base/sso/instance/users-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: credential-sso 5 | type: Opaque 6 | stringData: 7 | ADMIN_PASSWORD: mysecret 8 | ADMIN_USERNAME: admin -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/session-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: grafana-k8s-proxy 5 | type: Opaque 6 | stringData: 7 | session_secret: f7d49449-18b9-4906-93f3-76890a65cdb4 -------------------------------------------------------------------------------- /kustomize/env/grafana-monitoring-operator/operator-group.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1 2 | kind: OperatorGroup 3 | metadata: 4 | name: self 5 | spec: 6 | targetNamespaces: 7 | - grafana-monitoring 8 | -------------------------------------------------------------------------------- /kustomize/env/team-b-alerts/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-b 5 | 6 | resources: 7 | - ../../base/http-https-echo 8 | - alertmanagerconfig.yaml 9 | -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - cluster-monitoring-config.yaml 6 | - user-workload-monitoring-config.yaml 7 | 8 | -------------------------------------------------------------------------------- /kustomize/env/team-a-namespace/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-a 5 | 6 | resources: 7 | - namespace.yaml 8 | - operator-group.yaml 9 | - access.yaml 10 | 11 | -------------------------------------------------------------------------------- /kustomize/env/team-b-namespace/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-b 5 | 6 | resources: 7 | - namespace.yaml 8 | - operator-group.yaml 9 | - access.yaml 10 | 11 | -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards-openshift/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - dashboards-ocp410.yaml 6 | # - dashboards-ocp411.yaml 7 | - grafana-dashboards-openshift.yaml 8 | -------------------------------------------------------------------------------- /kustomize/env/team-a-alerts/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-a 5 | 6 | resources: 7 | - ../../base/http-https-echo 8 | - smtp-auth.yaml 9 | - alertmanagerconfig.yaml 10 | -------------------------------------------------------------------------------- /kustomize/env/team-a-operators/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-a 5 | 6 | resources: 7 | - ../../base/grafana/operator 8 | - ../../base/sso/operator 9 | - ../../base/amq/operator 10 | - grafana-rolebinding.yaml 11 | -------------------------------------------------------------------------------- /kustomize/env/team-b-operators/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-b 5 | 6 | resources: 7 | - ../../base/grafana/operator 8 | - ../../base/sso/operator 9 | - ../../base/amq/operator 10 | - grafana-rolebinding.yaml 11 | -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - session-secret.yaml 6 | - ocp-injected-certs.yaml 7 | - grafana-admin.yaml 8 | # - grafana-editor.yaml 9 | # - grafana-readonly.yaml 10 | - datasource.yaml 11 | 12 | -------------------------------------------------------------------------------- /kustomize/env/grafana-monitoring-operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: grafana-monitoring 5 | 6 | resources: 7 | - namespace.yaml 8 | - operator-group.yaml 9 | - ../../base/grafana/operator 10 | - grafana-cluster-monitoring-view.yaml 11 | -------------------------------------------------------------------------------- /kustomize/env/team-a-operators/grafana-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: grafana-thanos 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: view 9 | subjects: 10 | - kind: ServiceAccount 11 | name: grafana-thanos -------------------------------------------------------------------------------- /kustomize/env/team-b-operators/grafana-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: grafana-thanos 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: view 9 | subjects: 10 | - kind: ServiceAccount 11 | name: grafana-thanos -------------------------------------------------------------------------------- /kustomize/base/amq/operator/operator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1alpha1 2 | kind: Subscription 3 | metadata: 4 | name: amq-broker-rhel8 5 | spec: 6 | channel: 7.10.x 7 | installPlanApproval: Automatic 8 | name: amq-broker-rhel8 9 | source: redhat-operators 10 | sourceNamespace: openshift-marketplace 11 | -------------------------------------------------------------------------------- /kustomize/base/http-https-echo/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: http-https-echo 5 | spec: 6 | selector: 7 | app: http-https-echo 8 | ports: 9 | - name: http 10 | port: 8080 11 | protocol: TCP 12 | targetPort: 8080 13 | - name: https 14 | port: 8443 15 | protocol: TCP 16 | targetPort: 8443 -------------------------------------------------------------------------------- /kustomize/env/grafana-monitoring-operator/grafana-cluster-monitoring-view.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: grafana-cluster-monitoring-view 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: cluster-monitoring-view 9 | subjects: 10 | - kind: ServiceAccount 11 | name: grafana-thanos -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - grafana-dashboards.yaml 6 | 7 | configMapGenerator: 8 | - name: grafana-dashboards 9 | files: 10 | - dashboard-artemis.json 11 | - dashboard-jmx.json 12 | - dashboard-sso.json 13 | 14 | generatorOptions: 15 | disableNameSuffixHash: true 16 | -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring-alerts/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | # OpenShift v4.10 uses the same Alertmanager as default alerts 6 | # - cluster-monitoring-config.yaml 7 | # In OpenShift v4.11 we should rather use a separate Alertmanager for user-defined alerts 8 | - user-workload-monitoring-config.yaml 9 | 10 | -------------------------------------------------------------------------------- /kustomize/env/team-a-namespace/networkpolicy.yaml: -------------------------------------------------------------------------------- 1 | # This is only needed if we already have other NetworkPolicy in the namespace, otherwise all incoming traffic is allowed by default 2 | apiVersion: networking.k8s.io/v1 3 | kind: NetworkPolicy 4 | metadata: 5 | name: allow-from-user-workload-monitoring 6 | spec: 7 | ingress: 8 | - from: 9 | - namespaceSelector: 10 | matchLabels: 11 | name: openshift-user-workload-monitoring 12 | podSelector: {} -------------------------------------------------------------------------------- /kustomize/base/amq/instance/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | name: broker 5 | spec: 6 | endpoints: 7 | - interval: 15s 8 | port: wconsj-.* 9 | # scheme: https 10 | # tlsConfig: 11 | # insecureSkipVerify: true 12 | podTargetLabels: 13 | - ActiveMQArtemis 14 | - application 15 | selector: 16 | matchLabels: 17 | ActiveMQArtemis: broker 18 | jobLabel: ActiveMQArtemis -------------------------------------------------------------------------------- /kustomize/base/grafana/operator/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # This serviceaccount will be used to access the Thanos endpoints 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: grafana-thanos 6 | --- 7 | # Generate a permanent SA token - not done by default OpenShift 4.11+ 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: grafana-thanos-token 12 | annotations: 13 | kubernetes.io/service-account.name: grafana-thanos 14 | type: kubernetes.io/service-account-token -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring-alerts/cluster-monitoring-config.yaml: -------------------------------------------------------------------------------- 1 | # In OpenShift 4.10 you can't use a separate Alertmanager for user-defined alerts, so we enable "alertmanagerMain" in this CM instead. 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: cluster-monitoring-config 6 | namespace: openshift-monitoring 7 | data: 8 | config.yaml: | 9 | enableUserWorkload: true 10 | prometheusK8s: 11 | alertmanagerMain: 12 | enableUserAlertmanagerConfig: true -------------------------------------------------------------------------------- /kustomize/base/grafana/operator/subscription.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1alpha1 2 | kind: Subscription 3 | metadata: 4 | name: grafana 5 | spec: 6 | channel: v4 7 | installPlanApproval: Automatic 8 | name: grafana-operator 9 | source: community-operators 10 | sourceNamespace: openshift-marketplace 11 | config: 12 | resources: 13 | limits: 14 | cpu: 200m 15 | memory: 300Mi 16 | requests: 17 | cpu: 100m 18 | memory: 150Mi 19 | 20 | -------------------------------------------------------------------------------- /kustomize/base/http-https-echo/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: http-https-echo 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: http-https-echo 10 | template: 11 | metadata: 12 | labels: 13 | app: http-https-echo 14 | spec: 15 | containers: 16 | - name: main 17 | image: mendhak/http-https-echo:28 18 | env: 19 | - name: ECHO_BACK_TO_CLIENT 20 | value: "false" 21 | -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring/cluster-monitoring-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: cluster-monitoring-config 5 | namespace: openshift-monitoring 6 | data: 7 | config.yaml: | 8 | enableUserWorkload: true 9 | # Parameters for Platform prometheus 10 | prometheusK8s: 11 | # retention: 15d 12 | # retentionSize: 90GB 13 | # volumeClaimTemplate: 14 | # spec: 15 | # resources: 16 | # requests: 17 | # storage: 100Gi -------------------------------------------------------------------------------- /uninstall.sh: -------------------------------------------------------------------------------- 1 | # Delete teams 2 | oc delete -k kustomize/env/team-a-workload 3 | oc delete -k kustomize/env/team-a-operators 4 | oc delete -k kustomize/env/team-b-workload 5 | oc delete -k kustomize/env/team-b-operators 6 | sleep 10 7 | oc delete -k kustomize/env/team-a-namespace 8 | oc delete -k kustomize/env/team-b-namespace 9 | 10 | # Delete grafana-monitoring 11 | oc delete -k kustomize/env/grafana-monitoring 12 | oc delete -k kustomize/env/grafana-monitoring-operator 13 | 14 | # Remove user-workload-monitoring 15 | oc delete -k kustomize/env/openshift-monitoring 16 | -------------------------------------------------------------------------------- /kustomize/base/sso/operator/sso-operator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: operators.coreos.com/v1alpha1 2 | kind: Subscription 3 | metadata: 4 | name: rhsso-operator 5 | spec: 6 | channel: stable 7 | installPlanApproval: Automatic 8 | name: rhsso-operator 9 | source: redhat-operators 10 | sourceNamespace: openshift-marketplace 11 | # startingCSV: rhsso-operator.7.5.2-opr-003 12 | # spec: 13 | # channel: alpha 14 | # installPlanApproval: Automatic 15 | # name: keycloak-operator 16 | # source: community-operators 17 | # sourceNamespace: openshift-marketplace 18 | 19 | 20 | -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring/user-workload-monitoring-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: user-workload-monitoring-config 5 | namespace: openshift-user-workload-monitoring 6 | data: 7 | config.yaml: | 8 | # Parameters for User prometheus 9 | prometheus: 10 | # retention: 1d 11 | # retentionSize: 4GB 12 | # resources: 13 | # requests: 14 | # cpu: 6m 15 | # memory: 30Mi 16 | # volumeClaimTemplate: 17 | # spec: 18 | # resources: 19 | # requests: 20 | # storage: 5Gi 21 | -------------------------------------------------------------------------------- /kustomize/env/grafana-monitoring/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: grafana-monitoring 5 | 6 | resources: 7 | - ../../base/grafana/instance 8 | - ../../base/grafana/dashboards 9 | # - ../../base/grafana/dashboards-openshift 10 | 11 | patches: 12 | - target: 13 | kind: GrafanaDataSource 14 | name: thanos 15 | # oc sa get-token grafana-thanos 16 | patch: |- 17 | - op: replace 18 | path: /spec/datasources/0/secureJsonData/httpHeaderValue1 19 | value: Bearer eyJhbGciO... 20 | - target: 21 | kind: Grafana 22 | name: grafana 23 | patch: |- 24 | - op: add 25 | path: /spec/containers/0/args/- 26 | value: '-openshift-sar={"resource": "namespaces", "verb": "get"}' 27 | 28 | -------------------------------------------------------------------------------- /kustomize/env/openshift-monitoring-alerts/user-workload-monitoring-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: user-workload-monitoring-config 5 | namespace: openshift-user-workload-monitoring 6 | data: 7 | config.yaml: | 8 | prometheus: 9 | thanosRuler: 10 | # retention: 24h 11 | # resources: 12 | # requests: 13 | # cpu: 1m 14 | # memory: 21Mi 15 | # volumeClaimTemplate: 16 | # spec: 17 | # resources: 18 | # requests: 19 | # storage: 3Gi 20 | # We use this separate Alertmanager 21 | # to avoid sending user-defined alerts 22 | # through the default notification channels 23 | alertmanager: 24 | enabled: true 25 | enableAlertmanagerConfig: true 26 | # resources: 27 | # requests: 28 | # cpu: 4m 29 | # memory: 40Mi 30 | # volumeClaimTemplate: 31 | # spec: 32 | # resources: 33 | # requests: 34 | # storage: 3Gi -------------------------------------------------------------------------------- /kustomize/base/amq/instance/broker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: broker.amq.io/v1beta1 2 | kind: ActiveMQArtemis 3 | metadata: 4 | name: broker 5 | spec: 6 | upgrades: 7 | enabled: true 8 | minor: true 9 | version: 7.10.0 10 | adminUser: admin 11 | adminPassword: admin 12 | acceptors: 13 | - name: main 14 | # expose: true 15 | port: 61617 16 | protocols: amqp,core 17 | # sslEnabled: true 18 | # sslSecret: amq-broker-ssl-acceptor 19 | suppressInternalManagementObjects: true 20 | console: 21 | expose: true 22 | # sslEnabled: true 23 | # sslSecret: amq-broker-ssl-console 24 | deploymentPlan: 25 | image: placeholder 26 | persistenceEnabled: true 27 | enableMetricsPlugin: true 28 | requireLogin: true 29 | livenessProbe: 30 | initialDelaySeconds: 240 31 | size: 2 32 | resources: 33 | requests: 34 | cpu: 50m 35 | memory: 512Mi 36 | limits: 37 | cpu: "1" 38 | memory: 2Gi 39 | brokerProperties: 40 | - globalMaxSize=100m -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards-openshift/README.adoc: -------------------------------------------------------------------------------- 1 | = OpenShift dashboards 2 | 3 | These Grafana dashboards were copied from source of https://github.com/openshift/cluster-monitoring-operator/blob/release-4.10/assets/grafana/dashboard-definitions.yaml[cluster-monitoring-operator]. 4 | 5 | * Namespace, labels and annotations were removed from the ConfigMaps as they are not relevant outside the context of an OpenShift distribution. 6 | * `dashboards-ocp410.yaml` is for OpenShift v4.10, while `dashboards-ocp410.yaml` is for OpenShift v4.11 7 | ** The only one line difference is using `cluster:node_cpu:ratio_rate5m` in _grafana-dashboard-k8s-resources-cluster_ 8 | * These dashboards are no longer maintained after OpenShift v4.11, so additional work is needed to make them work on newer versions 9 | 10 | The dashboards work fine deployed to the cluster scoped Grafana instance to visualize standard OpenShift "infra" metrics 11 | 12 | * Some dashboards also work in namespace scoped Grafana. 13 | ** Remove existing `namespace=~` filters to avoid "Bad Request". 14 | -------------------------------------------------------------------------------- /kustomize/env/team-b-workload/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-b 5 | 6 | resources: 7 | - ../../base/amq/instance 8 | - ../../base/sso/instance 9 | - ../../base/grafana/instance 10 | - ../../base/grafana/dashboards 11 | 12 | patches: 13 | - target: 14 | kind: GrafanaDataSource 15 | name: thanos 16 | # oc sa get-token grafana-serviceaccount 17 | patch: |- 18 | - op: replace 19 | path: /spec/datasources/0/url 20 | value: https://thanos-querier.openshift-monitoring.svc.cluster.local:9092 21 | - op: add 22 | path: /spec/datasources/0/jsonData/customQueryParameters 23 | value: namespace=team-b 24 | - op: replace 25 | path: /spec/datasources/0/secureJsonData/httpHeaderValue1 26 | value: Bearer eyJhbGc... 27 | - target: 28 | kind: Grafana 29 | name: grafana 30 | patch: |- 31 | - op: add 32 | path: /spec/containers/0/args/- 33 | value: '-openshift-sar={"resource": "services", "verb": "get", "namespace":"team-b"}' -------------------------------------------------------------------------------- /kustomize/env/team-a-workload/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: team-a 5 | 6 | resources: 7 | - ../../base/amq/instance 8 | - ../../base/sso/instance 9 | - ../../base/grafana/instance 10 | - ../../base/grafana/dashboards 11 | 12 | patches: 13 | - target: 14 | kind: GrafanaDataSource 15 | name: thanos 16 | # oc sa get-token grafana-serviceaccount 17 | patch: |- 18 | - op: replace 19 | path: /spec/datasources/0/url 20 | value: https://thanos-querier.openshift-monitoring.svc.cluster.local:9092 21 | - op: add 22 | path: /spec/datasources/0/jsonData/customQueryParameters 23 | value: namespace=team-a 24 | - op: replace 25 | path: /spec/datasources/0/secureJsonData/httpHeaderValue1 26 | value: Bearer eyJhbGci... 27 | - target: 28 | kind: Grafana 29 | name: grafana 30 | patch: |- 31 | - op: add 32 | path: /spec/containers/0/args/- 33 | value: '-openshift-sar={"resource": "services", "verb": "get", "namespace":"team-a"}' 34 | -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/datasource.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: integreatly.org/v1alpha1 2 | kind: GrafanaDataSource 3 | metadata: 4 | labels: 5 | app: grafana 6 | name: thanos 7 | spec: 8 | datasources: 9 | - access: proxy 10 | editable: true 11 | isDefault: false 12 | jsonData: 13 | # customQueryParameters: namespace=mynamespace 14 | httpHeaderName1: Authorization 15 | # timeInterval is required for queries with $__rate_interval, it should be set to the usual scrape time in Prometheus 16 | timeInterval: 15s 17 | tlsSkipVerify: true 18 | name: Prometheus 19 | secureJsonData: 20 | httpHeaderValue1: >- 21 | Bearer [use grafana-thanos token] 22 | type: prometheus 23 | version: 1 24 | # Cluster scoped 25 | url: 'https://thanos-querier.openshift-monitoring.svc.cluster.local:9091' 26 | # Namespaced scoped - requires 'customQueryParameters: namespace=mynamespace' above 27 | # url: 'https://thanos-querier.openshift-monitoring.svc.cluster.local:9092' 28 | name: grafana-datasource.yaml -------------------------------------------------------------------------------- /kustomize/base/amq/instance/alerts.yaml: -------------------------------------------------------------------------------- 1 | # Alert - Can use Platform and User metrics too 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: PrometheusRule 4 | metadata: 5 | name: amq 6 | spec: 7 | groups: 8 | - name: general.rules 9 | # interval: 15s 10 | rules: 11 | - alert: ArtemisMessageCountSum 12 | expr: sum by (pod) (artemis_message_count) > 10 13 | for: 1m 14 | labels: 15 | # namespace is automatically added 16 | severity: warning 17 | app: amq 18 | annotations: 19 | summary: Total message count is high 20 | description: Broker {{ $labels.pod }} - Total message count is high 21 | message: Total message count is {{ $value }} 22 | - alert: ArtemisMessageCountExpiryOrDLQ 23 | expr: artemis_message_count{queue=~'ExpiryQueue|DLQ'} > 0 24 | labels: 25 | severity: critical 26 | app: amq 27 | annotations: 28 | summary: Messages on {{ $labels.queue }} queue 29 | description: Broker {{ $labels.namespace }}/{{ $labels.pod }} - Messages on {{ $labels.queue }} 30 | message: Messages count on {{ $labels.queue }} is {{ $value }} 31 | 32 | -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards/grafana-dashboards.yaml: -------------------------------------------------------------------------------- 1 | # based on https://grafana.com/grafana/dashboards/9087 2 | apiVersion: integreatly.org/v1alpha1 3 | kind: GrafanaDashboard 4 | metadata: 5 | name: artemis 6 | labels: 7 | app: grafana 8 | monitoring-key: middleware 9 | spec: 10 | datasources: 11 | - datasourceName: Prometheus 12 | inputName: DS_PROMETHEUS 13 | configMapRef: 14 | name: grafana-dashboards 15 | key: dashboard-artemis.json 16 | --- 17 | # based on https://grafana.com/grafana/dashboards/7727 18 | apiVersion: integreatly.org/v1alpha1 19 | kind: GrafanaDashboard 20 | metadata: 21 | name: jmx 22 | labels: 23 | app: grafana 24 | monitoring-key: middleware 25 | spec: 26 | datasources: 27 | - datasourceName: Prometheus 28 | inputName: DS_PROMETHEUS 29 | configMapRef: 30 | name: grafana-dashboards 31 | key: dashboard-jmx.json 32 | --- 33 | apiVersion: integreatly.org/v1alpha1 34 | kind: GrafanaDashboard 35 | metadata: 36 | # The SSO operator creates a GrafanaDashboard "keycloak", so we need to use a different name 37 | name: sso 38 | labels: 39 | app: grafana 40 | monitoring-key: middleware 41 | spec: 42 | datasources: 43 | - datasourceName: Prometheus 44 | inputName: DS_PROMETHEUS 45 | configMapRef: 46 | name: grafana-dashboards 47 | key: dashboard-sso.json -------------------------------------------------------------------------------- /kustomize/env/team-a-alerts/alertmanagerconfig.yaml: -------------------------------------------------------------------------------- 1 | # How to send notifications 2 | # OpenShift 4.10 3 | apiVersion: monitoring.coreos.com/v1alpha1 4 | # OpenShift 4.11 5 | # apiVersion: monitoring.coreos.com/v1beta1 6 | kind: AlertmanagerConfig 7 | metadata: 8 | name: alert-notifications 9 | spec: 10 | route: 11 | receiver: default 12 | # Wait before sending first 13 | # groupWait: 30s 14 | groupWait: 5s 15 | # Min wait time between sending - if there were changes in alert status 16 | # groupInterval: 5m 17 | groupInterval: 30s 18 | # Wait time before repeating - if there were no changes in alert status 19 | # repeatInterval: 1h 20 | repeatInterval: 10m 21 | receivers: 22 | - name: default 23 | webhookConfigs: 24 | # Alertmanager is calling from the uer-workload-monitoring-namespace (or openshift-monitoring), so we need to use the full service domain name 25 | - url: http://http-https-echo.team-a.svc:8080/ 26 | # https://requestbin.com/r/enixp86lo2g2/2HFTeVekRSAlrJtkNKKN8r4o7Kp 27 | # - url: https://enixp86lo2g2.x.pipedream.net/ 28 | --- 29 | # Email alert 30 | apiVersion: monitoring.coreos.com/v1alpha1 31 | kind: AlertmanagerConfig 32 | metadata: 33 | name: alert-notifications-email 34 | spec: 35 | route: 36 | receiver: default 37 | groupWait: 5s 38 | groupInterval: 30s 39 | repeatInterval: 1h 40 | receivers: 41 | - name: default 42 | emailConfigs: 43 | - to: "balazs.szeti.test@gmail.com" 44 | from: "balazs.szeti.test@gmail.com" 45 | smarthost: "smtp.gmail.com:587" 46 | authUsername: "balazs.szeti.test@gmail.com" 47 | authPassword: 48 | name: smtp-auth 49 | key: password 50 | authIdentity: "balazs.szeti.test@gmail.com" 51 | -------------------------------------------------------------------------------- /kustomize/env/team-b-alerts/alertmanagerconfig.yaml: -------------------------------------------------------------------------------- 1 | # How to send notifications 2 | apiVersion: monitoring.coreos.com/v1alpha1 3 | kind: AlertmanagerConfig 4 | metadata: 5 | name: alert-notifications 6 | spec: 7 | route: 8 | # Default receiver is required - causes error in prometheus-operator. 9 | # It's also the receiver for sub-routes if it's not overwritten. 10 | receiver: default 11 | # Wait before sending first 12 | # groupWait: 30s 13 | groupWait: 5s 14 | # Min wait time between sending - if there were changes in alert status 15 | # groupInterval: 5m 16 | groupInterval: 30s 17 | # Wait time before repeating - if there were no changes in alert status 18 | # repeatInterval: 4h 19 | repeatInterval: 10m 20 | routes: 21 | # The default route/receiver is only used when no sub-route was matched - even if all had "continue:true" 22 | - receiver: sub1 23 | matchers: 24 | - name: severity 25 | matchType: '=' 26 | value: critical 27 | continue: true 28 | - receiver: sub2 29 | repeatInterval: 3m 30 | matchers: 31 | - name: app 32 | matchType: '=' 33 | value: amq 34 | continue: true 35 | receivers: 36 | - name: default 37 | webhookConfigs: 38 | # https://requestbin.com/r/enixp86lo2g2/ 39 | # - url: https://enixp86lo2g2.x.pipedream.net/ 40 | - url: http://http-https-echo.team-b.svc:8080/default 41 | - name: sub1 42 | webhookConfigs: 43 | # https://requestbin.com/r/enflqc06gmfu8 44 | # - url: https://enflqc06gmfu8.x.pipedream.net/ 45 | - url: http://http-https-echo.team-b.svc:8080/sub1 46 | - name: sub2 47 | webhookConfigs: 48 | # https://requestbin.com/r/enfiu0plafpkq 49 | # - url: https://enfiu0plafpkq.x.pipedream.net/ 50 | - url: http://http-https-echo.team-b.svc:8080/sub2 -------------------------------------------------------------------------------- /kustomize/env/team-a-namespace/access.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: users-access 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: admin 9 | subjects: 10 | - kind: User 11 | name: user1 12 | --- 13 | # This role is needed to create ServiceMonitor and PrometheusRule. Alternatively if Prometheus operator was installed via OperatorHub on the cluster anywhere, it's part of the default view/edit/admin ClusterRoles already. 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: monitoring-edit 18 | roleRef: 19 | apiGroup: rbac.authorization.k8s.io 20 | kind: ClusterRole 21 | name: monitoring-edit 22 | subjects: 23 | - kind: User 24 | name: user1 25 | --- 26 | # This is needed to create AlertmanagerConfig 27 | apiVersion: rbac.authorization.k8s.io/v1 28 | kind: RoleBinding 29 | metadata: 30 | name: alert-routing-edit 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: alert-routing-edit 35 | subjects: 36 | - kind: User 37 | name: user1 38 | --- 39 | apiVersion: rbac.authorization.k8s.io/v1 40 | kind: ClusterRoleBinding 41 | metadata: 42 | name: operatorhub-users-team-a 43 | roleRef: 44 | apiGroup: rbac.authorization.k8s.io 45 | kind: ClusterRole 46 | name: aggregate-olm-view 47 | subjects: 48 | - apiGroup: rbac.authorization.k8s.io 49 | kind: User 50 | name: user1 51 | --- 52 | apiVersion: rbac.authorization.k8s.io/v1 53 | kind: Role 54 | metadata: 55 | name: olm-installplan-approval 56 | rules: 57 | - apiGroups: 58 | - operators.coreos.com 59 | resources: 60 | - installplans 61 | verbs: 62 | - update 63 | - patch 64 | - list 65 | --- 66 | apiVersion: rbac.authorization.k8s.io/v1 67 | kind: RoleBinding 68 | metadata: 69 | name: olm-installplan-approval 70 | roleRef: 71 | apiGroup: rbac.authorization.k8s.io 72 | kind: Role 73 | name: olm-installplan-approval 74 | subjects: 75 | - apiGroup: rbac.authorization.k8s.io 76 | kind: User 77 | name: user1 -------------------------------------------------------------------------------- /kustomize/env/team-b-namespace/access.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: users-access 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: admin 9 | subjects: 10 | - kind: User 11 | name: user2 12 | --- 13 | # This role is needed to create ServiceMonitor and PrometheusRule. Alternatively if Prometheus operator was installed via OperatorHub on the cluster anywhere, it's part of the default view/edit/admin ClusterRoles already. 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: monitoring-edit 18 | roleRef: 19 | apiGroup: rbac.authorization.k8s.io 20 | kind: ClusterRole 21 | name: monitoring-edit 22 | subjects: 23 | - kind: User 24 | name: user2 25 | --- 26 | # This is needed to create AlertmanagerConfig 27 | apiVersion: rbac.authorization.k8s.io/v1 28 | kind: RoleBinding 29 | metadata: 30 | name: alert-routing-edit 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: alert-routing-edit 35 | subjects: 36 | - kind: User 37 | name: user2 38 | --- 39 | apiVersion: rbac.authorization.k8s.io/v1 40 | kind: ClusterRoleBinding 41 | metadata: 42 | name: operatorhub-users-team-b 43 | roleRef: 44 | apiGroup: rbac.authorization.k8s.io 45 | kind: ClusterRole 46 | name: aggregate-olm-view 47 | subjects: 48 | - apiGroup: rbac.authorization.k8s.io 49 | kind: User 50 | name: user2 51 | --- 52 | apiVersion: rbac.authorization.k8s.io/v1 53 | kind: Role 54 | metadata: 55 | name: olm-installplan-approval 56 | rules: 57 | - apiGroups: 58 | - operators.coreos.com 59 | resources: 60 | - installplans 61 | verbs: 62 | - update 63 | - patch 64 | - list 65 | --- 66 | apiVersion: rbac.authorization.k8s.io/v1 67 | kind: RoleBinding 68 | metadata: 69 | name: olm-installplan-approval 70 | roleRef: 71 | apiGroup: rbac.authorization.k8s.io 72 | kind: Role 73 | name: olm-installplan-approval 74 | subjects: 75 | - apiGroup: rbac.authorization.k8s.io 76 | kind: User 77 | name: user2 -------------------------------------------------------------------------------- /kustomize/base/sso/instance/sso-instance.yaml: -------------------------------------------------------------------------------- 1 | # Server 2 | apiVersion: keycloak.org/v1alpha1 3 | kind: Keycloak 4 | metadata: 5 | name: sso 6 | labels: 7 | app: sso 8 | spec: 9 | externalAccess: 10 | enabled: true 11 | instances: 2 12 | extensions: 13 | - https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar 14 | keycloakDeploymentSpec: 15 | resources: 16 | requests: 17 | cpu: 100m 18 | memory: 750Mi 19 | limits: 20 | cpu: 1 21 | memory: 1Gi 22 | postgresDeploymentSpec: 23 | resources: 24 | requests: 25 | cpu: 50m 26 | memory: 100Mi 27 | limits: 28 | cpu: 1 29 | memory: 500Mi 30 | --- 31 | # Realm 32 | apiVersion: keycloak.org/v1alpha1 33 | kind: KeycloakRealm 34 | metadata: 35 | name: myrealm 36 | labels: 37 | realm: myrealm 38 | spec: 39 | instanceSelector: 40 | matchLabels: 41 | app: sso 42 | realm: 43 | realm: myrealm 44 | enabled: true 45 | eventsListeners: 46 | - jboss-logging 47 | - metrics-listener 48 | --- 49 | apiVersion: keycloak.org/v1alpha1 50 | kind: KeycloakClient 51 | metadata: 52 | name: artemis-broker 53 | spec: 54 | realmSelector: 55 | matchLabels: 56 | realm: myrealm 57 | client: 58 | clientId: artemis-broker 59 | enabled: true 60 | publicClient: false 61 | clientAuthenticatorType: client-secret 62 | secret: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 63 | fullScopeAllowed: true 64 | directAccessGrantsEnabled: true 65 | protocol: openid-connect 66 | defaultClientScopes: 67 | - web-origins 68 | - role_list 69 | - profile 70 | - roles 71 | - email 72 | optionalClientScopes: 73 | - address 74 | - phone 75 | - offline_access 76 | - microprofile-jwt 77 | roles: 78 | - name: admin 79 | --- 80 | apiVersion: keycloak.org/v1alpha1 81 | kind: KeycloakUser 82 | metadata: 83 | name: admin 84 | spec: 85 | realmSelector: 86 | matchLabels: 87 | realm: myrealm 88 | user: 89 | username: admin 90 | enabled: true 91 | firstName: AMQ 92 | lastName: Admin 93 | emailVerified: false 94 | clientRoles: 95 | artemis-broker: 96 | - admin 97 | credentials: 98 | - type: password 99 | value: secret 100 | 101 | -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/grafana-readonly.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: integreatly.org/v1alpha1 2 | kind: Grafana 3 | metadata: 4 | name: grafana 5 | spec: 6 | config: 7 | log: 8 | mode: "console" 9 | level: "warn" 10 | auth: 11 | disable_login_form: True 12 | disable_signout_menu: True 13 | auth.basic: 14 | enabled: True 15 | auth.anonymous: 16 | enabled: True 17 | containers: 18 | - args: 19 | - '-provider=openshift' 20 | - '-pass-basic-auth=false' 21 | - '-https-address=:9091' 22 | - '-http-address=' 23 | - '-email-domain=*' 24 | - '-upstream=http://localhost:3000' 25 | # - '-openshift-sar={"resource": "namespaces", "verb": "get","namespace":"example"}' 26 | - '-tls-cert=/etc/tls/private/tls.crt' 27 | - '-tls-key=/etc/tls/private/tls.key' 28 | - '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token' 29 | - '-cookie-secret-file=/etc/proxy/secrets/session_secret' 30 | - '-openshift-service-account=grafana-serviceaccount' 31 | - '-openshift-ca=/etc/pki/tls/cert.pem' 32 | - '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' 33 | - '-openshift-ca=/etc/grafana-configmaps/ocp-injected-certs/ca-bundle.crt' 34 | - '-skip-auth-regex=^/metrics' 35 | image: 'quay.io/openshift/origin-oauth-proxy:4.9' 36 | name: grafana-proxy 37 | ports: 38 | - containerPort: 9091 39 | name: grafana-proxy 40 | resources: {} 41 | volumeMounts: 42 | - mountPath: /etc/tls/private 43 | name: secret-grafana-k8s-tls 44 | readOnly: false 45 | - mountPath: /etc/proxy/secrets 46 | name: secret-grafana-k8s-proxy 47 | readOnly: false 48 | secrets: 49 | - grafana-k8s-tls 50 | - grafana-k8s-proxy 51 | configMaps: 52 | - ocp-injected-certs 53 | service: 54 | ports: 55 | - name: grafana-proxy 56 | port: 9091 57 | protocol: TCP 58 | targetPort: grafana-proxy 59 | annotations: 60 | service.alpha.openshift.io/serving-cert-secret-name: grafana-k8s-tls 61 | resources: 62 | limits: 63 | cpu: '1' 64 | memory: 256Mi 65 | requests: 66 | cpu: 100m 67 | memory: 128Mi 68 | initResources: 69 | limits: 70 | cpu: '1' 71 | memory: 256Mi 72 | requests: 73 | cpu: 100m 74 | memory: 128Mi 75 | ingress: 76 | enabled: True 77 | targetPort: grafana-proxy 78 | termination: reencrypt 79 | client: 80 | preferService: True 81 | serviceAccount: 82 | annotations: 83 | serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"grafana-route"}}' 84 | dashboardLabelSelector: 85 | - matchExpressions: 86 | - { key: "monitoring-key", operator: In, values: ['middleware','openshift'] } -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/grafana-editor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: integreatly.org/v1alpha1 2 | kind: Grafana 3 | metadata: 4 | name: grafana 5 | spec: 6 | config: 7 | log: 8 | mode: "console" 9 | level: "info" 10 | auth: 11 | disable_login_form: True 12 | disable_signout_menu: True 13 | auth.basic: 14 | enabled: True 15 | auth.anonymous: 16 | enabled: False 17 | auth.proxy: 18 | auto_sign_up: true 19 | enabled: true 20 | header_name: X-Forwarded-User 21 | headers: 'Email:X-Forwarded-Email ' 22 | users: 23 | auto_assign_org_role: Editor 24 | containers: 25 | - args: 26 | - '-provider=openshift' 27 | - '-pass-basic-auth=false' 28 | - '-https-address=:9091' 29 | - '-http-address=' 30 | - '-email-domain=*' 31 | - '-upstream=http://localhost:3000' 32 | # - '-openshift-sar={"resource": "namespaces", "verb": "get","namespace":"example"}' 33 | - '-tls-cert=/etc/tls/private/tls.crt' 34 | - '-tls-key=/etc/tls/private/tls.key' 35 | - '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token' 36 | - '-cookie-secret-file=/etc/proxy/secrets/session_secret' 37 | - '-openshift-service-account=grafana-serviceaccount' 38 | - '-openshift-ca=/etc/pki/tls/cert.pem' 39 | - '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' 40 | - '-openshift-ca=/etc/grafana-configmaps/ocp-injected-certs/ca-bundle.crt' 41 | - '-skip-auth-regex=^/metrics' 42 | image: 'quay.io/openshift/origin-oauth-proxy:4.10' 43 | name: grafana-proxy 44 | ports: 45 | - containerPort: 9091 46 | name: grafana-proxy 47 | resources: {} 48 | volumeMounts: 49 | - mountPath: /etc/tls/private 50 | name: secret-grafana-k8s-tls 51 | readOnly: false 52 | - mountPath: /etc/proxy/secrets 53 | name: secret-grafana-k8s-proxy 54 | readOnly: false 55 | secrets: 56 | - grafana-k8s-tls 57 | - grafana-k8s-proxy 58 | configMaps: 59 | - ocp-injected-certs 60 | service: 61 | ports: 62 | - name: grafana-proxy 63 | port: 9091 64 | protocol: TCP 65 | targetPort: grafana-proxy 66 | annotations: 67 | service.alpha.openshift.io/serving-cert-secret-name: grafana-k8s-tls 68 | resources: 69 | limits: 70 | cpu: '1' 71 | memory: 256Mi 72 | requests: 73 | cpu: 100m 74 | memory: 128Mi 75 | initResources: 76 | limits: 77 | cpu: '1' 78 | memory: 256Mi 79 | requests: 80 | cpu: 100m 81 | memory: 128Mi 82 | ingress: 83 | enabled: True 84 | targetPort: grafana-proxy 85 | termination: reencrypt 86 | client: 87 | preferService: True 88 | serviceAccount: 89 | annotations: 90 | serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"grafana-route"}}' 91 | dashboardLabelSelector: 92 | - matchExpressions: 93 | - { key: "monitoring-key", operator: In, values: ['middleware','openshift'] } -------------------------------------------------------------------------------- /kustomize/base/grafana/instance/grafana-admin.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: integreatly.org/v1alpha1 2 | kind: Grafana 3 | metadata: 4 | name: grafana 5 | spec: 6 | # baseImage: docker.io/grafana/grafana:8.5.15 7 | config: 8 | log: 9 | mode: "console" 10 | level: "info" 11 | auth: 12 | disable_login_form: True 13 | disable_signout_menu: True 14 | auth.basic: 15 | enabled: True 16 | auth.anonymous: 17 | enabled: False 18 | auth.proxy: 19 | auto_sign_up: true 20 | enabled: true 21 | header_name: X-Forwarded-User 22 | headers: 'Email:X-Forwarded-Email ' 23 | users: 24 | auto_assign_org_role: Admin 25 | containers: 26 | - args: 27 | - '-provider=openshift' 28 | - '-pass-basic-auth=false' 29 | - '-https-address=:9091' 30 | - '-http-address=' 31 | - '-email-domain=*' 32 | - '-upstream=http://localhost:3000' 33 | # - '-openshift-sar={"resource": "namespaces", "verb": "get","namespace":"example"}' 34 | - '-tls-cert=/etc/tls/private/tls.crt' 35 | - '-tls-key=/etc/tls/private/tls.key' 36 | - '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token' 37 | - '-cookie-secret-file=/etc/proxy/secrets/session_secret' 38 | - '-openshift-service-account=grafana-serviceaccount' 39 | - '-openshift-ca=/etc/pki/tls/cert.pem' 40 | - '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' 41 | - '-openshift-ca=/etc/grafana-configmaps/ocp-injected-certs/ca-bundle.crt' 42 | - '-skip-auth-regex=^/metrics' 43 | # image: 'quay.io/openshift/origin-oauth-proxy:4.10' 44 | image: 'quay.io/openshift/origin-oauth-proxy:4.11' 45 | name: grafana-proxy 46 | ports: 47 | - containerPort: 9091 48 | name: grafana-proxy 49 | resources: {} 50 | volumeMounts: 51 | - mountPath: /etc/tls/private 52 | name: secret-grafana-k8s-tls 53 | readOnly: false 54 | - mountPath: /etc/proxy/secrets 55 | name: secret-grafana-k8s-proxy 56 | readOnly: false 57 | secrets: 58 | - grafana-k8s-tls 59 | - grafana-k8s-proxy 60 | configMaps: 61 | - ocp-injected-certs 62 | service: 63 | ports: 64 | - name: grafana-proxy 65 | port: 9091 66 | protocol: TCP 67 | targetPort: grafana-proxy 68 | annotations: 69 | service.alpha.openshift.io/serving-cert-secret-name: grafana-k8s-tls 70 | resources: 71 | limits: 72 | cpu: '1' 73 | memory: 256Mi 74 | requests: 75 | cpu: 100m 76 | memory: 128Mi 77 | initResources: 78 | limits: 79 | cpu: '1' 80 | memory: 256Mi 81 | requests: 82 | cpu: 100m 83 | memory: 128Mi 84 | ingress: 85 | enabled: True 86 | targetPort: grafana-proxy 87 | termination: reencrypt 88 | client: 89 | preferService: True 90 | serviceAccount: 91 | annotations: 92 | serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"grafana-route"}}' 93 | dashboardLabelSelector: 94 | - matchExpressions: 95 | - { key: "monitoring-key", operator: In, values: ['middleware','openshift'] } -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | # Enable User Workload Monitoring - https://docs.openshift.com/container-platform/4.11/monitoring/enabling-monitoring-for-user-defined-projects.html 2 | oc apply -k kustomize/env/openshift-monitoring 3 | 4 | ### Grafana cluster-scoped ### 5 | 6 | # Create grafana-monitoring namespace and install Grafana operator 7 | oc apply -k kustomize/env/grafana-monitoring-operator 8 | while ! oc wait --for condition=established crd/grafanas.integreatly.org; do sleep 1; done 9 | 10 | # Create Grafana instance in grafana-monitoring namespace 11 | SATOKEN=`oc extract secret/grafana-thanos-token -n grafana-monitoring --keys=token --to=-` 12 | # Also can get token from auto-created SA Secret: 13 | # SASECRET=$(oc get secret -o jsonpath='{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="grafana-thanos")].metadata.name}' --field-selector type=="kubernetes.io/service-account-token") 14 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/grafana-monitoring/kustomization.yaml 15 | # Could also patch DataSource afterwards intead of "sed" 16 | # oc patch GrafanaDataSource thanos --type=json -p='[{"op":"replace","path": "/spec/datasources/0/secureJsonData/httpHeaderValue1", "value": "'"Bearer $SATOKEN"'" }]' 17 | oc apply -k kustomize/env/grafana-monitoring 18 | 19 | ### App namespaces ### 20 | 21 | # Create team namespaces - as cluster-admin 22 | oc apply -k kustomize/env/team-a-namespace 23 | oc apply -k kustomize/env/team-b-namespace 24 | 25 | 26 | # Team-a - [Optional: Switch to user1] 27 | 28 | # Create team-a namespace and install Grafana, AMQ and SSO operator 29 | oc apply -k kustomize/env/team-a-operators 30 | 31 | # The first time you want to wait for CRDs to become ready on the cluster - Only works as cluster-admin 32 | while ! oc wait --for condition=established crd/activemqartemises.broker.amq.io; do sleep 1; done 33 | while ! oc wait --for condition=established crd/keycloaks.keycloak.org; do sleep 1; done 34 | 35 | # Workload + Grafana namespace scoped 36 | SATOKEN=`oc extract secret/grafana-thanos-token -n team-a --keys=token --to=-` 37 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/team-a-workload/kustomization.yaml 38 | oc apply -k kustomize/env/team-a-workload 39 | 40 | 41 | # Team-b - [Optional: Switch to user2] 42 | 43 | # Create team-b namespace and install Grafana, AMQ and SSO operator 44 | oc apply -k kustomize/env/team-b-operators 45 | 46 | SATOKEN=`oc extract secret/grafana-thanos-token -n team-b --keys=token --to=-` 47 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/team-b-workload/kustomization.yaml 48 | oc apply -k kustomize/env/team-b-workload 49 | 50 | ### Alerts ### 51 | 52 | # [Optional: Switch back to cluster-admin] 53 | 54 | # Enable alert routing for user-defined alerts 55 | oc apply -k kustomize/env/openshift-monitoring-alerts 56 | 57 | # Team-a - [Optional: Switch to user1] 58 | 59 | # AlertmanagerConfig 60 | oc apply -k kustomize/env/team-a-alerts 61 | 62 | # Send message to DLQ to trigger alert 63 | while ! oc wait -n team-a --for condition=Ready --timeout=180s pod/broker-ss-0; do sleep 1; done 64 | oc exec -n team-a broker-ss-0 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 65 | while ! oc wait -n team-a --for condition=Ready --timeout=180s pod/broker-ss-1; do sleep 1; done 66 | oc exec -n team-a broker-ss-1 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 67 | 68 | # Team-b - [Optional: Switch to user2] 69 | 70 | # AlertmanagerConfig 71 | oc apply -k kustomize/env/team-b-alerts 72 | 73 | # Send message to DLQ to trigger alert 74 | while ! oc wait -n team-b --for condition=Ready --timeout=180s pod/broker-ss-0; do sleep 1; done 75 | oc exec -n team-b broker-ss-0 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 76 | -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards-openshift/grafana-dashboards-openshift.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: integreatly.org/v1alpha1 2 | kind: GrafanaDashboard 3 | metadata: 4 | name: grafana-dashboard-cluster-total 5 | labels: 6 | app: grafana 7 | monitoring-key: openshift 8 | spec: 9 | # No datasource input is needed for these dashboards 10 | configMapRef: 11 | name: grafana-dashboard-cluster-total 12 | key: cluster-total.json 13 | --- 14 | apiVersion: integreatly.org/v1alpha1 15 | kind: GrafanaDashboard 16 | metadata: 17 | name: grafana-dashboard-etcd 18 | labels: 19 | app: grafana 20 | monitoring-key: openshift 21 | spec: 22 | configMapRef: 23 | name: grafana-dashboard-etcd 24 | key: etcd.json 25 | --- 26 | apiVersion: integreatly.org/v1alpha1 27 | kind: GrafanaDashboard 28 | metadata: 29 | name: grafana-dashboard-k8s-resources-cluster 30 | labels: 31 | app: grafana 32 | monitoring-key: openshift 33 | spec: 34 | configMapRef: 35 | name: grafana-dashboard-k8s-resources-cluster 36 | key: k8s-resources-cluster.json 37 | --- 38 | apiVersion: integreatly.org/v1alpha1 39 | kind: GrafanaDashboard 40 | metadata: 41 | name: grafana-dashboard-k8s-resources-namespace 42 | labels: 43 | app: grafana 44 | monitoring-key: openshift 45 | spec: 46 | configMapRef: 47 | name: grafana-dashboard-k8s-resources-namespace 48 | key: k8s-resources-namespace.json 49 | --- 50 | apiVersion: integreatly.org/v1alpha1 51 | kind: GrafanaDashboard 52 | metadata: 53 | name: grafana-dashboard-k8s-resources-node 54 | labels: 55 | app: grafana 56 | monitoring-key: openshift 57 | spec: 58 | configMapRef: 59 | name: grafana-dashboard-k8s-resources-node 60 | key: k8s-resources-node.json 61 | --- 62 | apiVersion: integreatly.org/v1alpha1 63 | kind: GrafanaDashboard 64 | metadata: 65 | name: grafana-dashboard-k8s-resources-pod 66 | labels: 67 | app: grafana 68 | monitoring-key: openshift 69 | spec: 70 | configMapRef: 71 | name: grafana-dashboard-k8s-resources-pod 72 | key: k8s-resources-pod.json 73 | --- 74 | apiVersion: integreatly.org/v1alpha1 75 | kind: GrafanaDashboard 76 | metadata: 77 | name: grafana-dashboard-k8s-resources-workload 78 | labels: 79 | app: grafana 80 | monitoring-key: openshift 81 | spec: 82 | configMapRef: 83 | name: grafana-dashboard-k8s-resources-workload 84 | key: k8s-resources-workload.json 85 | --- 86 | apiVersion: integreatly.org/v1alpha1 87 | kind: GrafanaDashboard 88 | metadata: 89 | name: grafana-dashboard-k8s-resources-workloads-namespace 90 | labels: 91 | app: grafana 92 | monitoring-key: openshift 93 | spec: 94 | configMapRef: 95 | name: grafana-dashboard-k8s-resources-workloads-namespace 96 | key: k8s-resources-workloads-namespace.json 97 | --- 98 | apiVersion: integreatly.org/v1alpha1 99 | kind: GrafanaDashboard 100 | metadata: 101 | name: grafana-dashboard-namespace-by-pod 102 | labels: 103 | app: grafana 104 | monitoring-key: openshift 105 | spec: 106 | configMapRef: 107 | name: grafana-dashboard-namespace-by-pod 108 | key: namespace-by-pod.json 109 | --- 110 | apiVersion: integreatly.org/v1alpha1 111 | kind: GrafanaDashboard 112 | metadata: 113 | name: grafana-dashboard-node-cluster-rsrc-use 114 | labels: 115 | app: grafana 116 | monitoring-key: openshift 117 | spec: 118 | configMapRef: 119 | name: grafana-dashboard-node-cluster-rsrc-use 120 | key: node-cluster-rsrc-use.json 121 | --- 122 | apiVersion: integreatly.org/v1alpha1 123 | kind: GrafanaDashboard 124 | metadata: 125 | name: grafana-dashboard-node-rsrc-use 126 | labels: 127 | app: grafana 128 | monitoring-key: openshift 129 | spec: 130 | configMapRef: 131 | name: grafana-dashboard-node-rsrc-use 132 | key: node-rsrc-use.json 133 | --- 134 | apiVersion: integreatly.org/v1alpha1 135 | kind: GrafanaDashboard 136 | metadata: 137 | name: grafana-dashboard-pod-total 138 | labels: 139 | app: grafana 140 | monitoring-key: openshift 141 | spec: 142 | configMapRef: 143 | name: grafana-dashboard-pod-total 144 | key: pod-total.json 145 | --- 146 | apiVersion: integreatly.org/v1alpha1 147 | kind: GrafanaDashboard 148 | metadata: 149 | name: grafana-dashboard-prometheus 150 | labels: 151 | app: grafana 152 | monitoring-key: openshift 153 | spec: 154 | configMapRef: 155 | name: grafana-dashboard-prometheus 156 | key: prometheus.json 157 | --- 158 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = OpenShift User Workload Monitoring with Grafana 2 | 3 | == Overview 4 | 5 | OpenShift comes with a https://docs.openshift.com/container-platform/4.11/monitoring/monitoring-overview.html[User Workload Monitoring] feature that can be used to collect _Prometheus_ metrics directly from deployed applications, similarly how metrics are collected from the platform itself. Platform and user metrics can both be queried through a joint Thanos Querier endpoint that's also exposed by a Route. 6 | 7 | The default _Observe_ section of the OpenShift UI provides dashboards for the platform metrics, but user metrics can only be queried by _PromQL_ queries. This example shows how to use _Grafana_ to access these metrics and use them on dashbards. 8 | 9 | There are two different ways to access these metrics: 10 | 11 | - Deploy a cluster-scoped Grafana that can access metrics from the whole cluster - cluster-admin pemission is required 12 | - Deploy namespace-scoped Grafana instances that can access metrics for one namespace only - permission to the namespace is enough 13 | 14 | [NOTE] 15 | Minimum OpenShift v4.10 is required for the Grafana dashboards with variables based on `label_values()`, the related api endpoints are not enabled in earlier versions. 16 | 17 | == Installation 18 | 19 | === Enable User Workload Monitoring 20 | 21 | First we need to https://docs.openshift.com/container-platform/4.11/monitoring/enabling-monitoring-for-user-defined-projects.html[enable] User Workload Monitoring by creating the related link:kustomize/env/openshift-monitoring[ConfigMaps]. 22 | 23 | ``` 24 | oc apply -k kustomize/env/openshift-monitoring 25 | ``` 26 | 27 | To test that user workload monitoring is up and running try to hit the internal or external endpoint: 28 | 29 | ``` 30 | # From outside the cluster - HTTP 401 Unauthorized is expected 31 | curl -vk 'https://federate-openshift-user-workload-monitoring.apps.[cluster domain]/federate' 32 | 33 | # From a Pod inside the cluster - HTTP 401 Unauthorized is expected 34 | curl -vk 'https://prometheus-user-workload.openshift-user-workload-monitoring.svc:9092/federate' 35 | ``` 36 | 37 | === Deploy a cluster-scoped Grafana 38 | 39 | Deploy a Grafana in `grafana-monitoring` namespace. Create a _ServiceAccount_ (`grafana-thanos`) that has cluster-scoped `cluster-monitoring-view` permission (_ClusterRoleBinding_) and add its token as `Bearer` auth header in the _GrafanaDatasource_. 40 | 41 | ``` 42 | oc apply -k kustomize/env/grafana-monitoring-operator 43 | 44 | SATOKEN=`oc extract secret/grafana-thanos-token -n grafana-monitoring --keys=token --to=-` 45 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/grafana-monitoring/kustomization.yaml 46 | 47 | oc apply -k kustomize/env/grafana-monitoring 48 | ``` 49 | 50 | [NOTE] 51 | In OpenShift v4.11 `oc sa get-token` is deprecated, `oc create token grafana-thanos` creates temporary tokens only. Create long-lived token Secret manually using the `kubernetes.io/service-account.name` annotation. 52 | 53 | Note: 54 | 55 | - This _GrafanaDatasource_ connects to endpoint `thanos-querier.openshift-monitoring.svc` on port *9091*, that requires cluster-scoped `cluster-monitoring-view` permission (for the _ServiceAccount_) 56 | - The _ServiceAccount_ token is added in _GrafanaDatasource_ as: 57 | 58 | ``` 59 | jsonData: 60 | httpHeaderName1: Authorization 61 | secureJsonData: 62 | httpHeaderValue1: Bearer [use grafana-thanos token] 63 | ``` 64 | 65 | - Access to Grafana UI requires cluster scoped permission by setting `-openshift-sar={"resource": "namespaces", "verb": "get"}`. This could be less restrictive and changed to let login anyone with access to the `grafana-monitoring` namespace by setting something like `-openshift-sar={"resource": "services", "verb": "get", "namespace":"grafana-monitoring"}` 66 | 67 | [NOTE] 68 | Make sure you use the `origin-oauth-proxy` image matching the cluster version 69 | 70 | === Deploy team specific namespaces 71 | 72 | Create namespaces `team-a` and `team-b` with an _OperatorGroup_ to enable operator installation and give `admin` and `monitoring-edit` permissions to "user1" and "user2": 73 | 74 | ``` 75 | oc apply -k kustomize/env/team-a-namespace 76 | oc apply -k kustomize/env/team-b-namespace 77 | ``` 78 | 79 | [NOTE] 80 | At this point we can switch to "user1" to manage "team-a" and "user2" for "team-b" as cluster-scoped permissions are not needed to install operators and deploy Grafana with namespace-scoped access to metrics. This means that application teams in an enterprise environment can deploy this solution in their own namespaces without assistance required from cluster admins. 81 | 82 | Deploy Grafana and our applications to monitor (using Red Hat SSO and AMQ Broker in this example for demonstration purposes): 83 | 84 | ``` 85 | oc apply -k kustomize/env/team-a-operators 86 | 87 | SATOKEN=`oc extract secret/grafana-thanos-token -n team-a --keys=token --to=-` 88 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/team-a-workload/kustomization.yaml 89 | 90 | oc apply -k kustomize/env/team-a-workload 91 | ``` 92 | 93 | And similarly for `team-b`: 94 | 95 | ``` 96 | oc apply -k kustomize/env/team-b-operators 97 | 98 | SATOKEN=`oc extract secret/grafana-thanos-token -n team-b --keys=token --to=-` 99 | sed -i '' "s/Bearer .*/Bearer $SATOKEN/" kustomize/env/team-b-workload/kustomization.yaml 100 | 101 | oc apply -k kustomize/env/team-b-workload 102 | ``` 103 | 104 | Note: 105 | 106 | - These _GrafanaDatasources_ connect to endpoint `thanos-querier.openshift-monitoring.svc` on port *9092* 107 | - The `Bearer` token used belongs to the _ServiceAccount_ having only namespace scoped `view` permission (_RoleBinding_) 108 | - This endpoint requires a matching `namespace=team-a` parameter, so we need to add `customQueryParameters: namespace=team-a`, so it's automatically added to the sent queries. (Though we have "namespace" filter in our queries used on the dashboards in this example) 109 | - Grafana UI requires only namespace scoped access to login: `-openshift-sar={"resource": "services", "verb": "get", "namespace":"team-a"}` 110 | 111 | [NOTE] 112 | If you have `NetworkPolicy` in your application namespaces, make sure incoming traffic from namespace `openshift-user-workload-monitoring` is link:kustomize/env/team-a-namespace/networkpolicy.yaml[enabled]. 113 | 114 | == Grafana 115 | 116 | Open the URL of created Grafana Routes (run `oc get route -ojsonpath='{$.spec.host}' grafana-route`) in all three namespaces and access one of the three deployed dashboard (AMQ, JMX, SSO): 117 | 118 | - grafana-monitoring: Both namespaces shows up in the _Namespace_ selector, but needs cluster level permission to login 119 | - team-a: Only metrics from namespace "team-a" shows up, but _user1_ can login 120 | - team-b: Only metrics from namespace "team-b" shows up, but _user2_ can login 121 | 122 | 123 | 124 | == Additional info 125 | 126 | Related blog: https://cloud.redhat.com/blog/thanos-querier-versus-thanos-querier 127 | 128 | To see the difference between the thanos-querier endpoints on port 9091 and 9092 we can run some curl commands. Port 9091 is exposed by a Route, for 9092 we can do port-forward: 129 | 130 | ``` 131 | BEARER_CLUSTER="$(oc extract secret/grafana-thanos-token -n grafana-monitoring --keys=token --to=-)" 132 | BEARER_TEAMA="$(oc extract secret/grafana-thanos-token -n team-a --keys=token --to=-)" 133 | BEARER_TEAMB="$(oc extract secret/grafana-thanos-token -n team-b --keys=token --to=-)" 134 | 135 | # Cluster scoped endpoint 136 | curl -vk -H "Authorization: Bearer $BEARER_CLUSTER" 'https://thanos-querier-openshift-monitoring.apps.[cluster domain]/api/v1/query?query=up' 137 | 138 | # Namespace scoped endpoint - the "namespace" filter is required 139 | oc port-forward -n openshift-monitoring service/thanos-querier 9092 9092 140 | curl -vk -H "Authorization: Bearer $BEARER_TEAMA" 'https://localhost:9092/api/v1/query?query=up&namespace=team-a' 141 | curl -vk -H "Authorization: Bearer $BEARER_TEAMB" 'https://localhost:9092/api/v1/query?query=up&namespace=team-b' 142 | ``` 143 | 144 | Instead of a _ServiceAccount_ token we can also use our own user token (`oc whoami -t`) as Bearer header. 145 | 146 | == Alerting 147 | 148 | 149 | 150 | === Alerting Rules 151 | 152 | OpenShift comes with default platform alerts defined in the `openshift-*` namespaces. See _Alerting rules_ on the UI or run `oc get PrometheusRules -A -oyaml | grep 'alert:'`. These alerts are evaluated by Prometheus running in the `openshift-monitoring` namespace and should not be modified. 153 | 154 | // OpenShift v4.11 has AlertingRule for custom rules: https://github.com/openshift/api/blob/master/monitoring/v1alpha1/0000_50_monitoring_01_alertingrules.crd.yaml 155 | 156 | To configure https://docs.openshift.com/container-platform/4.11/monitoring/managing-alerts.html#managing-alerting-rules-for-user-defined-projects_managing-alerts[alerts for User Workload Monitoring metrics] we create `PrometheusRule` resources in our namespaces (`team-a`,`team-b`). The Keycloak operator deploys these by default, for AMQ see link:kustomize/base/amq/instance/alerts.yaml[]. 157 | 158 | These rules are evaluated by _Thanos Ruler_ in `openshift-user-workload-monitoring` namespace, so we can use _User_ and _Platform_ metrics too in the expressions. The actual rule config snippets generated by the operator from PrometheusRules are stored in a ConfigMap, try `oc describe ConfigMap -n openshift-user-workload-monitoring -l thanos-ruler-name=user-workload`. These custom _User_ alerting rules are visible on the OpenShift UI in _Developer_ view on the _Observe / Alerts_ page. 159 | 160 | For details about rules see https://docs.openshift.com/container-platform/4.11/rest_api/monitoring_apis/prometheusrule-monitoring-coreos-com-v1.html[PrometheusRule spec] and https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/[Prometheus doc]. 161 | 162 | 163 | [NOTE] 164 | The `namespace` filter is automatically added to all metrics used in the expressions, which makes the _PrometheusRule_ resources usable in any namespaces, alerts will be based only on metrics coming from that namespace. The `namespace` (and `alertname`) label is also automatically added to the alerts when they fire. 165 | 166 | Role `monitoring-edit` or `monitoring-rules-edit` is required in the namespace to create _PrometheusRules_. 167 | 168 | === Notifications 169 | 170 | Receivers for the standard platform alerts can be configured using the https://docs.openshift.com/container-platform/4.11/monitoring/managing-alerts.html#configuring-alert-receivers_managing-alerts[OpenShift UI] or editing the https://docs.openshift.com/container-platform/4.11/monitoring/managing-alerts.html#applying-custom-alertmanager-configuration_managing-alerts[alertmanager-main] Secret directly. The easiest is to configure a `Default` receiver to get all alerts. Alerts are grouped by namespace, with a default `group_interval: 30s` (initial wait) `group_interval: 5m` (wait time before notifications about changes) and `repeat_interval: 12h` (time before repeating an unchanged notification), see https://prometheus.io/docs/alerting/latest/configuration/#route[details]. 171 | 172 | While the _User_ alerts are also included in the _Default_ alert notifications, it's not practical to send _User_ alerts (relevant to application teams) to the same channels as _Platform_ alerts (relevant for cluster admins). Fortunately we can define custom, namespace scoped https://docs.openshift.com/container-platform/4.11/monitoring/managing-alerts.html#creating-alert-routing-for-user-defined-projects_managing-alerts[alert notification routes] by creating `AlertmanagerConfigs`. 173 | 174 | We need to enable this custom alerting feature by adding `enableUserAlertmanagerConfig: true` to `cluster-monitoring-config` ConfigMap (for OpenShift v4.11+ see <>): 175 | 176 | ``` 177 | config.yaml: | 178 | ... 179 | alertmanagerMain: 180 | enableUserAlertmanagerConfig: true 181 | ``` 182 | 183 | Then we can create _AlertmanagerConfigs_ in our namespaces (`team-a`,`team-b`), for example: 184 | 185 | ``` 186 | apiVersion: monitoring.coreos.com/v1alpha1 187 | kind: AlertmanagerConfig 188 | metadata: 189 | name: alert-notifications 190 | spec: 191 | route: 192 | receiver: default 193 | groupWait: 30s 194 | groupInterval: 5m 195 | repeatInterval: 12h 196 | receivers: 197 | - name: default 198 | webhookConfigs: 199 | - url: https://webhook.example.com/ 200 | ``` 201 | 202 | The alert notifications are sent by _Alertmanager_ in the `openshift-monitoring` namespace (unless separate _Alertmanager_ instance is enabled for user-defined alerts). 203 | The actual alerting route config snippets generated by the operator from _AlertmanagerConfigs_ (merged with the cluster level `alertmanager-main` config) are stored in a Secret, try `oc extract -n openshift-monitoring --to=- secret/alertmanager-main-generated`. 204 | 205 | For details about alerting routes see https://docs.openshift.com/container-platform/4.11/rest_api/monitoring_apis/alertmanagerconfig-monitoring-coreos-com-v1beta1.html[AlertmanagerConfig spec] and https://prometheus.io/docs/alerting/latest/configuration/[Prometheus doc]. 206 | 207 | Note: 208 | 209 | * If an _AlertmanagerConfig_ is created in the namespace, the _Default_ notification channel is not used anymore for user alerts in that namespace 210 | * Group by `namespace` is automatically added, but we can add additional labels (e.g. `pod`) to get separate notifications accordingly 211 | * Matchers for `namespace` is automatically added, so custom _AlertmanagerConfigs_ route only alerts from the namespace they were created in 212 | * We can have multiple _AlertmanagerConfigs_ in a namespace, they are merged together 213 | * For a more complex _AlertmanagerConfig_ with sub-routes see link:kustomize/env/team-b-workload/alertmanagerconfig.yaml[]. 214 | 215 | ClusterRole `alert-routing-edit` is required in the namespace to create _AlertmanagerConfigs_. 216 | 217 | === Separate Alertmanager for user-defined alerts 218 | 219 | In OpenShift v4.11+ a https://docs.openshift.com/container-platform/4.11/monitoring/enabling-alert-routing-for-user-defined-projects.html#enabling-a-separate-alertmanager-instance-for-user-defined-alert-routing_enabling-alert-routing-for-user-defined-projects[separate Alertmanager instance] can be deployed in the `openshift-user-workload-monitoring` namespace to separate notifications coming from _User_ and _Platform_ alerts. This is a good practice to avoid _User_ alerts being sent through the _Platfom_ notification routes accidentally if _AlertmanagerConfigs_ were not created in a namespace. 220 | 221 | Enable in `user-workload-monitoring-config` _ConfigMap_ (instead of `cluster-monitoring-config`): 222 | 223 | ``` 224 | config.yaml: | 225 | ... 226 | alertmanager: 227 | enabled: true 228 | enableAlertmanagerConfig: true 229 | ``` 230 | 231 | === Test alerting 232 | 233 | _PrometheusRules_ were already deployed in the previous steps and they visible under _Observe_ in the OpenShift UI. 234 | 235 | Setup custom alert notifications: 236 | 237 | ``` 238 | # Enable alert routing for user-defined alerts 239 | oc apply -k kustomize/env/openshift-monitoring-alerts 240 | # Team-a - [Optional: Switch to user1] 241 | oc apply -k kustomize/env/team-a-alerts 242 | # Team-b - [Optional: Switch to user2] 243 | oc apply -k kustomize/env/team-b-alerts 244 | ``` 245 | 246 | 247 | Send message to DLQ to trigger alerts: 248 | ``` 249 | oc exec -n team-a broker-ss-0 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 250 | oc exec -n team-a broker-ss-1 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 251 | oc exec -n team-b broker-ss-0 -- sh -c '/home/jboss/amq-broker/bin/artemis producer --message-count 1 --destination DLQ --url tcp://$(hostname):61617 --user admin --password admin' 252 | 253 | ``` 254 | 255 | == OpenShift 4.10 v 4.11 differences 256 | 257 | Files to check: 258 | 259 | * Separate Alertmanager in link:kustomize/env/openshift-monitoring-alerts/kustomization.yaml[] 260 | * OAuth proxy for Grafana in link:kustomize/base/grafana/instance/grafana-admin.yaml[] 261 | * API version in link:kustomize/env/team-a-alerts/alertmanagerconfig.yaml[] and link:kustomize/env/team-b-alerts/alertmanagerconfig.yaml[] 262 | * OpenShift dashboards in link:kustomize/base/grafana/dashboards-openshift/kustomization.yaml[] 263 | -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards/dashboard-sso.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS", 5 | "label": "Prometheus", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "gauge", 16 | "name": "Gauge", 17 | "version": "" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "7.5.15" 24 | }, 25 | { 26 | "type": "panel", 27 | "id": "graph", 28 | "name": "Graph", 29 | "version": "" 30 | }, 31 | { 32 | "type": "datasource", 33 | "id": "prometheus", 34 | "name": "Prometheus", 35 | "version": "1.0.0" 36 | }, 37 | { 38 | "type": "panel", 39 | "id": "stat", 40 | "name": "Stat", 41 | "version": "" 42 | } 43 | ], 44 | "annotations": { 45 | "list": [ 46 | { 47 | "builtIn": 1, 48 | "datasource": "${DS_PROMETHEUS}", 49 | "enable": true, 50 | "hide": true, 51 | "iconColor": "rgba(0, 211, 255, 1)", 52 | "name": "Annotations & Alerts", 53 | "type": "dashboard" 54 | } 55 | ] 56 | }, 57 | "description": "Monitoring Keycloak metrics", 58 | "editable": true, 59 | "gnetId": null, 60 | "graphTooltip": 1, 61 | "id": null, 62 | "iteration": 1651252487415, 63 | "links": [], 64 | "panels": [ 65 | { 66 | "cacheTimeout": null, 67 | "datasource": "${DS_PROMETHEUS}", 68 | "fieldConfig": { 69 | "defaults": { 70 | "color": { 71 | "mode": "thresholds" 72 | }, 73 | "mappings": [], 74 | "max": 100, 75 | "thresholds": { 76 | "mode": "absolute", 77 | "steps": [ 78 | { 79 | "color": "green", 80 | "value": null 81 | }, 82 | { 83 | "color": "red", 84 | "value": 80 85 | } 86 | ] 87 | }, 88 | "unit": "percent" 89 | }, 90 | "overrides": [] 91 | }, 92 | "gridPos": { 93 | "h": 5, 94 | "w": 7, 95 | "x": 0, 96 | "y": 0 97 | }, 98 | "id": 55, 99 | "interval": null, 100 | "links": [], 101 | "maxDataPoints": 3, 102 | "options": { 103 | "reduceOptions": { 104 | "calcs": [ 105 | "lastNotNull" 106 | ], 107 | "fields": "", 108 | "values": false 109 | }, 110 | "showThresholdLabels": false, 111 | "showThresholdMarkers": true, 112 | "text": {} 113 | }, 114 | "pluginVersion": "7.5.15", 115 | "targets": [ 116 | { 117 | "exemplar": true, 118 | "expr": "sum(jvm_memory_bytes_used{namespace=\"$namespace\", pod=\"keycloak-0\", area=\"heap\"})*100/sum(jvm_memory_bytes_max{namespace=\"$namespace\", pod=\"keycloak-0\", area=\"heap\"})", 119 | "format": "time_series", 120 | "hide": false, 121 | "interval": "", 122 | "intervalFactor": 1, 123 | "legendFormat": "keycloak-0", 124 | "refId": "C" 125 | }, 126 | { 127 | "exemplar": true, 128 | "expr": "sum(jvm_memory_bytes_used{namespace=\"$namespace\", pod=\"keycloak-1\", area=\"heap\"})*100/sum(jvm_memory_bytes_max{namespace=\"$namespace\", pod=\"keycloak-1\", area=\"heap\"})", 129 | "format": "time_series", 130 | "hide": false, 131 | "interval": "", 132 | "intervalFactor": 1, 133 | "legendFormat": "keycloak-1", 134 | "refId": "D" 135 | } 136 | ], 137 | "timeFrom": null, 138 | "timeShift": null, 139 | "title": "Heap used", 140 | "type": "gauge" 141 | }, 142 | { 143 | "datasource": "${DS_PROMETHEUS}", 144 | "fieldConfig": { 145 | "defaults": { 146 | "color": { 147 | "mode": "thresholds" 148 | }, 149 | "mappings": [], 150 | "thresholds": { 151 | "mode": "absolute", 152 | "steps": [ 153 | { 154 | "color": "green", 155 | "value": null 156 | }, 157 | { 158 | "color": "red", 159 | "value": 80 160 | } 161 | ] 162 | } 163 | }, 164 | "overrides": [] 165 | }, 166 | "gridPos": { 167 | "h": 5, 168 | "w": 5, 169 | "x": 7, 170 | "y": 0 171 | }, 172 | "id": 25, 173 | "links": [], 174 | "options": { 175 | "colorMode": "value", 176 | "graphMode": "area", 177 | "justifyMode": "auto", 178 | "orientation": "auto", 179 | "reduceOptions": { 180 | "calcs": [ 181 | "lastNotNull" 182 | ], 183 | "fields": "", 184 | "values": false 185 | }, 186 | "text": {}, 187 | "textMode": "auto" 188 | }, 189 | "pluginVersion": "7.5.15", 190 | "targets": [ 191 | { 192 | "exemplar": true, 193 | "expr": "sum(up{namespace=\"$namespace\", job=\"keycloak\"})", 194 | "format": "time_series", 195 | "hide": false, 196 | "interval": "", 197 | "intervalFactor": 1, 198 | "legendFormat": "Keycloak Instances", 199 | "refId": "A" 200 | }, 201 | { 202 | "exemplar": true, 203 | "expr": "kube_deployment_status_condition{namespace=\"$namespace\", deployment=\"keycloak-postgresql\", status=\"true\", condition=\"Available\"}", 204 | "format": "time_series", 205 | "hide": false, 206 | "instant": false, 207 | "interval": "", 208 | "intervalFactor": 1, 209 | "legendFormat": "{{deployment}}", 210 | "refId": "B" 211 | } 212 | ], 213 | "timeFrom": null, 214 | "timeShift": null, 215 | "title": "Pods status", 216 | "type": "stat" 217 | }, 218 | { 219 | "datasource": "${DS_PROMETHEUS}", 220 | "fieldConfig": { 221 | "defaults": { 222 | "color": { 223 | "mode": "thresholds" 224 | }, 225 | "decimals": 0, 226 | "mappings": [], 227 | "thresholds": { 228 | "mode": "absolute", 229 | "steps": [ 230 | { 231 | "color": "green", 232 | "value": null 233 | }, 234 | { 235 | "color": "red", 236 | "value": 80 237 | } 238 | ] 239 | }, 240 | "unit": "none" 241 | }, 242 | "overrides": [] 243 | }, 244 | "gridPos": { 245 | "h": 5, 246 | "w": 3, 247 | "x": 12, 248 | "y": 0 249 | }, 250 | "id": 65, 251 | "links": [], 252 | "options": { 253 | "colorMode": "value", 254 | "graphMode": "area", 255 | "justifyMode": "auto", 256 | "orientation": "auto", 257 | "reduceOptions": { 258 | "calcs": [ 259 | "lastNotNull" 260 | ], 261 | "fields": "", 262 | "values": false 263 | }, 264 | "text": {}, 265 | "textMode": "value" 266 | }, 267 | "pluginVersion": "7.5.15", 268 | "targets": [ 269 | { 270 | "exemplar": true, 271 | "expr": "sum(increase(keycloak_response_errors{namespace=\"$namespace\", job=\"keycloak\"}[5m]))", 272 | "format": "time_series", 273 | "interval": "", 274 | "intervalFactor": 1, 275 | "legendFormat": "", 276 | "refId": "A" 277 | } 278 | ], 279 | "timeFrom": null, 280 | "timeShift": null, 281 | "title": "HTTP Errors count [5m]", 282 | "type": "stat" 283 | }, 284 | { 285 | "datasource": "${DS_PROMETHEUS}", 286 | "fieldConfig": { 287 | "defaults": { 288 | "color": { 289 | "mode": "thresholds" 290 | }, 291 | "decimals": 0, 292 | "mappings": [], 293 | "thresholds": { 294 | "mode": "absolute", 295 | "steps": [ 296 | { 297 | "color": "green", 298 | "value": null 299 | }, 300 | { 301 | "color": "red", 302 | "value": 80 303 | } 304 | ] 305 | }, 306 | "unit": "none" 307 | }, 308 | "overrides": [] 309 | }, 310 | "gridPos": { 311 | "h": 5, 312 | "w": 4, 313 | "x": 15, 314 | "y": 0 315 | }, 316 | "id": 66, 317 | "links": [], 318 | "options": { 319 | "colorMode": "value", 320 | "graphMode": "area", 321 | "justifyMode": "auto", 322 | "orientation": "auto", 323 | "reduceOptions": { 324 | "calcs": [ 325 | "lastNotNull" 326 | ], 327 | "fields": "", 328 | "values": false 329 | }, 330 | "text": {}, 331 | "textMode": "value" 332 | }, 333 | "pluginVersion": "7.5.15", 334 | "targets": [ 335 | { 336 | "exemplar": true, 337 | "expr": "sum(jboss_datasources_pool_active_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}) by (pod)", 338 | "format": "time_series", 339 | "interval": "", 340 | "intervalFactor": 1, 341 | "legendFormat": "{{pod}}", 342 | "refId": "A" 343 | } 344 | ], 345 | "timeFrom": null, 346 | "timeShift": null, 347 | "title": "DB active connections", 348 | "type": "stat" 349 | }, 350 | { 351 | "cacheTimeout": null, 352 | "datasource": "${DS_PROMETHEUS}", 353 | "fieldConfig": { 354 | "defaults": { 355 | "color": { 356 | "mode": "thresholds" 357 | }, 358 | "mappings": [], 359 | "thresholds": { 360 | "mode": "absolute", 361 | "steps": [ 362 | { 363 | "color": "green", 364 | "value": null 365 | }, 366 | { 367 | "color": "red", 368 | "value": 80 369 | } 370 | ] 371 | } 372 | }, 373 | "overrides": [] 374 | }, 375 | "gridPos": { 376 | "h": 5, 377 | "w": 5, 378 | "x": 19, 379 | "y": 0 380 | }, 381 | "id": 54, 382 | "interval": null, 383 | "links": [], 384 | "maxDataPoints": 3, 385 | "options": { 386 | "colorMode": "value", 387 | "graphMode": "area", 388 | "justifyMode": "auto", 389 | "orientation": "auto", 390 | "reduceOptions": { 391 | "calcs": [ 392 | "lastNotNull" 393 | ], 394 | "fields": "", 395 | "values": false 396 | }, 397 | "text": {}, 398 | "textMode": "auto" 399 | }, 400 | "pluginVersion": "7.5.15", 401 | "targets": [ 402 | { 403 | "exemplar": true, 404 | "expr": "jboss_jgroups_num_members{namespace=\"$namespace\", container=\"keycloak\"}", 405 | "format": "time_series", 406 | "hide": false, 407 | "interval": "", 408 | "intervalFactor": 1, 409 | "legendFormat": "{{pod}} view", 410 | "refId": "A" 411 | } 412 | ], 413 | "timeFrom": null, 414 | "timeShift": null, 415 | "title": "JGroups cluster view", 416 | "type": "stat" 417 | }, 418 | { 419 | "collapsed": false, 420 | "datasource": null, 421 | "gridPos": { 422 | "h": 1, 423 | "w": 24, 424 | "x": 0, 425 | "y": 5 426 | }, 427 | "id": 53, 428 | "panels": [], 429 | "title": "Database and Caches", 430 | "type": "row" 431 | }, 432 | { 433 | "aliasColors": {}, 434 | "bars": false, 435 | "dashLength": 10, 436 | "dashes": false, 437 | "datasource": "${DS_PROMETHEUS}", 438 | "fieldConfig": { 439 | "defaults": {}, 440 | "overrides": [] 441 | }, 442 | "fill": 1, 443 | "fillGradient": 0, 444 | "gridPos": { 445 | "h": 8, 446 | "w": 12, 447 | "x": 0, 448 | "y": 6 449 | }, 450 | "hiddenSeries": false, 451 | "id": 48, 452 | "legend": { 453 | "avg": false, 454 | "current": false, 455 | "max": false, 456 | "min": false, 457 | "show": true, 458 | "total": false, 459 | "values": false 460 | }, 461 | "lines": true, 462 | "linewidth": 1, 463 | "links": [], 464 | "nullPointMode": "null", 465 | "options": { 466 | "alertThreshold": true 467 | }, 468 | "percentage": false, 469 | "pluginVersion": "7.5.15", 470 | "pointradius": 2, 471 | "points": false, 472 | "renderer": "flot", 473 | "seriesOverrides": [], 474 | "spaceLength": 10, 475 | "stack": false, 476 | "steppedLine": false, 477 | "targets": [ 478 | { 479 | "exemplar": false, 480 | "expr": "sum(jboss_datasources_pool_active_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}) by (pod)", 481 | "format": "time_series", 482 | "interval": "", 483 | "intervalFactor": 1, 484 | "legendFormat": "Active for {{pod}}", 485 | "refId": "A" 486 | }, 487 | { 488 | "exemplar": true, 489 | "expr": "sum(jboss_datasources_pool_available_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}) by (pod)", 490 | "format": "time_series", 491 | "hide": false, 492 | "interval": "", 493 | "legendFormat": "Available for {{pod}}", 494 | "refId": "B" 495 | } 496 | ], 497 | "thresholds": [], 498 | "timeFrom": null, 499 | "timeRegions": [], 500 | "timeShift": null, 501 | "title": "Number of database connections in pool", 502 | "tooltip": { 503 | "shared": true, 504 | "sort": 0, 505 | "value_type": "individual" 506 | }, 507 | "transformations": [], 508 | "type": "graph", 509 | "xaxis": { 510 | "buckets": null, 511 | "mode": "time", 512 | "name": null, 513 | "show": true, 514 | "values": [] 515 | }, 516 | "yaxes": [ 517 | { 518 | "format": "short", 519 | "label": null, 520 | "logBase": 1, 521 | "max": null, 522 | "min": null, 523 | "show": true 524 | }, 525 | { 526 | "format": "short", 527 | "label": null, 528 | "logBase": 1, 529 | "max": null, 530 | "min": null, 531 | "show": true 532 | } 533 | ], 534 | "yaxis": { 535 | "align": false, 536 | "alignLevel": null 537 | } 538 | }, 539 | { 540 | "aliasColors": {}, 541 | "bars": false, 542 | "dashLength": 10, 543 | "dashes": false, 544 | "datasource": "${DS_PROMETHEUS}", 545 | "fieldConfig": { 546 | "defaults": {}, 547 | "overrides": [] 548 | }, 549 | "fill": 1, 550 | "fillGradient": 0, 551 | "gridPos": { 552 | "h": 8, 553 | "w": 12, 554 | "x": 12, 555 | "y": 6 556 | }, 557 | "hiddenSeries": false, 558 | "id": 49, 559 | "legend": { 560 | "avg": false, 561 | "current": false, 562 | "max": false, 563 | "min": false, 564 | "show": true, 565 | "total": false, 566 | "values": false 567 | }, 568 | "lines": true, 569 | "linewidth": 1, 570 | "links": [], 571 | "nullPointMode": "null", 572 | "options": { 573 | "alertThreshold": true 574 | }, 575 | "percentage": false, 576 | "pluginVersion": "7.5.15", 577 | "pointradius": 2, 578 | "points": false, 579 | "renderer": "flot", 580 | "seriesOverrides": [], 581 | "spaceLength": 10, 582 | "stack": false, 583 | "steppedLine": false, 584 | "targets": [ 585 | { 586 | "exemplar": true, 587 | "expr": "increase(jboss_datasources_pool_xacommit_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}[30m])", 588 | "format": "time_series", 589 | "interval": "", 590 | "intervalFactor": 1, 591 | "legendFormat": "Number of commits for {{pod}}", 592 | "refId": "A" 593 | }, 594 | { 595 | "exemplar": true, 596 | "expr": "increase(jboss_datasources_pool_xarollback_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}[30m])", 597 | "format": "time_series", 598 | "interval": "", 599 | "intervalFactor": 1, 600 | "legendFormat": "Number of rollbacks for {{pod}}", 601 | "refId": "B" 602 | }, 603 | { 604 | "exemplar": true, 605 | "expr": "increase(jboss_datasources_pool_blocking_failure_count{namespace=\"$namespace\", xa_data_source=\"keycloak_postgresql-DB\"}[30m])", 606 | "format": "time_series", 607 | "interval": "", 608 | "intervalFactor": 1, 609 | "legendFormat": "Number of failures for {{pod}}", 610 | "refId": "C" 611 | } 612 | ], 613 | "thresholds": [], 614 | "timeFrom": null, 615 | "timeRegions": [], 616 | "timeShift": null, 617 | "title": "Number of transactions [30m]", 618 | "tooltip": { 619 | "shared": true, 620 | "sort": 0, 621 | "value_type": "individual" 622 | }, 623 | "type": "graph", 624 | "xaxis": { 625 | "buckets": null, 626 | "mode": "time", 627 | "name": null, 628 | "show": true, 629 | "values": [] 630 | }, 631 | "yaxes": [ 632 | { 633 | "format": "short", 634 | "label": null, 635 | "logBase": 1, 636 | "max": null, 637 | "min": null, 638 | "show": true 639 | }, 640 | { 641 | "format": "short", 642 | "label": null, 643 | "logBase": 1, 644 | "max": null, 645 | "min": null, 646 | "show": true 647 | } 648 | ], 649 | "yaxis": { 650 | "align": false, 651 | "alignLevel": null 652 | } 653 | }, 654 | { 655 | "aliasColors": {}, 656 | "bars": false, 657 | "dashLength": 10, 658 | "dashes": false, 659 | "datasource": "${DS_PROMETHEUS}", 660 | "fieldConfig": { 661 | "defaults": {}, 662 | "overrides": [] 663 | }, 664 | "fill": 1, 665 | "fillGradient": 0, 666 | "gridPos": { 667 | "h": 12, 668 | "w": 24, 669 | "x": 0, 670 | "y": 14 671 | }, 672 | "hiddenSeries": false, 673 | "id": 45, 674 | "legend": { 675 | "alignAsTable": true, 676 | "avg": false, 677 | "current": true, 678 | "max": false, 679 | "min": false, 680 | "rightSide": true, 681 | "show": true, 682 | "total": false, 683 | "values": true 684 | }, 685 | "lines": true, 686 | "linewidth": 1, 687 | "links": [], 688 | "nullPointMode": "null", 689 | "options": { 690 | "alertThreshold": true 691 | }, 692 | "percentage": false, 693 | "pluginVersion": "7.5.15", 694 | "pointradius": 2, 695 | "points": false, 696 | "renderer": "flot", 697 | "seriesOverrides": [], 698 | "spaceLength": 10, 699 | "stack": false, 700 | "steppedLine": false, 701 | "targets": [ 702 | { 703 | "exemplar": true, 704 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"actionTokens\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)", 705 | "format": "time_series", 706 | "interval": "", 707 | "intervalFactor": 1, 708 | "legendFormat": "actionTokens for {{pod}}", 709 | "refId": "A" 710 | }, 711 | { 712 | "exemplar": true, 713 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"authenticationSessions\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)", 714 | "format": "time_series", 715 | "interval": "", 716 | "intervalFactor": 1, 717 | "legendFormat": "authenticationSessions for {{pod}}", 718 | "refId": "B" 719 | }, 720 | { 721 | "exemplar": true, 722 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"clientSessions\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 723 | "format": "time_series", 724 | "interval": "", 725 | "intervalFactor": 1, 726 | "legendFormat": "clientSessions for {{pod}}", 727 | "refId": "C" 728 | }, 729 | { 730 | "exemplar": true, 731 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"loginFailures\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 732 | "format": "time_series", 733 | "interval": "", 734 | "intervalFactor": 1, 735 | "legendFormat": "loginFailures for {{pod}}", 736 | "refId": "D" 737 | }, 738 | { 739 | "exemplar": true, 740 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"offlineClientSessions\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 741 | "format": "time_series", 742 | "interval": "", 743 | "intervalFactor": 1, 744 | "legendFormat": "offlineClientSessions for {{pod}}", 745 | "refId": "E" 746 | }, 747 | { 748 | "exemplar": true, 749 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"offlineSessions\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 750 | "format": "time_series", 751 | "interval": "", 752 | "intervalFactor": 1, 753 | "legendFormat": "offlineSessions for {{pod}}", 754 | "refId": "F" 755 | }, 756 | { 757 | "exemplar": true, 758 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"sessions\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 759 | "format": "time_series", 760 | "interval": "", 761 | "intervalFactor": 1, 762 | "legendFormat": "sessions for {{pod}}", 763 | "refId": "G" 764 | }, 765 | { 766 | "exemplar": true, 767 | "expr": "sum(jboss_infinispan_number_of_entries{cache=\"work\", cache_container=\"keycloak\", namespace=\"$namespace\"}) by (pod)\r", 768 | "hide": false, 769 | "interval": "", 770 | "legendFormat": "work for {{pod}}", 771 | "refId": "H" 772 | } 773 | ], 774 | "thresholds": [], 775 | "timeFrom": null, 776 | "timeRegions": [], 777 | "timeShift": null, 778 | "title": "Number of entries in caches", 779 | "tooltip": { 780 | "shared": true, 781 | "sort": 0, 782 | "value_type": "individual" 783 | }, 784 | "type": "graph", 785 | "xaxis": { 786 | "buckets": null, 787 | "mode": "time", 788 | "name": null, 789 | "show": true, 790 | "values": [] 791 | }, 792 | "yaxes": [ 793 | { 794 | "format": "short", 795 | "label": null, 796 | "logBase": 1, 797 | "max": null, 798 | "min": null, 799 | "show": true 800 | }, 801 | { 802 | "format": "short", 803 | "label": null, 804 | "logBase": 1, 805 | "max": null, 806 | "min": null, 807 | "show": true 808 | } 809 | ], 810 | "yaxis": { 811 | "align": false, 812 | "alignLevel": null 813 | } 814 | }, 815 | { 816 | "collapsed": false, 817 | "datasource": null, 818 | "gridPos": { 819 | "h": 1, 820 | "w": 24, 821 | "x": 0, 822 | "y": 26 823 | }, 824 | "id": 33, 825 | "panels": [], 826 | "title": "Endpoints", 827 | "type": "row" 828 | }, 829 | { 830 | "aliasColors": {}, 831 | "bars": false, 832 | "cacheTimeout": null, 833 | "dashLength": 10, 834 | "dashes": false, 835 | "datasource": "${DS_PROMETHEUS}", 836 | "description": "", 837 | "fieldConfig": { 838 | "defaults": {}, 839 | "overrides": [] 840 | }, 841 | "fill": 1, 842 | "fillGradient": 0, 843 | "gridPos": { 844 | "h": 13, 845 | "w": 24, 846 | "x": 0, 847 | "y": 27 848 | }, 849 | "hiddenSeries": false, 850 | "id": 59, 851 | "legend": { 852 | "alignAsTable": true, 853 | "avg": true, 854 | "current": false, 855 | "max": true, 856 | "min": true, 857 | "show": true, 858 | "sort": "avg", 859 | "sortDesc": true, 860 | "total": false, 861 | "values": true 862 | }, 863 | "lines": true, 864 | "linewidth": 1, 865 | "links": [], 866 | "nullPointMode": "null", 867 | "options": { 868 | "alertThreshold": true 869 | }, 870 | "percentage": false, 871 | "pluginVersion": "7.5.15", 872 | "pointradius": 2, 873 | "points": false, 874 | "renderer": "flot", 875 | "seriesOverrides": [], 876 | "spaceLength": 10, 877 | "stack": false, 878 | "steppedLine": false, 879 | "targets": [ 880 | { 881 | "exemplar": true, 882 | "expr": "rate(keycloak_request_duration_sum{namespace=\"$namespace\", job=\"keycloak\", resource=~\"realms,realms/myrealm/.+\"}[30m]) /rate(keycloak_request_duration_count{namespace=\"$namespace\", job=\"keycloak\", resource=~\"realms,realms/myrealm/.+\"}[30m])", 883 | "format": "time_series", 884 | "hide": false, 885 | "instant": false, 886 | "interval": "", 887 | "intervalFactor": 1, 888 | "legendFormat": "{{ pod }} {{method}} {{resource}}", 889 | "refId": "A" 890 | } 891 | ], 892 | "thresholds": [], 893 | "timeFrom": null, 894 | "timeRegions": [], 895 | "timeShift": null, 896 | "title": "Realm myrealm HTTP endpoints [30m]", 897 | "tooltip": { 898 | "shared": true, 899 | "sort": 0, 900 | "value_type": "individual" 901 | }, 902 | "type": "graph", 903 | "xaxis": { 904 | "buckets": null, 905 | "mode": "time", 906 | "name": null, 907 | "show": true, 908 | "values": [] 909 | }, 910 | "yaxes": [ 911 | { 912 | "format": "ms", 913 | "label": null, 914 | "logBase": 1, 915 | "max": null, 916 | "min": null, 917 | "show": true 918 | }, 919 | { 920 | "format": "short", 921 | "label": null, 922 | "logBase": 1, 923 | "max": null, 924 | "min": null, 925 | "show": true 926 | } 927 | ], 928 | "yaxis": { 929 | "align": false, 930 | "alignLevel": null 931 | } 932 | }, 933 | { 934 | "collapsed": false, 935 | "datasource": null, 936 | "gridPos": { 937 | "h": 1, 938 | "w": 24, 939 | "x": 0, 940 | "y": 40 941 | }, 942 | "id": 35, 943 | "panels": [], 944 | "title": "Detailed metrics", 945 | "type": "row" 946 | }, 947 | { 948 | "aliasColors": {}, 949 | "bars": false, 950 | "dashLength": 10, 951 | "dashes": false, 952 | "datasource": "${DS_PROMETHEUS}", 953 | "fieldConfig": { 954 | "defaults": {}, 955 | "overrides": [] 956 | }, 957 | "fill": 1, 958 | "fillGradient": 0, 959 | "gridPos": { 960 | "h": 12, 961 | "w": 12, 962 | "x": 0, 963 | "y": 41 964 | }, 965 | "hiddenSeries": false, 966 | "hideTimeOverride": false, 967 | "id": 39, 968 | "legend": { 969 | "alignAsTable": true, 970 | "avg": true, 971 | "current": true, 972 | "hideEmpty": true, 973 | "hideZero": true, 974 | "max": true, 975 | "min": true, 976 | "rightSide": false, 977 | "show": true, 978 | "sideWidth": 70, 979 | "total": false, 980 | "values": true 981 | }, 982 | "lines": true, 983 | "linewidth": 1, 984 | "links": [], 985 | "nullPointMode": "connected", 986 | "options": { 987 | "alertThreshold": true 988 | }, 989 | "percentage": false, 990 | "pluginVersion": "7.5.15", 991 | "pointradius": 5, 992 | "points": false, 993 | "renderer": "flot", 994 | "seriesOverrides": [], 995 | "spaceLength": 10, 996 | "stack": false, 997 | "steppedLine": false, 998 | "targets": [ 999 | { 1000 | "exemplar": true, 1001 | "expr": "jvm_memory_bytes_used{area=\"heap\", namespace=\"$namespace\", pod=~\"keycloak-.+\"}", 1002 | "format": "time_series", 1003 | "interval": "", 1004 | "intervalFactor": 1, 1005 | "legendFormat": "JVM Heap for {{pod}}", 1006 | "refId": "E" 1007 | }, 1008 | { 1009 | "exemplar": true, 1010 | "expr": "jvm_memory_bytes_max{area=\"heap\",namespace=\"$namespace\", pod=~\"keycloak-.+\"}", 1011 | "format": "time_series", 1012 | "interval": "", 1013 | "intervalFactor": 1, 1014 | "legendFormat": "Max heap for {{pod}}", 1015 | "refId": "B" 1016 | } 1017 | ], 1018 | "thresholds": [], 1019 | "timeFrom": null, 1020 | "timeRegions": [], 1021 | "timeShift": null, 1022 | "title": "JVM Memory Usage", 1023 | "tooltip": { 1024 | "shared": true, 1025 | "sort": 0, 1026 | "value_type": "individual" 1027 | }, 1028 | "type": "graph", 1029 | "xaxis": { 1030 | "buckets": null, 1031 | "mode": "time", 1032 | "name": null, 1033 | "show": true, 1034 | "values": [] 1035 | }, 1036 | "yaxes": [ 1037 | { 1038 | "decimals": null, 1039 | "format": "bytes", 1040 | "label": "", 1041 | "logBase": 1, 1042 | "max": null, 1043 | "min": null, 1044 | "show": true 1045 | }, 1046 | { 1047 | "format": "short", 1048 | "label": null, 1049 | "logBase": 1, 1050 | "max": null, 1051 | "min": null, 1052 | "show": true 1053 | } 1054 | ], 1055 | "yaxis": { 1056 | "align": false, 1057 | "alignLevel": null 1058 | } 1059 | }, 1060 | { 1061 | "aliasColors": {}, 1062 | "bars": false, 1063 | "dashLength": 10, 1064 | "dashes": false, 1065 | "datasource": "${DS_PROMETHEUS}", 1066 | "fieldConfig": { 1067 | "defaults": {}, 1068 | "overrides": [] 1069 | }, 1070 | "fill": 1, 1071 | "fillGradient": 0, 1072 | "gridPos": { 1073 | "h": 12, 1074 | "w": 12, 1075 | "x": 12, 1076 | "y": 41 1077 | }, 1078 | "hiddenSeries": false, 1079 | "id": 31, 1080 | "legend": { 1081 | "alignAsTable": false, 1082 | "avg": false, 1083 | "current": false, 1084 | "max": false, 1085 | "min": false, 1086 | "show": true, 1087 | "total": false, 1088 | "values": false 1089 | }, 1090 | "lines": true, 1091 | "linewidth": 1, 1092 | "links": [], 1093 | "nullPointMode": "null", 1094 | "options": { 1095 | "alertThreshold": true 1096 | }, 1097 | "percentage": false, 1098 | "pluginVersion": "7.5.15", 1099 | "pointradius": 2, 1100 | "points": false, 1101 | "renderer": "flot", 1102 | "seriesOverrides": [], 1103 | "spaceLength": 10, 1104 | "stack": false, 1105 | "steppedLine": false, 1106 | "targets": [ 1107 | { 1108 | "exemplar": true, 1109 | "expr": "kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"cpu\", pod=~\"keycloak-.+\"}", 1110 | "format": "time_series", 1111 | "interval": "", 1112 | "intervalFactor": 1, 1113 | "legendFormat": "CPU Limit for {{pod}}", 1114 | "refId": "A" 1115 | }, 1116 | { 1117 | "exemplar": true, 1118 | "expr": "kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\", pod=~\"keycloak-.+\"}", 1119 | "format": "time_series", 1120 | "interval": "", 1121 | "intervalFactor": 1, 1122 | "legendFormat": "CPU Request for {{pod}}", 1123 | "refId": "C" 1124 | }, 1125 | { 1126 | "exemplar": true, 1127 | "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", container=\"keycloak\", pod=~\"keycloak-.+\"} ", 1128 | "format": "time_series", 1129 | "interval": "", 1130 | "intervalFactor": 1, 1131 | "legendFormat": "CPU Load for {{pod}}", 1132 | "refId": "B" 1133 | } 1134 | ], 1135 | "thresholds": [], 1136 | "timeFrom": null, 1137 | "timeRegions": [], 1138 | "timeShift": null, 1139 | "title": "CPU Load", 1140 | "tooltip": { 1141 | "shared": true, 1142 | "sort": 0, 1143 | "value_type": "individual" 1144 | }, 1145 | "type": "graph", 1146 | "xaxis": { 1147 | "buckets": null, 1148 | "mode": "time", 1149 | "name": null, 1150 | "show": true, 1151 | "values": [] 1152 | }, 1153 | "yaxes": [ 1154 | { 1155 | "format": "short", 1156 | "label": null, 1157 | "logBase": 1, 1158 | "max": null, 1159 | "min": null, 1160 | "show": true 1161 | }, 1162 | { 1163 | "format": "short", 1164 | "label": null, 1165 | "logBase": 1, 1166 | "max": null, 1167 | "min": null, 1168 | "show": true 1169 | } 1170 | ], 1171 | "yaxis": { 1172 | "align": false, 1173 | "alignLevel": null 1174 | } 1175 | }, 1176 | { 1177 | "aliasColors": {}, 1178 | "bars": false, 1179 | "cacheTimeout": null, 1180 | "dashLength": 10, 1181 | "dashes": false, 1182 | "datasource": "${DS_PROMETHEUS}", 1183 | "fieldConfig": { 1184 | "defaults": {}, 1185 | "overrides": [] 1186 | }, 1187 | "fill": 1, 1188 | "fillGradient": 0, 1189 | "gridPos": { 1190 | "h": 11, 1191 | "w": 12, 1192 | "x": 0, 1193 | "y": 53 1194 | }, 1195 | "hiddenSeries": false, 1196 | "id": 38, 1197 | "legend": { 1198 | "alignAsTable": true, 1199 | "avg": false, 1200 | "current": false, 1201 | "max": true, 1202 | "min": true, 1203 | "show": true, 1204 | "total": false, 1205 | "values": true 1206 | }, 1207 | "lines": true, 1208 | "linewidth": 1, 1209 | "links": [], 1210 | "nullPointMode": "null as zero", 1211 | "options": { 1212 | "alertThreshold": true 1213 | }, 1214 | "percentage": false, 1215 | "pluginVersion": "7.5.15", 1216 | "pointradius": 2, 1217 | "points": false, 1218 | "renderer": "flot", 1219 | "seriesOverrides": [], 1220 | "spaceLength": 10, 1221 | "stack": false, 1222 | "steppedLine": false, 1223 | "targets": [ 1224 | { 1225 | "exemplar": true, 1226 | "expr": "rate(jvm_gc_collection_seconds_sum{namespace=\"$namespace\",pod=~\"keycloak-.+\"}[1m])", 1227 | "format": "time_series", 1228 | "interval": "", 1229 | "intervalFactor": 5, 1230 | "legendFormat": "{{gc}} {{pod}}", 1231 | "refId": "A" 1232 | } 1233 | ], 1234 | "thresholds": [], 1235 | "timeFrom": null, 1236 | "timeRegions": [], 1237 | "timeShift": null, 1238 | "title": "Total GC Time [1m]", 1239 | "tooltip": { 1240 | "shared": true, 1241 | "sort": 0, 1242 | "value_type": "individual" 1243 | }, 1244 | "type": "graph", 1245 | "xaxis": { 1246 | "buckets": null, 1247 | "mode": "time", 1248 | "name": null, 1249 | "show": true, 1250 | "values": [] 1251 | }, 1252 | "yaxes": [ 1253 | { 1254 | "format": "s", 1255 | "label": null, 1256 | "logBase": 1, 1257 | "max": null, 1258 | "min": null, 1259 | "show": true 1260 | }, 1261 | { 1262 | "format": "short", 1263 | "label": null, 1264 | "logBase": 1, 1265 | "max": null, 1266 | "min": null, 1267 | "show": true 1268 | } 1269 | ], 1270 | "yaxis": { 1271 | "align": false, 1272 | "alignLevel": null 1273 | } 1274 | }, 1275 | { 1276 | "aliasColors": {}, 1277 | "bars": false, 1278 | "cacheTimeout": null, 1279 | "dashLength": 10, 1280 | "dashes": false, 1281 | "datasource": "${DS_PROMETHEUS}", 1282 | "fieldConfig": { 1283 | "defaults": {}, 1284 | "overrides": [] 1285 | }, 1286 | "fill": 1, 1287 | "fillGradient": 0, 1288 | "gridPos": { 1289 | "h": 11, 1290 | "w": 12, 1291 | "x": 12, 1292 | "y": 53 1293 | }, 1294 | "hiddenSeries": false, 1295 | "id": 37, 1296 | "legend": { 1297 | "alignAsTable": true, 1298 | "avg": false, 1299 | "current": true, 1300 | "max": true, 1301 | "min": true, 1302 | "show": true, 1303 | "total": false, 1304 | "values": true 1305 | }, 1306 | "lines": true, 1307 | "linewidth": 1, 1308 | "links": [], 1309 | "nullPointMode": "null as zero", 1310 | "options": { 1311 | "alertThreshold": true 1312 | }, 1313 | "percentage": false, 1314 | "pluginVersion": "7.5.15", 1315 | "pointradius": 2, 1316 | "points": false, 1317 | "renderer": "flot", 1318 | "seriesOverrides": [], 1319 | "spaceLength": 10, 1320 | "stack": false, 1321 | "steppedLine": false, 1322 | "targets": [ 1323 | { 1324 | "exemplar": true, 1325 | "expr": "jvm_gc_collection_seconds_count{namespace=\"$namespace\",pod=~\"keycloak-.+\"}", 1326 | "format": "time_series", 1327 | "interval": "", 1328 | "intervalFactor": 1, 1329 | "legendFormat": "{{gc}} {{pod}}", 1330 | "refId": "A" 1331 | } 1332 | ], 1333 | "thresholds": [], 1334 | "timeFrom": null, 1335 | "timeRegions": [], 1336 | "timeShift": null, 1337 | "title": "Number of GC Collections [30m]", 1338 | "tooltip": { 1339 | "shared": true, 1340 | "sort": 0, 1341 | "value_type": "individual" 1342 | }, 1343 | "type": "graph", 1344 | "xaxis": { 1345 | "buckets": null, 1346 | "mode": "time", 1347 | "name": null, 1348 | "show": true, 1349 | "values": [] 1350 | }, 1351 | "yaxes": [ 1352 | { 1353 | "format": "short", 1354 | "label": null, 1355 | "logBase": 1, 1356 | "max": null, 1357 | "min": null, 1358 | "show": true 1359 | }, 1360 | { 1361 | "format": "short", 1362 | "label": null, 1363 | "logBase": 1, 1364 | "max": null, 1365 | "min": null, 1366 | "show": true 1367 | } 1368 | ], 1369 | "yaxis": { 1370 | "align": false, 1371 | "alignLevel": null 1372 | } 1373 | }, 1374 | { 1375 | "aliasColors": {}, 1376 | "bars": false, 1377 | "cacheTimeout": null, 1378 | "dashLength": 10, 1379 | "dashes": false, 1380 | "datasource": "${DS_PROMETHEUS}", 1381 | "fieldConfig": { 1382 | "defaults": {}, 1383 | "overrides": [] 1384 | }, 1385 | "fill": 1, 1386 | "fillGradient": 0, 1387 | "gridPos": { 1388 | "h": 15, 1389 | "w": 12, 1390 | "x": 0, 1391 | "y": 64 1392 | }, 1393 | "hiddenSeries": false, 1394 | "id": 50, 1395 | "legend": { 1396 | "alignAsTable": true, 1397 | "avg": false, 1398 | "current": true, 1399 | "max": true, 1400 | "min": true, 1401 | "show": true, 1402 | "total": false, 1403 | "values": true 1404 | }, 1405 | "lines": true, 1406 | "linewidth": 1, 1407 | "links": [], 1408 | "nullPointMode": "null", 1409 | "options": { 1410 | "alertThreshold": true 1411 | }, 1412 | "percentage": false, 1413 | "pluginVersion": "7.5.15", 1414 | "pointradius": 2, 1415 | "points": false, 1416 | "renderer": "flot", 1417 | "seriesOverrides": [], 1418 | "spaceLength": 10, 1419 | "stack": false, 1420 | "steppedLine": false, 1421 | "targets": [ 1422 | { 1423 | "exemplar": true, 1424 | "expr": "jvm_threads_current{namespace=\"$namespace\", pod=~\"keycloak-.+\"}", 1425 | "format": "time_series", 1426 | "interval": "", 1427 | "intervalFactor": 1, 1428 | "legendFormat": "JVM current threads {{pod}}", 1429 | "refId": "A" 1430 | }, 1431 | { 1432 | "exemplar": true, 1433 | "expr": "jvm_threads_daemon{namespace=\"$namespace\", pod=~\"keycloak-.+\"}", 1434 | "format": "time_series", 1435 | "interval": "", 1436 | "intervalFactor": 1, 1437 | "legendFormat": "JVM daemon threads {{pod}}", 1438 | "refId": "B" 1439 | }, 1440 | { 1441 | "exemplar": true, 1442 | "expr": "jvm_threads_deadlocked{namespace=\"$namespace\", pod=~\"keycloak-.+\"}", 1443 | "format": "time_series", 1444 | "interval": "", 1445 | "intervalFactor": 1, 1446 | "legendFormat": "JVM deadlocked threads {{pod}}", 1447 | "refId": "C" 1448 | } 1449 | ], 1450 | "thresholds": [], 1451 | "timeFrom": null, 1452 | "timeRegions": [], 1453 | "timeShift": null, 1454 | "title": "IO Threads", 1455 | "tooltip": { 1456 | "shared": true, 1457 | "sort": 0, 1458 | "value_type": "individual" 1459 | }, 1460 | "type": "graph", 1461 | "xaxis": { 1462 | "buckets": null, 1463 | "mode": "time", 1464 | "name": null, 1465 | "show": true, 1466 | "values": [] 1467 | }, 1468 | "yaxes": [ 1469 | { 1470 | "format": "short", 1471 | "label": null, 1472 | "logBase": 1, 1473 | "max": null, 1474 | "min": null, 1475 | "show": true 1476 | }, 1477 | { 1478 | "format": "short", 1479 | "label": null, 1480 | "logBase": 1, 1481 | "max": null, 1482 | "min": null, 1483 | "show": true 1484 | } 1485 | ], 1486 | "yaxis": { 1487 | "align": false, 1488 | "alignLevel": null 1489 | } 1490 | }, 1491 | { 1492 | "aliasColors": {}, 1493 | "bars": false, 1494 | "dashLength": 10, 1495 | "dashes": false, 1496 | "datasource": "${DS_PROMETHEUS}", 1497 | "fieldConfig": { 1498 | "defaults": {}, 1499 | "overrides": [] 1500 | }, 1501 | "fill": 1, 1502 | "fillGradient": 0, 1503 | "gridPos": { 1504 | "h": 15, 1505 | "w": 12, 1506 | "x": 12, 1507 | "y": 64 1508 | }, 1509 | "hiddenSeries": false, 1510 | "hideTimeOverride": false, 1511 | "id": 12, 1512 | "legend": { 1513 | "alignAsTable": true, 1514 | "avg": true, 1515 | "current": true, 1516 | "hideEmpty": true, 1517 | "hideZero": true, 1518 | "max": true, 1519 | "min": true, 1520 | "rightSide": false, 1521 | "show": true, 1522 | "sideWidth": 70, 1523 | "total": false, 1524 | "values": true 1525 | }, 1526 | "lines": true, 1527 | "linewidth": 1, 1528 | "links": [], 1529 | "nullPointMode": "connected", 1530 | "options": { 1531 | "alertThreshold": true 1532 | }, 1533 | "percentage": false, 1534 | "pluginVersion": "7.5.15", 1535 | "pointradius": 5, 1536 | "points": false, 1537 | "renderer": "flot", 1538 | "seriesOverrides": [], 1539 | "spaceLength": 10, 1540 | "stack": false, 1541 | "steppedLine": false, 1542 | "targets": [ 1543 | { 1544 | "exemplar": true, 1545 | "expr": "kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\", pod=~\"keycloak-.+\"}", 1546 | "format": "time_series", 1547 | "hide": false, 1548 | "instant": false, 1549 | "interval": "", 1550 | "intervalFactor": 1, 1551 | "legendFormat": "Requests {{pod}}", 1552 | "refId": "A" 1553 | }, 1554 | { 1555 | "exemplar": true, 1556 | "expr": "kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\", pod=~\"keycloak-.+\"}", 1557 | "format": "time_series", 1558 | "hide": false, 1559 | "interval": "", 1560 | "intervalFactor": 1, 1561 | "legendFormat": "Limits {{pod}}", 1562 | "refId": "C" 1563 | }, 1564 | { 1565 | "exemplar": true, 1566 | "expr": "container_memory_rss{namespace=\"$namespace\", container=\"keycloak\", pod=~\"keycloak-.+\"}", 1567 | "format": "time_series", 1568 | "hide": false, 1569 | "instant": false, 1570 | "interval": "", 1571 | "intervalFactor": 1, 1572 | "legendFormat": "RSS for {{pod}}", 1573 | "refId": "B" 1574 | } 1575 | ], 1576 | "thresholds": [], 1577 | "timeFrom": null, 1578 | "timeRegions": [], 1579 | "timeShift": null, 1580 | "title": "Pod Memory Usage", 1581 | "tooltip": { 1582 | "shared": true, 1583 | "sort": 0, 1584 | "value_type": "individual" 1585 | }, 1586 | "type": "graph", 1587 | "xaxis": { 1588 | "buckets": null, 1589 | "mode": "time", 1590 | "name": null, 1591 | "show": true, 1592 | "values": [] 1593 | }, 1594 | "yaxes": [ 1595 | { 1596 | "decimals": null, 1597 | "format": "bytes", 1598 | "label": "", 1599 | "logBase": 1, 1600 | "max": null, 1601 | "min": "0", 1602 | "show": true 1603 | }, 1604 | { 1605 | "format": "short", 1606 | "label": null, 1607 | "logBase": 1, 1608 | "max": null, 1609 | "min": null, 1610 | "show": true 1611 | } 1612 | ], 1613 | "yaxis": { 1614 | "align": false, 1615 | "alignLevel": null 1616 | } 1617 | } 1618 | ], 1619 | "refresh": "10s", 1620 | "schemaVersion": 27, 1621 | "style": "dark", 1622 | "tags": [ 1623 | "Keycloak" 1624 | ], 1625 | "templating": { 1626 | "list": [ 1627 | { 1628 | "allValue": null, 1629 | "current": {}, 1630 | "datasource": "${DS_PROMETHEUS}", 1631 | "definition": "label_values(keycloak_request_duration_count,namespace)", 1632 | "description": null, 1633 | "error": null, 1634 | "hide": 0, 1635 | "includeAll": true, 1636 | "label": "Namespace", 1637 | "multi": false, 1638 | "name": "namespace", 1639 | "options": [], 1640 | "query": { 1641 | "query": "label_values(keycloak_request_duration_count,namespace)", 1642 | "refId": "Prometheus-namespace-Variable-Query" 1643 | }, 1644 | "refresh": 1, 1645 | "regex": "", 1646 | "skipUrlSync": false, 1647 | "sort": 0, 1648 | "tagValuesQuery": "", 1649 | "tags": [], 1650 | "tagsQuery": "", 1651 | "type": "query", 1652 | "useTags": false 1653 | } 1654 | ] 1655 | }, 1656 | "time": { 1657 | "from": "now-6h", 1658 | "to": "now" 1659 | }, 1660 | "timepicker": { 1661 | "hidden": false, 1662 | "refresh_intervals": [ 1663 | "5s", 1664 | "10s", 1665 | "30s", 1666 | "1m", 1667 | "5m", 1668 | "15m", 1669 | "30m", 1670 | "1h", 1671 | "2h", 1672 | "1d" 1673 | ], 1674 | "time_options": [ 1675 | "5m", 1676 | "15m", 1677 | "1h", 1678 | "6h", 1679 | "12h", 1680 | "24h", 1681 | "2d", 1682 | "7d", 1683 | "30d" 1684 | ] 1685 | }, 1686 | "timezone": "", 1687 | "title": "SSO Metrics", 1688 | "uid": "keycloak-spi", 1689 | "version": 1 1690 | } -------------------------------------------------------------------------------- /kustomize/base/grafana/dashboards/dashboard-artemis.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_PROMETHEUS", 5 | "label": "Prometheus", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "gauge", 16 | "name": "Gauge", 17 | "version": "" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "7.5.16" 24 | }, 25 | { 26 | "type": "panel", 27 | "id": "graph", 28 | "name": "Graph", 29 | "version": "" 30 | }, 31 | { 32 | "type": "datasource", 33 | "id": "prometheus", 34 | "name": "Prometheus", 35 | "version": "1.0.0" 36 | }, 37 | { 38 | "type": "panel", 39 | "id": "singlestat", 40 | "name": "Singlestat", 41 | "version": "" 42 | }, 43 | { 44 | "type": "panel", 45 | "id": "stat", 46 | "name": "Stat", 47 | "version": "" 48 | } 49 | ], 50 | "annotations": { 51 | "list": [ 52 | { 53 | "builtIn": 1, 54 | "datasource": "${DS_PROMETHEUS}", 55 | "enable": true, 56 | "hide": true, 57 | "iconColor": "rgba(0, 211, 255, 1)", 58 | "name": "Annotations & Alerts", 59 | "type": "dashboard" 60 | } 61 | ] 62 | }, 63 | "description": "Dashboard for monitoring Apache Artemis", 64 | "editable": true, 65 | "gnetId": null, 66 | "graphTooltip": 0, 67 | "id": null, 68 | "iteration": 1662125982837, 69 | "links": [], 70 | "panels": [ 71 | { 72 | "cacheTimeout": null, 73 | "colorBackground": false, 74 | "colorPostfix": false, 75 | "colorValue": true, 76 | "colors": [ 77 | "#299c46", 78 | "#bf1b00", 79 | "#d44a3a" 80 | ], 81 | "datasource": "${DS_PROMETHEUS}", 82 | "fieldConfig": { 83 | "defaults": {}, 84 | "overrides": [] 85 | }, 86 | "format": "none", 87 | "gauge": { 88 | "maxValue": 100, 89 | "minValue": 0, 90 | "show": false, 91 | "thresholdLabels": false, 92 | "thresholdMarkers": true 93 | }, 94 | "gridPos": { 95 | "h": 2, 96 | "w": 24, 97 | "x": 0, 98 | "y": 0 99 | }, 100 | "id": 17, 101 | "interval": null, 102 | "links": [], 103 | "mappingType": 1, 104 | "mappingTypes": [ 105 | { 106 | "name": "value to text", 107 | "value": 1 108 | }, 109 | { 110 | "name": "range to text", 111 | "value": 2 112 | } 113 | ], 114 | "maxDataPoints": 100, 115 | "nullPointMode": "connected", 116 | "nullText": null, 117 | "postfix": "", 118 | "postfixFontSize": "50%", 119 | "prefix": "", 120 | "prefixFontSize": "50%", 121 | "rangeMaps": [ 122 | { 123 | "from": "null", 124 | "text": "N/A", 125 | "to": "null" 126 | } 127 | ], 128 | "sparkline": { 129 | "fillColor": "rgba(31, 118, 189, 0.18)", 130 | "full": false, 131 | "lineColor": "rgb(31, 120, 193)", 132 | "show": false 133 | }, 134 | "tableColumn": "", 135 | "targets": [ 136 | { 137 | "expr": "up{pod=\"$instance\", namespace=\"$namespace\"}", 138 | "format": "time_series", 139 | "instant": true, 140 | "intervalFactor": 1, 141 | "legendFormat": "", 142 | "refId": "A" 143 | } 144 | ], 145 | "thresholds": "True,False", 146 | "title": "Started", 147 | "type": "singlestat", 148 | "valueFontSize": "80%", 149 | "valueMaps": [ 150 | { 151 | "op": "=", 152 | "text": "True", 153 | "value": "1" 154 | }, 155 | { 156 | "op": "=", 157 | "text": "False", 158 | "value": "0" 159 | } 160 | ], 161 | "valueName": "current" 162 | }, 163 | { 164 | "datasource": "${DS_PROMETHEUS}", 165 | "fieldConfig": { 166 | "defaults": { 167 | "color": { 168 | "mode": "thresholds" 169 | }, 170 | "decimals": 0, 171 | "mappings": [], 172 | "max": 100, 173 | "min": 0, 174 | "thresholds": { 175 | "mode": "absolute", 176 | "steps": [ 177 | { 178 | "color": "green", 179 | "value": null 180 | }, 181 | { 182 | "color": "red", 183 | "value": 80 184 | } 185 | ] 186 | }, 187 | "unit": "percent" 188 | }, 189 | "overrides": [] 190 | }, 191 | "gridPos": { 192 | "h": 6, 193 | "w": 5, 194 | "x": 0, 195 | "y": 2 196 | }, 197 | "id": 19, 198 | "options": { 199 | "orientation": "auto", 200 | "reduceOptions": { 201 | "calcs": [ 202 | "lastNotNull" 203 | ], 204 | "fields": "", 205 | "values": false 206 | }, 207 | "showThresholdLabels": false, 208 | "showThresholdMarkers": true, 209 | "text": {} 210 | }, 211 | "pluginVersion": "7.5.16", 212 | "targets": [ 213 | { 214 | "expr": "sum(avg_over_time(jvm_memory_used_bytes{area=\"heap\", pod=\"$instance\", namespace=\"$namespace\"}[1m]))*100/sum(avg_over_time(jvm_memory_max_bytes{area=\"heap\", pod=\"$instance\", namespace=\"$namespace\"}[5m]))", 215 | "refId": "A" 216 | } 217 | ], 218 | "timeFrom": null, 219 | "timeShift": null, 220 | "title": "Heap Usage", 221 | "type": "gauge" 222 | }, 223 | { 224 | "datasource": "${DS_PROMETHEUS}", 225 | "fieldConfig": { 226 | "defaults": { 227 | "color": { 228 | "mode": "thresholds" 229 | }, 230 | "decimals": 0, 231 | "mappings": [], 232 | "max": 100, 233 | "min": 0, 234 | "thresholds": { 235 | "mode": "absolute", 236 | "steps": [ 237 | { 238 | "color": "green", 239 | "value": null 240 | }, 241 | { 242 | "color": "yellow", 243 | "value": 80 244 | } 245 | ] 246 | }, 247 | "unit": "percent" 248 | }, 249 | "overrides": [] 250 | }, 251 | "gridPos": { 252 | "h": 6, 253 | "w": 5, 254 | "x": 5, 255 | "y": 2 256 | }, 257 | "id": 1, 258 | "options": { 259 | "orientation": "auto", 260 | "reduceOptions": { 261 | "calcs": [ 262 | "lastNotNull" 263 | ], 264 | "fields": "", 265 | "values": false 266 | }, 267 | "showThresholdLabels": false, 268 | "showThresholdMarkers": true, 269 | "text": {} 270 | }, 271 | "pluginVersion": "7.5.16", 272 | "targets": [ 273 | { 274 | "exemplar": true, 275 | "expr": "sum by (broker) ( artemis_address_memory_usage_percentage{pod=\"$instance\", namespace=\"$namespace\"} )", 276 | "hide": false, 277 | "interval": "", 278 | "legendFormat": "{{ broker }}", 279 | "refId": "A" 280 | } 281 | ], 282 | "timeFrom": null, 283 | "timeShift": null, 284 | "title": "Global Max Usage", 285 | "type": "gauge" 286 | }, 287 | { 288 | "cacheTimeout": null, 289 | "datasource": "${DS_PROMETHEUS}", 290 | "fieldConfig": { 291 | "defaults": { 292 | "color": { 293 | "mode": "thresholds" 294 | }, 295 | "mappings": [ 296 | { 297 | "id": 0, 298 | "op": "=", 299 | "text": "N/A", 300 | "type": 1, 301 | "value": "null" 302 | } 303 | ], 304 | "thresholds": { 305 | "mode": "absolute", 306 | "steps": [ 307 | { 308 | "color": "green", 309 | "value": null 310 | } 311 | ] 312 | }, 313 | "unit": "short" 314 | }, 315 | "overrides": [] 316 | }, 317 | "gridPos": { 318 | "h": 6, 319 | "w": 4, 320 | "x": 10, 321 | "y": 2 322 | }, 323 | "id": 23, 324 | "interval": null, 325 | "links": [], 326 | "maxDataPoints": 100, 327 | "options": { 328 | "colorMode": "value", 329 | "graphMode": "none", 330 | "justifyMode": "auto", 331 | "orientation": "horizontal", 332 | "reduceOptions": { 333 | "calcs": [ 334 | "lastNotNull" 335 | ], 336 | "fields": "", 337 | "values": false 338 | }, 339 | "text": {}, 340 | "textMode": "auto" 341 | }, 342 | "pluginVersion": "7.5.16", 343 | "targets": [ 344 | { 345 | "exemplar": true, 346 | "expr": "sum by (broker)(artemis_message_count{pod='$instance', namespace=\"$namespace\"})", 347 | "interval": "", 348 | "legendFormat": "", 349 | "refId": "A" 350 | } 351 | ], 352 | "timeFrom": null, 353 | "timeShift": null, 354 | "title": "Message Count", 355 | "type": "stat" 356 | }, 357 | { 358 | "datasource": "${DS_PROMETHEUS}", 359 | "fieldConfig": { 360 | "defaults": { 361 | "color": { 362 | "mode": "thresholds" 363 | }, 364 | "mappings": [], 365 | "noValue": "0", 366 | "thresholds": { 367 | "mode": "absolute", 368 | "steps": [ 369 | { 370 | "color": "purple", 371 | "value": null 372 | } 373 | ] 374 | }, 375 | "unit": "ops" 376 | }, 377 | "overrides": [] 378 | }, 379 | "gridPos": { 380 | "h": 6, 381 | "w": 5, 382 | "x": 14, 383 | "y": 2 384 | }, 385 | "id": 36, 386 | "options": { 387 | "colorMode": "value", 388 | "graphMode": "area", 389 | "justifyMode": "auto", 390 | "orientation": "auto", 391 | "reduceOptions": { 392 | "calcs": [ 393 | "mean" 394 | ], 395 | "fields": "", 396 | "values": false 397 | }, 398 | "text": {}, 399 | "textMode": "auto" 400 | }, 401 | "pluginVersion": "7.5.16", 402 | "targets": [ 403 | { 404 | "exemplar": true, 405 | "expr": "sum by (broker) ((rate(artemis_messages_added{pod=\"$instance\", namespace=\"$namespace\"}[1m])) + (rate(artemis_messages_acknowledged{pod=\"$instance\", namespace=\"$namespace\"}[1m])))", 406 | "instant": true, 407 | "interval": "", 408 | "legendFormat": "", 409 | "refId": "A" 410 | } 411 | ], 412 | "timeFrom": null, 413 | "timeShift": null, 414 | "title": "Throughput", 415 | "type": "stat" 416 | }, 417 | { 418 | "datasource": "${DS_PROMETHEUS}", 419 | "fieldConfig": { 420 | "defaults": { 421 | "color": { 422 | "mode": "thresholds" 423 | }, 424 | "decimals": 0, 425 | "mappings": [], 426 | "max": 1, 427 | "min": 0, 428 | "thresholds": { 429 | "mode": "absolute", 430 | "steps": [ 431 | { 432 | "color": "green", 433 | "value": null 434 | }, 435 | { 436 | "color": "#EAB839", 437 | "value": 0.6 438 | }, 439 | { 440 | "color": "red", 441 | "value": 0.8 442 | } 443 | ] 444 | }, 445 | "unit": "percentunit" 446 | }, 447 | "overrides": [] 448 | }, 449 | "gridPos": { 450 | "h": 6, 451 | "w": 5, 452 | "x": 19, 453 | "y": 2 454 | }, 455 | "id": 49, 456 | "options": { 457 | "orientation": "auto", 458 | "reduceOptions": { 459 | "calcs": [ 460 | "lastNotNull" 461 | ], 462 | "fields": "", 463 | "values": false 464 | }, 465 | "showThresholdLabels": false, 466 | "showThresholdMarkers": true, 467 | "text": {} 468 | }, 469 | "pluginVersion": "7.5.16", 470 | "targets": [ 471 | { 472 | "exemplar": true, 473 | "expr": "sum by (broker) ( artemis_disk_store_usage{pod=\"$instance\", namespace=\"$namespace\"} )", 474 | "hide": false, 475 | "interval": "", 476 | "legendFormat": "{{ broker }}", 477 | "refId": "A" 478 | } 479 | ], 480 | "timeFrom": null, 481 | "timeShift": null, 482 | "title": "Disk usage", 483 | "type": "gauge" 484 | }, 485 | { 486 | "collapsed": false, 487 | "datasource": null, 488 | "gridPos": { 489 | "h": 1, 490 | "w": 24, 491 | "x": 0, 492 | "y": 8 493 | }, 494 | "id": 47, 495 | "panels": [], 496 | "title": "Performance", 497 | "type": "row" 498 | }, 499 | { 500 | "aliasColors": {}, 501 | "bars": false, 502 | "dashLength": 10, 503 | "dashes": false, 504 | "datasource": "${DS_PROMETHEUS}", 505 | "fieldConfig": { 506 | "defaults": { 507 | "links": [] 508 | }, 509 | "overrides": [] 510 | }, 511 | "fill": 1, 512 | "fillGradient": 0, 513 | "gridPos": { 514 | "h": 7, 515 | "w": 24, 516 | "x": 0, 517 | "y": 9 518 | }, 519 | "hiddenSeries": false, 520 | "id": 51, 521 | "legend": { 522 | "alignAsTable": true, 523 | "avg": false, 524 | "current": true, 525 | "max": true, 526 | "min": false, 527 | "rightSide": true, 528 | "show": true, 529 | "sideWidth": 300, 530 | "total": false, 531 | "values": true 532 | }, 533 | "lines": true, 534 | "linewidth": 1, 535 | "links": [], 536 | "nullPointMode": "null", 537 | "options": { 538 | "alertThreshold": true 539 | }, 540 | "percentage": false, 541 | "pluginVersion": "7.5.16", 542 | "pointradius": 5, 543 | "points": false, 544 | "renderer": "flot", 545 | "seriesOverrides": [], 546 | "spaceLength": 10, 547 | "stack": false, 548 | "steppedLine": false, 549 | "targets": [ 550 | { 551 | "exemplar": true, 552 | "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\", pod=\"$instance\", namespace=\"$namespace\"})", 553 | "hide": false, 554 | "interval": "", 555 | "legendFormat": "Limit", 556 | "refId": "CPU limit" 557 | }, 558 | { 559 | "exemplar": true, 560 | "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", pod=\"$instance\", namespace=\"$namespace\"})", 561 | "hide": false, 562 | "interval": "", 563 | "legendFormat": "Request", 564 | "refId": "CPU request" 565 | }, 566 | { 567 | "exemplar": true, 568 | "expr": "sum(pod:container_cpu_usage:sum{pod=\"$instance\", namespace=\"$namespace\"})", 569 | "hide": false, 570 | "interval": "", 571 | "legendFormat": "Usage", 572 | "refId": "CPU Usage" 573 | } 574 | ], 575 | "thresholds": [], 576 | "timeFrom": null, 577 | "timeRegions": [], 578 | "timeShift": null, 579 | "title": "CPU Usage", 580 | "tooltip": { 581 | "shared": true, 582 | "sort": 0, 583 | "value_type": "individual" 584 | }, 585 | "type": "graph", 586 | "xaxis": { 587 | "buckets": null, 588 | "mode": "time", 589 | "name": null, 590 | "show": true, 591 | "values": [] 592 | }, 593 | "yaxes": [ 594 | { 595 | "format": "core", 596 | "label": null, 597 | "logBase": 1, 598 | "max": null, 599 | "min": "0", 600 | "show": true 601 | }, 602 | { 603 | "format": "core", 604 | "label": null, 605 | "logBase": 1, 606 | "max": null, 607 | "min": "0", 608 | "show": false 609 | } 610 | ], 611 | "yaxis": { 612 | "align": false, 613 | "alignLevel": null 614 | } 615 | }, 616 | { 617 | "aliasColors": {}, 618 | "bars": false, 619 | "dashLength": 10, 620 | "dashes": false, 621 | "datasource": "${DS_PROMETHEUS}", 622 | "fieldConfig": { 623 | "defaults": { 624 | "links": [] 625 | }, 626 | "overrides": [] 627 | }, 628 | "fill": 1, 629 | "fillGradient": 0, 630 | "gridPos": { 631 | "h": 7, 632 | "w": 24, 633 | "x": 0, 634 | "y": 16 635 | }, 636 | "hiddenSeries": false, 637 | "id": 7, 638 | "legend": { 639 | "alignAsTable": true, 640 | "avg": false, 641 | "current": true, 642 | "max": true, 643 | "min": false, 644 | "rightSide": true, 645 | "show": true, 646 | "sideWidth": 300, 647 | "total": false, 648 | "values": true 649 | }, 650 | "lines": true, 651 | "linewidth": 1, 652 | "links": [], 653 | "nullPointMode": "null", 654 | "options": { 655 | "alertThreshold": true 656 | }, 657 | "percentage": false, 658 | "pluginVersion": "7.5.16", 659 | "pointradius": 5, 660 | "points": false, 661 | "renderer": "flot", 662 | "seriesOverrides": [], 663 | "spaceLength": 10, 664 | "stack": false, 665 | "steppedLine": false, 666 | "targets": [ 667 | { 668 | "exemplar": true, 669 | "expr": "sum(jvm_memory_max_bytes{area=\"heap\", pod=\"$instance\", namespace=\"$namespace\"})", 670 | "hide": false, 671 | "interval": "", 672 | "legendFormat": "Heap Max", 673 | "refId": "Heap Max" 674 | }, 675 | { 676 | "exemplar": true, 677 | "expr": "sum(jvm_memory_used_bytes{area=\"heap\", pod=\"$instance\", namespace=\"$namespace\"})", 678 | "hide": false, 679 | "interval": "", 680 | "legendFormat": "Heap Usage", 681 | "refId": "Heap Usage" 682 | }, 683 | { 684 | "exemplar": true, 685 | "expr": "sum by (broker) (artemis_address_memory_usage{pod=\"$instance\", namespace=\"$namespace\"})", 686 | "format": "time_series", 687 | "interval": "", 688 | "intervalFactor": 1, 689 | "legendFormat": "Global Max Usage", 690 | "refId": "Address" 691 | } 692 | ], 693 | "thresholds": [], 694 | "timeFrom": null, 695 | "timeRegions": [], 696 | "timeShift": null, 697 | "title": "Memory Usage", 698 | "tooltip": { 699 | "shared": true, 700 | "sort": 0, 701 | "value_type": "individual" 702 | }, 703 | "type": "graph", 704 | "xaxis": { 705 | "buckets": null, 706 | "mode": "time", 707 | "name": null, 708 | "show": true, 709 | "values": [] 710 | }, 711 | "yaxes": [ 712 | { 713 | "format": "bytes", 714 | "label": null, 715 | "logBase": 1, 716 | "max": null, 717 | "min": "0", 718 | "show": true 719 | }, 720 | { 721 | "format": "short", 722 | "label": null, 723 | "logBase": 1, 724 | "max": null, 725 | "min": "0", 726 | "show": true 727 | } 728 | ], 729 | "yaxis": { 730 | "align": false, 731 | "alignLevel": null 732 | } 733 | }, 734 | { 735 | "aliasColors": {}, 736 | "bars": false, 737 | "dashLength": 10, 738 | "dashes": false, 739 | "datasource": "${DS_PROMETHEUS}", 740 | "decimals": 0, 741 | "fieldConfig": { 742 | "defaults": { 743 | "links": [] 744 | }, 745 | "overrides": [] 746 | }, 747 | "fill": 1, 748 | "fillGradient": 0, 749 | "gridPos": { 750 | "h": 7, 751 | "w": 12, 752 | "x": 0, 753 | "y": 23 754 | }, 755 | "hiddenSeries": false, 756 | "id": 37, 757 | "legend": { 758 | "alignAsTable": true, 759 | "avg": true, 760 | "current": true, 761 | "hideEmpty": true, 762 | "hideZero": true, 763 | "max": true, 764 | "min": false, 765 | "rightSide": true, 766 | "show": false, 767 | "sideWidth": 300, 768 | "total": false, 769 | "values": true 770 | }, 771 | "lines": true, 772 | "linewidth": 1, 773 | "links": [], 774 | "nullPointMode": "null", 775 | "options": { 776 | "alertThreshold": true 777 | }, 778 | "percentage": false, 779 | "pluginVersion": "7.5.16", 780 | "pointradius": 5, 781 | "points": false, 782 | "renderer": "flot", 783 | "seriesOverrides": [], 784 | "spaceLength": 10, 785 | "stack": false, 786 | "steppedLine": false, 787 | "targets": [ 788 | { 789 | "exemplar": true, 790 | "expr": "sum by (broker) (rate(artemis_messages_added{pod=\"$instance\", namespace=\"$namespace\"}[1m]))", 791 | "format": "time_series", 792 | "hide": false, 793 | "interval": "", 794 | "intervalFactor": 1, 795 | "legendFormat": "Producing", 796 | "refId": "Messages Produced" 797 | }, 798 | { 799 | "exemplar": true, 800 | "expr": "sum by (broker) (rate(artemis_messages_acknowledged{pod=\"$instance\", namespace=\"$namespace\"}[1m]))", 801 | "hide": false, 802 | "instant": false, 803 | "interval": "", 804 | "legendFormat": "Consuming", 805 | "refId": "Messages Consumed" 806 | } 807 | ], 808 | "thresholds": [], 809 | "timeFrom": null, 810 | "timeRegions": [], 811 | "timeShift": null, 812 | "title": "Message Throughput", 813 | "tooltip": { 814 | "shared": true, 815 | "sort": 2, 816 | "value_type": "individual" 817 | }, 818 | "type": "graph", 819 | "xaxis": { 820 | "buckets": null, 821 | "mode": "time", 822 | "name": null, 823 | "show": true, 824 | "values": [] 825 | }, 826 | "yaxes": [ 827 | { 828 | "decimals": 0, 829 | "format": "ops", 830 | "label": "", 831 | "logBase": 1, 832 | "max": null, 833 | "min": "0", 834 | "show": true 835 | }, 836 | { 837 | "decimals": 0, 838 | "format": "short", 839 | "label": "", 840 | "logBase": 1, 841 | "max": null, 842 | "min": "0", 843 | "show": false 844 | } 845 | ], 846 | "yaxis": { 847 | "align": false, 848 | "alignLevel": null 849 | } 850 | }, 851 | { 852 | "aliasColors": {}, 853 | "bars": false, 854 | "dashLength": 10, 855 | "dashes": false, 856 | "datasource": "${DS_PROMETHEUS}", 857 | "decimals": 0, 858 | "fieldConfig": { 859 | "defaults": { 860 | "links": [] 861 | }, 862 | "overrides": [] 863 | }, 864 | "fill": 1, 865 | "fillGradient": 0, 866 | "gridPos": { 867 | "h": 7, 868 | "w": 12, 869 | "x": 12, 870 | "y": 23 871 | }, 872 | "hiddenSeries": false, 873 | "id": 9, 874 | "legend": { 875 | "alignAsTable": true, 876 | "avg": false, 877 | "current": true, 878 | "hideEmpty": true, 879 | "hideZero": true, 880 | "max": true, 881 | "min": false, 882 | "rightSide": true, 883 | "show": false, 884 | "sideWidth": 500, 885 | "total": false, 886 | "values": true 887 | }, 888 | "lines": true, 889 | "linewidth": 1, 890 | "links": [], 891 | "nullPointMode": "null", 892 | "options": { 893 | "alertThreshold": true 894 | }, 895 | "percentage": false, 896 | "pluginVersion": "7.5.16", 897 | "pointradius": 5, 898 | "points": false, 899 | "renderer": "flot", 900 | "seriesOverrides": [], 901 | "spaceLength": 10, 902 | "stack": false, 903 | "steppedLine": false, 904 | "targets": [ 905 | { 906 | "exemplar": true, 907 | "expr": "sum by (broker) ( artemis_connection_count{pod=\"$instance\", namespace=\"$namespace\"} )", 908 | "format": "time_series", 909 | "interval": "", 910 | "intervalFactor": 1, 911 | "legendFormat": "Connection Count", 912 | "refId": "A" 913 | }, 914 | { 915 | "exemplar": true, 916 | "expr": "sum by (address) ( topk (20, artemis_consumer_count{pod=\"$instance\", namespace=\"$namespace\"} ) )", 917 | "format": "time_series", 918 | "interval": "", 919 | "intervalFactor": 1, 920 | "legendFormat": "{{ address }}", 921 | "refId": "B" 922 | } 923 | ], 924 | "thresholds": [], 925 | "timeFrom": null, 926 | "timeRegions": [], 927 | "timeShift": null, 928 | "title": "Connections and Consumers", 929 | "tooltip": { 930 | "shared": true, 931 | "sort": 2, 932 | "value_type": "individual" 933 | }, 934 | "type": "graph", 935 | "xaxis": { 936 | "buckets": null, 937 | "mode": "time", 938 | "name": null, 939 | "show": true, 940 | "values": [] 941 | }, 942 | "yaxes": [ 943 | { 944 | "decimals": 0, 945 | "format": "none", 946 | "label": null, 947 | "logBase": 1, 948 | "max": null, 949 | "min": null, 950 | "show": true 951 | }, 952 | { 953 | "decimals": 0, 954 | "format": "short", 955 | "label": "", 956 | "logBase": 1, 957 | "max": null, 958 | "min": null, 959 | "show": true 960 | } 961 | ], 962 | "yaxis": { 963 | "align": false, 964 | "alignLevel": null 965 | } 966 | }, 967 | { 968 | "collapsed": false, 969 | "datasource": null, 970 | "gridPos": { 971 | "h": 1, 972 | "w": 24, 973 | "x": 0, 974 | "y": 30 975 | }, 976 | "id": 45, 977 | "panels": [], 978 | "title": "Messages", 979 | "type": "row" 980 | }, 981 | { 982 | "aliasColors": {}, 983 | "bars": false, 984 | "dashLength": 10, 985 | "dashes": false, 986 | "datasource": "${DS_PROMETHEUS}", 987 | "decimals": 0, 988 | "fieldConfig": { 989 | "defaults": { 990 | "links": [] 991 | }, 992 | "overrides": [] 993 | }, 994 | "fill": 1, 995 | "fillGradient": 0, 996 | "gridPos": { 997 | "h": 7, 998 | "w": 24, 999 | "x": 0, 1000 | "y": 31 1001 | }, 1002 | "hiddenSeries": false, 1003 | "id": 11, 1004 | "legend": { 1005 | "alignAsTable": true, 1006 | "avg": false, 1007 | "current": true, 1008 | "hideEmpty": true, 1009 | "hideZero": true, 1010 | "max": true, 1011 | "min": false, 1012 | "rightSide": true, 1013 | "show": true, 1014 | "sideWidth": 500, 1015 | "total": false, 1016 | "values": true 1017 | }, 1018 | "lines": true, 1019 | "linewidth": 1, 1020 | "links": [], 1021 | "nullPointMode": "null", 1022 | "options": { 1023 | "alertThreshold": true 1024 | }, 1025 | "percentage": false, 1026 | "pluginVersion": "7.5.16", 1027 | "pointradius": 5, 1028 | "points": false, 1029 | "renderer": "flot", 1030 | "seriesOverrides": [], 1031 | "spaceLength": 10, 1032 | "stack": false, 1033 | "steppedLine": false, 1034 | "targets": [ 1035 | { 1036 | "exemplar": true, 1037 | "expr": "sum by (broker) (artemis_message_count{pod=\"$instance\", namespace=\"$namespace\"})", 1038 | "format": "time_series", 1039 | "hide": false, 1040 | "interval": "", 1041 | "intervalFactor": 1, 1042 | "legendFormat": "{{ broker }}", 1043 | "refId": "A" 1044 | }, 1045 | { 1046 | "exemplar": true, 1047 | "expr": "sum by (queue) ( topk (20, artemis_message_count{pod=\"$instance\", namespace=\"$namespace\", address!~\"\\\\$.*|activemq\\\\.notifications\"} ) )", 1048 | "format": "time_series", 1049 | "hide": false, 1050 | "interval": "", 1051 | "intervalFactor": 1, 1052 | "legendFormat": "{{ queue }}", 1053 | "refId": "B" 1054 | } 1055 | ], 1056 | "thresholds": [], 1057 | "timeFrom": null, 1058 | "timeRegions": [], 1059 | "timeShift": null, 1060 | "title": "Message Count", 1061 | "tooltip": { 1062 | "shared": true, 1063 | "sort": 2, 1064 | "value_type": "individual" 1065 | }, 1066 | "type": "graph", 1067 | "xaxis": { 1068 | "buckets": null, 1069 | "mode": "time", 1070 | "name": null, 1071 | "show": true, 1072 | "values": [] 1073 | }, 1074 | "yaxes": [ 1075 | { 1076 | "decimals": 0, 1077 | "format": "short", 1078 | "label": "", 1079 | "logBase": 1, 1080 | "max": null, 1081 | "min": "0", 1082 | "show": true 1083 | }, 1084 | { 1085 | "decimals": 0, 1086 | "format": "short", 1087 | "label": "", 1088 | "logBase": 1, 1089 | "max": null, 1090 | "min": "0", 1091 | "show": true 1092 | } 1093 | ], 1094 | "yaxis": { 1095 | "align": false, 1096 | "alignLevel": null 1097 | } 1098 | }, 1099 | { 1100 | "aliasColors": {}, 1101 | "bars": false, 1102 | "dashLength": 10, 1103 | "dashes": false, 1104 | "datasource": "${DS_PROMETHEUS}", 1105 | "decimals": 0, 1106 | "fieldConfig": { 1107 | "defaults": { 1108 | "links": [], 1109 | "unit": "bytes" 1110 | }, 1111 | "overrides": [] 1112 | }, 1113 | "fill": 1, 1114 | "fillGradient": 0, 1115 | "gridPos": { 1116 | "h": 7, 1117 | "w": 24, 1118 | "x": 0, 1119 | "y": 38 1120 | }, 1121 | "hiddenSeries": false, 1122 | "id": 50, 1123 | "legend": { 1124 | "alignAsTable": true, 1125 | "avg": false, 1126 | "current": true, 1127 | "hideEmpty": true, 1128 | "hideZero": true, 1129 | "max": true, 1130 | "min": false, 1131 | "rightSide": true, 1132 | "show": true, 1133 | "sideWidth": 500, 1134 | "total": false, 1135 | "values": true 1136 | }, 1137 | "lines": true, 1138 | "linewidth": 1, 1139 | "links": [], 1140 | "nullPointMode": "null", 1141 | "options": { 1142 | "alertThreshold": true 1143 | }, 1144 | "percentage": false, 1145 | "pluginVersion": "7.5.16", 1146 | "pointradius": 5, 1147 | "points": false, 1148 | "renderer": "flot", 1149 | "seriesOverrides": [], 1150 | "spaceLength": 10, 1151 | "stack": false, 1152 | "steppedLine": false, 1153 | "targets": [ 1154 | { 1155 | "exemplar": true, 1156 | "expr": "sum by (broker) (artemis_persistent_size{pod=\"$instance\", namespace=\"$namespace\"})", 1157 | "format": "time_series", 1158 | "hide": false, 1159 | "interval": "", 1160 | "intervalFactor": 1, 1161 | "legendFormat": "{{ broker }}", 1162 | "refId": "A" 1163 | }, 1164 | { 1165 | "exemplar": true, 1166 | "expr": "sum by (queue) ( topk (20, artemis_persistent_size{pod=\"$instance\", namespace=\"$namespace\", address!~\"\\\\$.*|activemq\\\\.notifications\"} ) )", 1167 | "format": "time_series", 1168 | "hide": false, 1169 | "interval": "", 1170 | "intervalFactor": 1, 1171 | "legendFormat": "{{ queue }}", 1172 | "refId": "B" 1173 | } 1174 | ], 1175 | "thresholds": [], 1176 | "timeFrom": null, 1177 | "timeRegions": [], 1178 | "timeShift": null, 1179 | "title": "Persistent Size", 1180 | "tooltip": { 1181 | "shared": true, 1182 | "sort": 2, 1183 | "value_type": "individual" 1184 | }, 1185 | "type": "graph", 1186 | "xaxis": { 1187 | "buckets": null, 1188 | "mode": "time", 1189 | "name": null, 1190 | "show": true, 1191 | "values": [] 1192 | }, 1193 | "yaxes": [ 1194 | { 1195 | "decimals": 0, 1196 | "format": "bytes", 1197 | "label": "", 1198 | "logBase": 1, 1199 | "max": null, 1200 | "min": "0", 1201 | "show": true 1202 | }, 1203 | { 1204 | "decimals": 0, 1205 | "format": "short", 1206 | "label": "", 1207 | "logBase": 1, 1208 | "max": null, 1209 | "min": "0", 1210 | "show": true 1211 | } 1212 | ], 1213 | "yaxis": { 1214 | "align": false, 1215 | "alignLevel": null 1216 | } 1217 | }, 1218 | { 1219 | "aliasColors": {}, 1220 | "bars": false, 1221 | "dashLength": 10, 1222 | "dashes": false, 1223 | "datasource": "${DS_PROMETHEUS}", 1224 | "fieldConfig": { 1225 | "defaults": {}, 1226 | "overrides": [] 1227 | }, 1228 | "fill": 1, 1229 | "fillGradient": 0, 1230 | "gridPos": { 1231 | "h": 7, 1232 | "w": 24, 1233 | "x": 0, 1234 | "y": 45 1235 | }, 1236 | "hiddenSeries": false, 1237 | "id": 31, 1238 | "legend": { 1239 | "alignAsTable": true, 1240 | "avg": false, 1241 | "current": true, 1242 | "hideEmpty": true, 1243 | "hideZero": true, 1244 | "max": true, 1245 | "min": false, 1246 | "rightSide": true, 1247 | "show": true, 1248 | "sideWidth": 500, 1249 | "total": false, 1250 | "values": true 1251 | }, 1252 | "lines": true, 1253 | "linewidth": 1, 1254 | "nullPointMode": "null", 1255 | "options": { 1256 | "alertThreshold": true 1257 | }, 1258 | "percentage": false, 1259 | "pluginVersion": "7.5.16", 1260 | "pointradius": 2, 1261 | "points": false, 1262 | "renderer": "flot", 1263 | "seriesOverrides": [], 1264 | "spaceLength": 10, 1265 | "stack": false, 1266 | "steppedLine": false, 1267 | "targets": [ 1268 | { 1269 | "exemplar": true, 1270 | "expr": "sum by (queue) ( topk (20, rate(artemis_messages_added{pod=\"$instance\", namespace=\"$namespace\", address!~\"\\\\$.*|activemq\\\\.notifications\"}[1m]) ) )", 1271 | "hide": false, 1272 | "interval": "", 1273 | "legendFormat": "{{ queue }}", 1274 | "queryType": "randomWalk", 1275 | "refId": "A" 1276 | } 1277 | ], 1278 | "thresholds": [], 1279 | "timeFrom": null, 1280 | "timeRegions": [], 1281 | "timeShift": null, 1282 | "title": "Messages Produced", 1283 | "tooltip": { 1284 | "shared": true, 1285 | "sort": 2, 1286 | "value_type": "individual" 1287 | }, 1288 | "type": "graph", 1289 | "xaxis": { 1290 | "buckets": null, 1291 | "mode": "time", 1292 | "name": null, 1293 | "show": true, 1294 | "values": [] 1295 | }, 1296 | "yaxes": [ 1297 | { 1298 | "format": "ops", 1299 | "label": null, 1300 | "logBase": 1, 1301 | "max": null, 1302 | "min": "0", 1303 | "show": true 1304 | }, 1305 | { 1306 | "format": "short", 1307 | "label": null, 1308 | "logBase": 1, 1309 | "max": null, 1310 | "min": null, 1311 | "show": true 1312 | } 1313 | ], 1314 | "yaxis": { 1315 | "align": false, 1316 | "alignLevel": null 1317 | } 1318 | }, 1319 | { 1320 | "aliasColors": {}, 1321 | "bars": false, 1322 | "dashLength": 10, 1323 | "dashes": false, 1324 | "datasource": "${DS_PROMETHEUS}", 1325 | "fieldConfig": { 1326 | "defaults": {}, 1327 | "overrides": [] 1328 | }, 1329 | "fill": 1, 1330 | "fillGradient": 0, 1331 | "gridPos": { 1332 | "h": 7, 1333 | "w": 24, 1334 | "x": 0, 1335 | "y": 52 1336 | }, 1337 | "hiddenSeries": false, 1338 | "id": 34, 1339 | "legend": { 1340 | "alignAsTable": true, 1341 | "avg": false, 1342 | "current": true, 1343 | "hideEmpty": true, 1344 | "hideZero": true, 1345 | "max": true, 1346 | "min": false, 1347 | "rightSide": true, 1348 | "show": true, 1349 | "sideWidth": 500, 1350 | "total": false, 1351 | "values": true 1352 | }, 1353 | "lines": true, 1354 | "linewidth": 1, 1355 | "nullPointMode": "null", 1356 | "options": { 1357 | "alertThreshold": true 1358 | }, 1359 | "percentage": false, 1360 | "pluginVersion": "7.5.16", 1361 | "pointradius": 2, 1362 | "points": false, 1363 | "renderer": "flot", 1364 | "seriesOverrides": [], 1365 | "spaceLength": 10, 1366 | "stack": false, 1367 | "steppedLine": false, 1368 | "targets": [ 1369 | { 1370 | "exemplar": true, 1371 | "expr": "sum by (queue) ( topk (20, rate(artemis_messages_acknowledged{pod=\"$instance\", namespace=\"$namespace\", address!~\"\\\\$.*|activemq\\\\.notifications\"}[1m]) ) )", 1372 | "hide": false, 1373 | "interval": "", 1374 | "legendFormat": "{{ queue }}", 1375 | "queryType": "randomWalk", 1376 | "refId": "A" 1377 | } 1378 | ], 1379 | "thresholds": [], 1380 | "timeFrom": null, 1381 | "timeRegions": [], 1382 | "timeShift": null, 1383 | "title": "Messages Consumed", 1384 | "tooltip": { 1385 | "shared": true, 1386 | "sort": 2, 1387 | "value_type": "individual" 1388 | }, 1389 | "type": "graph", 1390 | "xaxis": { 1391 | "buckets": null, 1392 | "mode": "time", 1393 | "name": null, 1394 | "show": true, 1395 | "values": [] 1396 | }, 1397 | "yaxes": [ 1398 | { 1399 | "format": "ops", 1400 | "label": null, 1401 | "logBase": 1, 1402 | "max": null, 1403 | "min": "0", 1404 | "show": true 1405 | }, 1406 | { 1407 | "format": "short", 1408 | "label": null, 1409 | "logBase": 1, 1410 | "max": null, 1411 | "min": null, 1412 | "show": true 1413 | } 1414 | ], 1415 | "yaxis": { 1416 | "align": false, 1417 | "alignLevel": null 1418 | } 1419 | }, 1420 | { 1421 | "aliasColors": {}, 1422 | "bars": false, 1423 | "dashLength": 10, 1424 | "dashes": false, 1425 | "datasource": "${DS_PROMETHEUS}", 1426 | "fieldConfig": { 1427 | "defaults": {}, 1428 | "overrides": [] 1429 | }, 1430 | "fill": 1, 1431 | "fillGradient": 0, 1432 | "gridPos": { 1433 | "h": 7, 1434 | "w": 24, 1435 | "x": 0, 1436 | "y": 59 1437 | }, 1438 | "hiddenSeries": false, 1439 | "id": 33, 1440 | "legend": { 1441 | "alignAsTable": true, 1442 | "avg": false, 1443 | "current": true, 1444 | "hideEmpty": true, 1445 | "hideZero": true, 1446 | "max": true, 1447 | "min": false, 1448 | "rightSide": true, 1449 | "show": true, 1450 | "sideWidth": 500, 1451 | "total": false, 1452 | "values": true 1453 | }, 1454 | "lines": true, 1455 | "linewidth": 1, 1456 | "nullPointMode": "null", 1457 | "options": { 1458 | "alertThreshold": true 1459 | }, 1460 | "percentage": false, 1461 | "pluginVersion": "7.5.16", 1462 | "pointradius": 2, 1463 | "points": false, 1464 | "renderer": "flot", 1465 | "seriesOverrides": [], 1466 | "spaceLength": 10, 1467 | "stack": false, 1468 | "steppedLine": false, 1469 | "targets": [ 1470 | { 1471 | "exemplar": true, 1472 | "expr": "sum by (queue) ( topk (20, rate(artemis_messages_expired{pod=\"$instance\", namespace=\"$namespace\"}[1m]) ) )", 1473 | "hide": false, 1474 | "interval": "", 1475 | "legendFormat": "{{ queue }}", 1476 | "queryType": "randomWalk", 1477 | "refId": "A" 1478 | } 1479 | ], 1480 | "thresholds": [], 1481 | "timeFrom": null, 1482 | "timeRegions": [], 1483 | "timeShift": null, 1484 | "title": "Messages Expired", 1485 | "tooltip": { 1486 | "shared": true, 1487 | "sort": 2, 1488 | "value_type": "individual" 1489 | }, 1490 | "type": "graph", 1491 | "xaxis": { 1492 | "buckets": null, 1493 | "mode": "time", 1494 | "name": null, 1495 | "show": true, 1496 | "values": [] 1497 | }, 1498 | "yaxes": [ 1499 | { 1500 | "format": "ops", 1501 | "label": null, 1502 | "logBase": 1, 1503 | "max": null, 1504 | "min": "0", 1505 | "show": true 1506 | }, 1507 | { 1508 | "format": "short", 1509 | "label": null, 1510 | "logBase": 1, 1511 | "max": null, 1512 | "min": null, 1513 | "show": true 1514 | } 1515 | ], 1516 | "yaxis": { 1517 | "align": false, 1518 | "alignLevel": null 1519 | } 1520 | }, 1521 | { 1522 | "aliasColors": {}, 1523 | "bars": false, 1524 | "dashLength": 10, 1525 | "dashes": false, 1526 | "datasource": "${DS_PROMETHEUS}", 1527 | "fieldConfig": { 1528 | "defaults": {}, 1529 | "overrides": [] 1530 | }, 1531 | "fill": 1, 1532 | "fillGradient": 0, 1533 | "gridPos": { 1534 | "h": 7, 1535 | "w": 24, 1536 | "x": 0, 1537 | "y": 66 1538 | }, 1539 | "hiddenSeries": false, 1540 | "id": 52, 1541 | "legend": { 1542 | "alignAsTable": true, 1543 | "avg": false, 1544 | "current": true, 1545 | "hideEmpty": true, 1546 | "hideZero": true, 1547 | "max": true, 1548 | "min": false, 1549 | "rightSide": true, 1550 | "show": true, 1551 | "sideWidth": 500, 1552 | "total": false, 1553 | "values": true 1554 | }, 1555 | "lines": true, 1556 | "linewidth": 1, 1557 | "nullPointMode": "null", 1558 | "options": { 1559 | "alertThreshold": true 1560 | }, 1561 | "percentage": false, 1562 | "pluginVersion": "7.5.16", 1563 | "pointradius": 2, 1564 | "points": false, 1565 | "renderer": "flot", 1566 | "seriesOverrides": [], 1567 | "spaceLength": 10, 1568 | "stack": false, 1569 | "steppedLine": false, 1570 | "targets": [ 1571 | { 1572 | "exemplar": true, 1573 | "expr": "sum by (queue) ( topk (20, rate(artemis_messages_killed{pod=\"$instance\", namespace=\"$namespace\"}[1m]) ) )", 1574 | "hide": false, 1575 | "interval": "", 1576 | "legendFormat": "{{ queue }}", 1577 | "queryType": "randomWalk", 1578 | "refId": "A" 1579 | } 1580 | ], 1581 | "thresholds": [], 1582 | "timeFrom": null, 1583 | "timeRegions": [], 1584 | "timeShift": null, 1585 | "title": "Messages to DLQ", 1586 | "tooltip": { 1587 | "shared": true, 1588 | "sort": 2, 1589 | "value_type": "individual" 1590 | }, 1591 | "type": "graph", 1592 | "xaxis": { 1593 | "buckets": null, 1594 | "mode": "time", 1595 | "name": null, 1596 | "show": true, 1597 | "values": [] 1598 | }, 1599 | "yaxes": [ 1600 | { 1601 | "format": "ops", 1602 | "label": null, 1603 | "logBase": 1, 1604 | "max": null, 1605 | "min": "0", 1606 | "show": true 1607 | }, 1608 | { 1609 | "format": "short", 1610 | "label": null, 1611 | "logBase": 1, 1612 | "max": null, 1613 | "min": null, 1614 | "show": true 1615 | } 1616 | ], 1617 | "yaxis": { 1618 | "align": false, 1619 | "alignLevel": null 1620 | } 1621 | }, 1622 | { 1623 | "collapsed": false, 1624 | "datasource": null, 1625 | "gridPos": { 1626 | "h": 1, 1627 | "w": 24, 1628 | "x": 0, 1629 | "y": 73 1630 | }, 1631 | "id": 43, 1632 | "panels": [], 1633 | "title": "Storage", 1634 | "type": "row" 1635 | }, 1636 | { 1637 | "aliasColors": {}, 1638 | "bars": false, 1639 | "dashLength": 10, 1640 | "dashes": false, 1641 | "datasource": "${DS_PROMETHEUS}", 1642 | "fieldConfig": { 1643 | "defaults": { 1644 | "unit": "percentunit" 1645 | }, 1646 | "overrides": [] 1647 | }, 1648 | "fill": 1, 1649 | "fillGradient": 0, 1650 | "gridPos": { 1651 | "h": 7, 1652 | "w": 12, 1653 | "x": 0, 1654 | "y": 74 1655 | }, 1656 | "hiddenSeries": false, 1657 | "id": 48, 1658 | "legend": { 1659 | "alignAsTable": true, 1660 | "avg": false, 1661 | "current": true, 1662 | "hideEmpty": true, 1663 | "hideZero": true, 1664 | "max": true, 1665 | "min": false, 1666 | "rightSide": true, 1667 | "show": false, 1668 | "sideWidth": 300, 1669 | "total": false, 1670 | "values": true 1671 | }, 1672 | "lines": true, 1673 | "linewidth": 1, 1674 | "nullPointMode": "null", 1675 | "options": { 1676 | "alertThreshold": true 1677 | }, 1678 | "percentage": false, 1679 | "pluginVersion": "7.5.16", 1680 | "pointradius": 2, 1681 | "points": false, 1682 | "renderer": "flot", 1683 | "seriesOverrides": [], 1684 | "spaceLength": 10, 1685 | "stack": false, 1686 | "steppedLine": false, 1687 | "targets": [ 1688 | { 1689 | "exemplar": true, 1690 | "expr": "sum by (broker) (artemis_disk_store_usage{pod=\"$instance\", namespace=\"$namespace\"})", 1691 | "hide": false, 1692 | "interval": "", 1693 | "legendFormat": "{{ broker }}", 1694 | "queryType": "randomWalk", 1695 | "refId": "A" 1696 | } 1697 | ], 1698 | "thresholds": [], 1699 | "timeFrom": null, 1700 | "timeRegions": [], 1701 | "timeShift": null, 1702 | "title": "Disk Usage", 1703 | "tooltip": { 1704 | "shared": true, 1705 | "sort": 2, 1706 | "value_type": "individual" 1707 | }, 1708 | "type": "graph", 1709 | "xaxis": { 1710 | "buckets": null, 1711 | "mode": "time", 1712 | "name": null, 1713 | "show": true, 1714 | "values": [] 1715 | }, 1716 | "yaxes": [ 1717 | { 1718 | "format": "percentunit", 1719 | "label": "", 1720 | "logBase": 1, 1721 | "max": null, 1722 | "min": null, 1723 | "show": true 1724 | }, 1725 | { 1726 | "format": "short", 1727 | "label": null, 1728 | "logBase": 1, 1729 | "max": null, 1730 | "min": null, 1731 | "show": true 1732 | } 1733 | ], 1734 | "yaxis": { 1735 | "align": false, 1736 | "alignLevel": null 1737 | } 1738 | }, 1739 | { 1740 | "aliasColors": {}, 1741 | "bars": false, 1742 | "dashLength": 10, 1743 | "dashes": false, 1744 | "datasource": "${DS_PROMETHEUS}", 1745 | "fieldConfig": { 1746 | "defaults": {}, 1747 | "overrides": [] 1748 | }, 1749 | "fill": 1, 1750 | "fillGradient": 0, 1751 | "gridPos": { 1752 | "h": 7, 1753 | "w": 12, 1754 | "x": 12, 1755 | "y": 74 1756 | }, 1757 | "hiddenSeries": false, 1758 | "id": 29, 1759 | "legend": { 1760 | "alignAsTable": true, 1761 | "avg": false, 1762 | "current": true, 1763 | "hideEmpty": true, 1764 | "hideZero": true, 1765 | "max": true, 1766 | "min": false, 1767 | "rightSide": true, 1768 | "show": false, 1769 | "sideWidth": 500, 1770 | "total": false, 1771 | "values": true 1772 | }, 1773 | "lines": true, 1774 | "linewidth": 1, 1775 | "nullPointMode": "null", 1776 | "options": { 1777 | "alertThreshold": true 1778 | }, 1779 | "percentage": false, 1780 | "pluginVersion": "7.5.16", 1781 | "pointradius": 2, 1782 | "points": false, 1783 | "renderer": "flot", 1784 | "seriesOverrides": [], 1785 | "spaceLength": 10, 1786 | "stack": false, 1787 | "steppedLine": false, 1788 | "targets": [ 1789 | { 1790 | "exemplar": true, 1791 | "expr": "sum by (broker) ( artemis_number_of_pages{pod=\"$instance\", namespace=\"$namespace\"} )", 1792 | "hide": false, 1793 | "interval": "", 1794 | "legendFormat": "{{ broker }}", 1795 | "queryType": "randomWalk", 1796 | "refId": "B" 1797 | }, 1798 | { 1799 | "exemplar": true, 1800 | "expr": "sum by (address) ( topk (20, artemis_number_of_pages{pod=\"$instance\", namespace=\"$namespace\"} ) )", 1801 | "interval": "", 1802 | "legendFormat": "{{ address }}", 1803 | "queryType": "randomWalk", 1804 | "refId": "A" 1805 | } 1806 | ], 1807 | "thresholds": [], 1808 | "timeFrom": null, 1809 | "timeRegions": [], 1810 | "timeShift": null, 1811 | "title": "Paging", 1812 | "tooltip": { 1813 | "shared": true, 1814 | "sort": 0, 1815 | "value_type": "individual" 1816 | }, 1817 | "type": "graph", 1818 | "xaxis": { 1819 | "buckets": null, 1820 | "mode": "time", 1821 | "name": null, 1822 | "show": true, 1823 | "values": [] 1824 | }, 1825 | "yaxes": [ 1826 | { 1827 | "format": "short", 1828 | "label": null, 1829 | "logBase": 1, 1830 | "max": null, 1831 | "min": null, 1832 | "show": true 1833 | }, 1834 | { 1835 | "format": "short", 1836 | "label": null, 1837 | "logBase": 1, 1838 | "max": null, 1839 | "min": null, 1840 | "show": true 1841 | } 1842 | ], 1843 | "yaxis": { 1844 | "align": false, 1845 | "alignLevel": null 1846 | } 1847 | } 1848 | ], 1849 | "refresh": false, 1850 | "schemaVersion": 27, 1851 | "style": "dark", 1852 | "tags": [ 1853 | "apache artemis" 1854 | ], 1855 | "templating": { 1856 | "list": [ 1857 | { 1858 | "allValue": null, 1859 | "current": {}, 1860 | "datasource": "${DS_PROMETHEUS}", 1861 | "definition": "label_values(artemis_connection_count, namespace)", 1862 | "description": null, 1863 | "error": null, 1864 | "hide": 0, 1865 | "includeAll": false, 1866 | "label": "namespace", 1867 | "multi": false, 1868 | "name": "namespace", 1869 | "options": [], 1870 | "query": { 1871 | "query": "label_values(artemis_connection_count, namespace)", 1872 | "refId": "Prometheus-namespace-Variable-Query" 1873 | }, 1874 | "refresh": 1, 1875 | "regex": "", 1876 | "skipUrlSync": false, 1877 | "sort": 1, 1878 | "tagValuesQuery": "", 1879 | "tags": [], 1880 | "tagsQuery": "", 1881 | "type": "query", 1882 | "useTags": false 1883 | }, 1884 | { 1885 | "allValue": null, 1886 | "current": {}, 1887 | "datasource": "${DS_PROMETHEUS}", 1888 | "definition": "label_values(artemis_connection_count{namespace=\"$namespace\"}, pod)", 1889 | "description": null, 1890 | "error": null, 1891 | "hide": 0, 1892 | "includeAll": false, 1893 | "label": "instance", 1894 | "multi": false, 1895 | "name": "instance", 1896 | "options": [], 1897 | "query": { 1898 | "query": "label_values(artemis_connection_count{namespace=\"$namespace\"}, pod)", 1899 | "refId": "Prometheus-instance-Variable-Query" 1900 | }, 1901 | "refresh": 1, 1902 | "regex": "", 1903 | "skipUrlSync": false, 1904 | "sort": 1, 1905 | "tagValuesQuery": "", 1906 | "tags": [], 1907 | "tagsQuery": "", 1908 | "type": "query", 1909 | "useTags": false 1910 | } 1911 | ] 1912 | }, 1913 | "time": { 1914 | "from": "now-1h", 1915 | "to": "now" 1916 | }, 1917 | "timepicker": { 1918 | "refresh_intervals": [ 1919 | "5s", 1920 | "10s", 1921 | "30s", 1922 | "1m", 1923 | "5m", 1924 | "15m", 1925 | "30m", 1926 | "1h", 1927 | "2h", 1928 | "1d" 1929 | ], 1930 | "time_options": [ 1931 | "5m", 1932 | "15m", 1933 | "1h", 1934 | "6h", 1935 | "12h", 1936 | "24h", 1937 | "2d", 1938 | "7d", 1939 | "30d" 1940 | ] 1941 | }, 1942 | "timezone": "utc", 1943 | "title": "AMQ Broker", 1944 | "uid": "amq-broker", 1945 | "version": 1 1946 | } --------------------------------------------------------------------------------