├── .coveragerc ├── .dockerignore ├── .gitignore ├── .buildkite ├── scripts │ ├── run-tests.sh │ ├── docker-image-build.sh │ └── docker-image-push.sh ├── pipeline.yml ├── pull-requests.json └── hooks │ └── pre-command ├── requirements.txt ├── requirements-dev.txt ├── test_setup.py ├── kuberwatcher.yml ├── tests ├── fixtures │ ├── override.yml │ ├── kube-system.yml │ ├── challenge.yml │ ├── disabled.yml │ └── test.yml ├── cassettes │ ├── test_getting_all_namespace │ ├── test_getting_namespace_with_overriden_config │ ├── test_getting_email_alerts_disabled_when_overriding_alerts │ └── test_get_all_pods └── kuberwatcher_test.py ├── Dockerfile ├── .ci └── jobs │ └── defaults.yml ├── catalog-info.yaml ├── k8s └── kuberwatcher.yml ├── Makefile ├── template.py ├── README.md ├── LICENSE └── kuberwatcher.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | venv/* 4 | insert_test_data.py 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !kuberwatcher.py 3 | !template.py 4 | !requirements.txt 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | htmlcov 2 | .coverage 3 | venv 4 | __pycache__ 5 | *.pyc 6 | .cache 7 | -------------------------------------------------------------------------------- /.buildkite/scripts/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | make test -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | elasticsearch==7.17.9 2 | certifi==2022.12.7 3 | kubernetes==25.3.0 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest==7.2.1 2 | pytest-cov==4.0.0 3 | chevron==0.14.0 4 | vcrpy==4.2.1 5 | -------------------------------------------------------------------------------- /test_setup.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) 3 | -------------------------------------------------------------------------------- /kuberwatcher.yml: -------------------------------------------------------------------------------- 1 | alerts.email: 'kuberwatcher-alerts@example.com' 2 | alerts.slack: 'slack-channel-name' 3 | kibana_url: 'https://kibana.example.com' 4 | -------------------------------------------------------------------------------- /tests/fixtures/override.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: override 6 | labels: 7 | watcher: enabled 8 | annotations: 9 | watcher.alerts.slack: '@michael.override' 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN mkdir /usr/src/app 4 | 5 | WORKDIR /usr/src/app 6 | 7 | COPY requirements.txt requirements.txt 8 | 9 | RUN pip install -r requirements.txt 10 | 11 | COPY . . 12 | 13 | CMD ["python", "kuberwatcher.py"] 14 | -------------------------------------------------------------------------------- /tests/fixtures/kube-system.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: elasticsearch 6 | namespace: kube-system 7 | spec: 8 | type: ExternalName 9 | externalName: 10.0.2.2 # minikube virtualbox host address 10 | ports: 11 | - port: 9200 12 | -------------------------------------------------------------------------------- /.buildkite/scripts/docker-image-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | set +x 5 | 6 | DOCKER_PASSWORD=$(vault read -field password secret/infra/prod/flavorchef) 7 | 8 | docker login -u flavorchef -p $DOCKER_PASSWORD docker.elastic.co 9 | 10 | unset DOCKER_PASSWORD 11 | set -x 12 | 13 | make build -------------------------------------------------------------------------------- /.buildkite/scripts/docker-image-push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | set +x 5 | 6 | DOCKER_PASSWORD=$(vault read -field password secret/infra/prod/flavorchef) 7 | 8 | docker login -u flavorchef -p $DOCKER_PASSWORD docker.elastic.co 9 | 10 | unset DOCKER_PASSWORD 11 | set -x 12 | 13 | make deploy -------------------------------------------------------------------------------- /tests/fixtures/challenge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Pod 4 | metadata: 5 | name: challenge 6 | namespace: test 7 | labels: 8 | watcher: enabled 9 | ownerReferences: 10 | - apiVersion: apps/v1 11 | kind: Challenge 12 | name: challenge 13 | uid: 5e014df1-0c28-451e-97f5-0edde20bab45 14 | spec: 15 | containers: 16 | - name: nginx 17 | image: nginx 18 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: ":package: kuberwatcher build and test" 3 | key: kuberwatcher-pr 4 | branches: "!master" 5 | commands: 6 | - .buildkite/scripts/run-tests.sh 7 | - .buildkite/scripts/docker-image-build.sh 8 | agents: 9 | provider: "gcp" 10 | 11 | - label: ":package: kuberwatcher push image" 12 | key: kuberwatcher-master 13 | branches: "master" 14 | commands: 15 | - .buildkite/scripts/docker-image-push.sh 16 | agents: 17 | provider: "gcp" -------------------------------------------------------------------------------- /.buildkite/pull-requests.json: -------------------------------------------------------------------------------- 1 | { 2 | "jobs": [ 3 | { 4 | "enabled": true, 5 | "pipeline_slug": "kuberwatcher", 6 | "allow_org_users": true, 7 | "build_on_commit": false, 8 | "build_on_comment": true, 9 | "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", 10 | "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", 11 | "skip_ci_labels": ["skip-ci"], 12 | "skip_ci_on_only_changed": ["\\.md$"] 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /.buildkite/hooks/pre-command: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | if [[ "$BUILDKITE_PIPELINE_SLUG" == "kuberwatcher" ]]; then 5 | 6 | set +x 7 | 8 | echo "------ Setting up Prod Vault -------" 9 | 10 | VAULT_ROLE_ID=$(vault read --field=role_id secret/ci/elastic-kuberwatcher/vault-prod) 11 | VAULT_SECRET_ID=$(vault read --field=secret_id secret/ci/elastic-kuberwatcher/vault-prod) 12 | export VAULT_ADDR=https://secrets.elastic.co:8200 13 | 14 | # Clear the previous token or it will cause the `vault write` below to fail with: 15 | # "error performing token check: failed to look up namespace from the token: no namespace" 16 | 17 | unset VAULT_TOKEN 18 | 19 | VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") 20 | export VAULT_TOKEN 21 | fi -------------------------------------------------------------------------------- /tests/fixtures/disabled.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: disabled 6 | --- 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | metadata: 10 | annotations: 11 | labels: 12 | run: nginx 13 | name: nginx 14 | namespace: disabled 15 | spec: 16 | replicas: 1 17 | selector: 18 | matchLabels: 19 | run: nginx 20 | template: 21 | metadata: 22 | labels: 23 | watcher: enabled 24 | run: nginx 25 | spec: 26 | containers: 27 | - image: nginx 28 | imagePullPolicy: Always 29 | name: nginx 30 | resources: {} 31 | --- 32 | apiVersion: apps/v1 33 | kind: Deployment 34 | metadata: 35 | annotations: 36 | labels: 37 | run: disabled 38 | name: disabled 39 | namespace: disabled 40 | spec: 41 | replicas: 1 42 | selector: 43 | matchLabels: 44 | run: disabled 45 | template: 46 | metadata: 47 | labels: 48 | run: disabled 49 | spec: 50 | containers: 51 | - image: nginx 52 | name: nginx 53 | -------------------------------------------------------------------------------- /tests/fixtures/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: test 6 | labels: 7 | watcher: enabled 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | annotations: 13 | labels: 14 | run: nginx 15 | name: nginx 16 | namespace: test 17 | spec: 18 | replicas: 1 19 | selector: 20 | matchLabels: 21 | run: nginx 22 | template: 23 | metadata: 24 | labels: 25 | run: nginx 26 | spec: 27 | containers: 28 | - image: nginx 29 | name: nginx 30 | --- 31 | apiVersion: v1 32 | kind: Pod 33 | metadata: 34 | name: nginx-pod 35 | namespace: test 36 | spec: 37 | containers: 38 | - name: nginx 39 | image: nginx 40 | --- 41 | apiVersion: batch/v1 42 | kind: CronJob 43 | metadata: 44 | name: hello 45 | namespace: test 46 | spec: 47 | schedule: "* * * * *" 48 | concurrencyPolicy: Forbid 49 | jobTemplate: 50 | spec: 51 | template: 52 | spec: 53 | containers: 54 | - name: hello 55 | image: hello-world 56 | restartPolicy: OnFailure 57 | -------------------------------------------------------------------------------- /.ci/jobs/defaults.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ##### GLOBAL METADATA 4 | 5 | - meta: 6 | cluster: devops-ci 7 | 8 | ##### JOB DEFAULTS 9 | 10 | - job: 11 | logrotate: 12 | daysToKeep: 30 13 | numToKeep: 100 14 | parameters: 15 | - string: 16 | name: branch_specifier 17 | default: master 18 | description: the Git branch specifier to build (<branchName>, <tagName>, 19 | <commitId>, etc.) 20 | properties: 21 | - github: 22 | url: https://github.com/elastic/kuberwatcher/ 23 | - inject: 24 | properties-content: HOME=$JENKINS_HOME 25 | node: linux 26 | scm: 27 | - git: 28 | name: origin 29 | credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba 30 | reference-repo: /var/lib/jenkins/.git-references/kuberwatcher.git 31 | branches: 32 | - ${branch_specifier} 33 | url: git@github.com:elastic/kuberwatcher.git 34 | basedir: '' 35 | wipe-workspace: 'True' 36 | wrappers: 37 | - ansicolor 38 | - timeout: 39 | type: absolute 40 | timeout: 10 41 | fail: true 42 | - timestamps 43 | publishers: 44 | - email: 45 | recipients: infra-root+build@elastic.co 46 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | # Buildkite pipeline for kuberwatcher. 2 | --- 3 | # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json 4 | apiVersion: backstage.io/v1alpha1 5 | kind: Resource 6 | metadata: 7 | name: buildkite-pipeline-kuberwatcher 8 | description: Buildkite Pipeline for kuberwatcher 9 | 10 | spec: 11 | type: buildkite-pipeline 12 | owner: group:infra-services 13 | system: buildkite 14 | implementation: 15 | apiVersion: buildkite.elastic.dev/v1 16 | kind: Pipeline 17 | metadata: 18 | name: kuberwatcher 19 | spec: 20 | repository: elastic/kuberwatcher 21 | pipeline_file: ".buildkite/pipeline.yml" 22 | provider_settings: 23 | build_pull_requests: true 24 | build_pull_request_ready_for_review: true 25 | publish_commit_status: true 26 | publish_commit_status_per_step: false 27 | publish_blocked_as_pending: true 28 | skip_pull_request_builds_for_existing_commits: true 29 | cancel_deleted_branch_builds: true 30 | teams: 31 | infra-services: 32 | access_level: MANAGE_BUILD_AND_READ 33 | everyone: 34 | access_level: READ_ONLY -------------------------------------------------------------------------------- /tests/cassettes/test_getting_all_namespace: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - application/json 7 | Content-Type: 8 | - application/json 9 | User-Agent: 10 | - OpenAPI-Generator/25.3.0/python 11 | method: GET 12 | uri: https://192.168.49.2:8443/api/v1/namespaces?labelSelector=watcher%3Denabled 13 | response: 14 | body: 15 | string: '{"kind":"NamespaceList","apiVersion":"v1","metadata":{"resourceVersion":"22851"},"items":[{"metadata":{"name":"override","uid":"0efc722f-6e71-4290-b7a2-bb3c8e21e90a","resourceVersion":"22645","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"override","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{\"watcher.alerts.slack\":\"@michael.override\"},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"override\"}}\n","watcher.alerts.slack":"@michael.override"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{},"f:watcher.alerts.slack":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}},{"metadata":{"name":"test","uid":"6dd878e2-5fc6-41ba-a6e9-3c5d4812fe32","resourceVersion":"22653","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"test","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"test\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}}]} 16 | 17 | ' 18 | headers: 19 | Audit-Id: 20 | - 65a767bf-8a8d-4c32-9892-f8f05b678cfa 21 | Cache-Control: 22 | - no-cache, private 23 | Content-Length: 24 | - '1847' 25 | Content-Type: 26 | - application/json 27 | Date: 28 | - Tue, 25 Apr 2023 20:31:15 GMT 29 | X-Kubernetes-Pf-Flowschema-Uid: 30 | - b6229760-c42a-4dc5-a942-f7612b973d7c 31 | X-Kubernetes-Pf-Prioritylevel-Uid: 32 | - f1a99473-5335-402d-ada4-0498e1d87d9b 33 | status: 34 | code: 200 35 | message: OK 36 | version: 1 37 | -------------------------------------------------------------------------------- /tests/cassettes/test_getting_namespace_with_overriden_config: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - application/json 7 | Content-Type: 8 | - application/json 9 | User-Agent: 10 | - OpenAPI-Generator/25.3.0/python 11 | method: GET 12 | uri: https://192.168.49.2:8443/api/v1/namespaces?labelSelector=watcher%3Denabled 13 | response: 14 | body: 15 | string: '{"kind":"NamespaceList","apiVersion":"v1","metadata":{"resourceVersion":"22851"},"items":[{"metadata":{"name":"override","uid":"0efc722f-6e71-4290-b7a2-bb3c8e21e90a","resourceVersion":"22645","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"override","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{\"watcher.alerts.slack\":\"@michael.override\"},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"override\"}}\n","watcher.alerts.slack":"@michael.override"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{},"f:watcher.alerts.slack":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}},{"metadata":{"name":"test","uid":"6dd878e2-5fc6-41ba-a6e9-3c5d4812fe32","resourceVersion":"22653","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"test","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"test\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}}]} 16 | 17 | ' 18 | headers: 19 | Audit-Id: 20 | - b426263d-0d09-4a55-afcd-be6408e58d12 21 | Cache-Control: 22 | - no-cache, private 23 | Content-Length: 24 | - '1847' 25 | Content-Type: 26 | - application/json 27 | Date: 28 | - Tue, 25 Apr 2023 20:31:15 GMT 29 | X-Kubernetes-Pf-Flowschema-Uid: 30 | - b6229760-c42a-4dc5-a942-f7612b973d7c 31 | X-Kubernetes-Pf-Prioritylevel-Uid: 32 | - f1a99473-5335-402d-ada4-0498e1d87d9b 33 | status: 34 | code: 200 35 | message: OK 36 | version: 1 37 | -------------------------------------------------------------------------------- /tests/cassettes/test_getting_email_alerts_disabled_when_overriding_alerts: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - application/json 7 | Content-Type: 8 | - application/json 9 | User-Agent: 10 | - OpenAPI-Generator/25.3.0/python 11 | method: GET 12 | uri: https://192.168.49.2:8443/api/v1/namespaces?labelSelector=watcher%3Denabled 13 | response: 14 | body: 15 | string: '{"kind":"NamespaceList","apiVersion":"v1","metadata":{"resourceVersion":"22851"},"items":[{"metadata":{"name":"override","uid":"0efc722f-6e71-4290-b7a2-bb3c8e21e90a","resourceVersion":"22645","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"override","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{\"watcher.alerts.slack\":\"@michael.override\"},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"override\"}}\n","watcher.alerts.slack":"@michael.override"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{},"f:watcher.alerts.slack":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}},{"metadata":{"name":"test","uid":"6dd878e2-5fc6-41ba-a6e9-3c5d4812fe32","resourceVersion":"22653","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"kubernetes.io/metadata.name":"test","watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Namespace\",\"metadata\":{\"annotations\":{},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"test\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:kubernetes.io/metadata.name":{},"f:watcher":{}}}}}]},"spec":{"finalizers":["kubernetes"]},"status":{"phase":"Active"}}]} 16 | 17 | ' 18 | headers: 19 | Audit-Id: 20 | - ed1cd951-d77e-4a52-af8e-698026270775 21 | Cache-Control: 22 | - no-cache, private 23 | Content-Length: 24 | - '1847' 25 | Content-Type: 26 | - application/json 27 | Date: 28 | - Tue, 25 Apr 2023 20:31:15 GMT 29 | X-Kubernetes-Pf-Flowschema-Uid: 30 | - b6229760-c42a-4dc5-a942-f7612b973d7c 31 | X-Kubernetes-Pf-Prioritylevel-Uid: 32 | - f1a99473-5335-402d-ada4-0498e1d87d9b 33 | status: 34 | code: 200 35 | message: OK 36 | version: 1 37 | -------------------------------------------------------------------------------- /k8s/kuberwatcher.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: kuberwatcher 6 | data: 7 | kuberwatcher.yml: |- 8 | --- 9 | alerts.email: example-alert@elastic.co 10 | alerts.slack: alerts 11 | kibana_url: https://kibana.example.com 12 | --- 13 | apiVersion: batch/v1 14 | kind: CronJob 15 | metadata: 16 | name: kuberwatcher 17 | spec: 18 | schedule: "*/5 * * * *" 19 | concurrencyPolicy: Forbid 20 | failedJobsHistoryLimit: 3 21 | successfulJobsHistoryLimit: 3 22 | jobTemplate: 23 | spec: 24 | template: 25 | spec: 26 | volumes: 27 | - name: kuberwatcher 28 | configMap: 29 | name: kuberwatcher 30 | defaultMode: 0600 31 | - name: es-certs 32 | secret: 33 | secretName: quickstart-es-http-certs-public 34 | containers: 35 | - name: kuberwatcher 36 | image: docker.elastic.co/kuberwatcher/kuberwatcher:7.1.1-1 37 | env: 38 | - name: ES_USERNAME 39 | value: elastic 40 | - name: ES_HOSTS 41 | value: https://quickstart-es-http:9200 42 | - name: ES_PASSWORD 43 | valueFrom: 44 | secretKeyRef: 45 | key: elastic 46 | name: quickstart-es-elastic-user 47 | - name: ES_CA_CERTS 48 | value: /mnt/certs/ca.crt 49 | resources: 50 | limits: 51 | cpu: 100m 52 | memory: 100Mi 53 | requests: 54 | cpu: 100m 55 | memory: 100Mi 56 | volumeMounts: 57 | - name: es-certs 58 | mountPath: /mnt/certs/ 59 | readOnly: true 60 | - mountPath: /usr/src/app/kuberwatcher.yml 61 | name: kuberwatcher 62 | subPath: kuberwatcher.yml 63 | restartPolicy: Never 64 | serviceAccount: kuberwatcher 65 | serviceAccountName: kuberwatcher 66 | --- 67 | apiVersion: rbac.authorization.k8s.io/v1 68 | kind: ClusterRole 69 | metadata: 70 | name: kuberwatcher-role 71 | rules: 72 | - apiGroups: 73 | - "" 74 | resources: 75 | - namespaces 76 | - pods 77 | verbs: 78 | - get 79 | - list 80 | --- 81 | apiVersion: v1 82 | kind: ServiceAccount 83 | metadata: 84 | name: kuberwatcher 85 | --- 86 | apiVersion: rbac.authorization.k8s.io/v1 87 | kind: ClusterRoleBinding 88 | metadata: 89 | name: kuberwatcher-rolebinding 90 | roleRef: 91 | apiGroup: rbac.authorization.k8s.io 92 | kind: ClusterRole 93 | name: kuberwatcher-role 94 | subjects: 95 | - kind: ServiceAccount 96 | name: kuberwatcher 97 | namespace: # namespace needs to be specified here where Kuberwatcher is running 98 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | SHELL:=/bin/bash -eu 4 | export PATH := ./bin:./venv/bin:$(PATH) 5 | 6 | VERSION = 7.17.9 7 | IMAGE = docker.elastic.co/kuberwatcher/kuberwatcher:${VERSION} 8 | STACK_VERSION = 7.17.9 9 | PASSWORD = changeme 10 | 11 | build: 12 | docker build -t ${IMAGE} . 13 | 14 | deps: 15 | docker rm -f kuberwatcher_es kuberwatcher_kibana || true 16 | docker run --rm -i -v $$(pwd):/app -w /app docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "elasticsearch-keystore create && echo $(PASSWORD) | elasticsearch-keystore add -x bootstrap.password && echo $(SLACK_URL) | elasticsearch-keystore add -x xpack.notification.slack.account.monitoring.secure_url && cp /usr/share/elasticsearch/config/elasticsearch.keystore ./" 17 | docker run --name kuberwatcher_es -d -p 9200:9200 -p 9300:9300 -v $(PWD)/elasticsearch.keystore:/usr/share/elasticsearch/config/elasticsearch.keystore -e "ELASTIC_PASSWORD=$(PASSWORD)" -e "xpack.security.enabled=true" -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) 18 | echo 'Waiting for elasticsearch to start' 19 | until curl -Is -u elastic:$(PASSWORD) localhost:9200 ; do printf '.' ; sleep 1; done 20 | docker run --name kuberwatcher_kibana --link kuberwatcher_es:elasticsearch -e "ELASTICSEARCH_USERNAME=elastic" -e "ELASTICSEARCH_PASSWORD=$(PASSWORD)" -d -p 5601:5601 docker.elastic.co/kibana/kibana:$(STACK_VERSION) 21 | echo 'Waiting for Kibana to start' 22 | until curl -u elastic:$(PASSWORD) -Is 'localhost:5601/api/status'; do printf '.'; sleep 1; done 23 | echo 'Activating X-Pack trial license' 24 | until curl -u elastic:$(PASSWORD) -Is -XPOST 'localhost:9200/_license/start_trial?acknowledge=true' ; do printf '.' ; sleep 1; done 25 | 26 | run: build 27 | docker run --rm -ti -v ${HOME}/.minikube:${HOME}/.minikube -v ~/.kube:/root/.kube/ --link kuberwatcher_es:elasticsearch -v $$(PWD):/app -w /app -e ES_PASSWORD="${ES_PASSWORD}" -e ES_USERNAME="${ES_USERNAME}" -e ES_HOSTS="${ES_HOSTS}" ${IMAGE} 28 | 29 | deploy: build 30 | docker push ${IMAGE} 31 | 32 | test-python: venv 33 | source ./venv/bin/activate 34 | pytest -v --cov=./ --cov-fail-under=100 --cov-report html 35 | 36 | report: test 37 | open htmlcov/index.html 38 | 39 | venv: requirements.txt requirements-dev.txt 40 | test -d venv || python3 -m venv venv 41 | source ./venv/bin/activate 42 | venv/bin/pip install -r requirements.txt -r requirements-dev.txt 43 | 44 | clean: 45 | docker rm -f kuberwatcher_kibana kuberwatcher_es || true 46 | rm -f elasticsearch.keystore || true 47 | kubectl delete -f tests/fixtures || true 48 | kubectl delete -f https://raw.githubusercontent.com/elastic/beats/v$(STACK_VERSION)/deploy/kubernetes/metricbeat-kubernetes.yaml || true 49 | 50 | test: 51 | export CI=$${CI:-'false'} && \ 52 | docker run --net=host -e ES_HOSTS="http://127.0.0.1:9200" -v ~/.kube:/.kube/ --user=$$UID -e CI=$$CI -i -v "$$PWD":/app -w /app python:3.10 /usr/bin/make test-python 53 | 54 | fixtures: 55 | cluster=$$(kubectl config current-context) && \ 56 | read -p "Going to install fixtures into cluster '$${cluster}'. Hit enter to continue" 57 | kubectl apply -f tests/fixtures 58 | kubectl apply -f https://raw.githubusercontent.com/elastic/beats/v$(STACK_VERSION)/deploy/kubernetes/metricbeat-kubernetes.yaml 59 | 60 | clean_cassettes: 61 | rm -rf tests/cassettes/* 62 | -------------------------------------------------------------------------------- /template.py: -------------------------------------------------------------------------------- 1 | template_open = '{{#ctx.payload.aggregations.result.hits.hits.0._source}}' 2 | template_close = template_open.replace('{{#','{{/') 3 | kibana_url = ( 4 | "{{ctx.metadata.kibana_url}}/app/kibana#/discover?" 5 | "_a=(columns:!(_source),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f," 6 | "index:'metricbeat-*',key:query,negate:!f,type:custom,value:'')," 7 | "query:(bool:(must:!((regexp:(kubernetes.pod.name:'{{ctx.metadata.regex}}'))," 8 | "(match:(metricset.name:'state_pod'))," 9 | "(match:(kubernetes.namespace:{{ctx.metadata.namespace}})))))))," 10 | "index:'metricbeat-*'," 11 | "interval:auto,query:(language:lucene,query:'')," 12 | "regexp:(language:lucene,query:'kubernetes.pod.name:test-nginx-%5B%5E-%5D%20-%5B%5E-%5D%20')," 13 | "sort:!('@timestamp',desc),time:(from:now%2FM,mode:quick,to:now%2FM))" 14 | "&_g=(refreshInterval:(display:Off,pause:!f,value:0)," 15 | "time:(from:now-15m,mode:quick,to:now))" 16 | ) 17 | watch_url = "{{ctx.metadata.kibana_url}}/app/management/insightsAndAlerting/watcher/watches/watch/{{ctx.metadata.name}}/status" 18 | 19 | slack_alert_template = "{template_open}*<{kibana_url}|{{{{ctx.metadata.name}}}}>* has `{{{{ctx.payload.aggregations.pods.value}}}}` not ready pod(s) <{watch_url}|[ack]>{{{{#ctx.metadata.docs}}}} <{{{{.}}}}|[docs]>{{{{/ctx.metadata.docs}}}}{template_close}".format(**locals()) 20 | email_alert_template = "{template_open}{{{{ctx.metadata.name}}}} has {{{{ctx.payload.aggregations.pods.value}}}} not ready pod(s) [ack]{{{{#ctx.metadata.docs}}}} [docs]{{{{/ctx.metadata.docs}}}}{template_close}".format(**locals()) 21 | 22 | k8s_template = { 23 | "metadata": { 24 | "name": "", 25 | "namespace": "", 26 | "regex": "", 27 | "kuberwatcher": "true", 28 | "kibana_url": "", 29 | "kibana_dashboard": "", 30 | "docs": "", 31 | "xpack" : { 32 | "type" : "json" 33 | }, 34 | }, 35 | "trigger": { 36 | "schedule": { 37 | "interval": "" 38 | } 39 | }, 40 | "input": { 41 | "search": { 42 | "request": { 43 | "search_type": "query_then_fetch", 44 | "indices": [ 45 | "metricbeat-*" 46 | ], 47 | "rest_total_hits_as_int": True, 48 | "body": { 49 | "aggs": { 50 | "result": { 51 | "top_hits": { 52 | "size": 1 53 | } 54 | }, 55 | "pods": { 56 | "cardinality": { 57 | "field": "kubernetes.pod.name" 58 | } 59 | }, 60 | "not_ready": { 61 | "terms": { 62 | "field": "kubernetes.pod.name", 63 | "min_doc_count": 12, 64 | "size": 100 65 | } 66 | } 67 | }, 68 | "query": { 69 | "bool": { 70 | "must_not": [], 71 | "must": [], 72 | "filter": [ 73 | { 74 | "range": { 75 | "@timestamp": { 76 | "gte": "now-{{ctx.metadata.window}}" 77 | } 78 | } 79 | } 80 | ] 81 | } 82 | } 83 | } 84 | } 85 | } 86 | }, 87 | "condition": {}, 88 | "actions": { 89 | "email_admin": { 90 | "throttle_period_in_millis": 300000, 91 | "email": { 92 | "profile": "standard", 93 | "subject": "{{#ctx.payload.aggregations.result.hits.hits.0._source}}{{ctx.metadata.name}} has {{ctx.payload.aggregations.pods.value}} not ready pod(s){{/ctx.payload.aggregations.result.hits.hits.0._source}}", 94 | "body": { 95 | "html": email_alert_template 96 | } 97 | } 98 | }, 99 | "notify-slack": { 100 | "throttle_period_in_millis": 300000, 101 | "slack": { 102 | "message": { 103 | "text": slack_alert_template 104 | } 105 | } 106 | } 107 | } 108 | } 109 | 110 | metricbeat_template = { 111 | "metadata": { 112 | "window": "300s", 113 | "subject": "No metricbeat data has been recieved in the last 5 minutes!" 114 | }, 115 | "trigger": { 116 | "schedule": { 117 | "interval": "60s" 118 | } 119 | }, 120 | "input": { 121 | "search": { 122 | "request": { 123 | "search_type": "query_then_fetch", 124 | "indices": [ 125 | "metricbeat-*" 126 | ], 127 | "rest_total_hits_as_int": True, 128 | "body": { 129 | "query": { 130 | "bool": { 131 | "must": [ 132 | { 133 | "match": { 134 | "metricset.name": "state_pod" 135 | } 136 | } 137 | ], 138 | "filter": [ 139 | { 140 | "range": { 141 | "@timestamp": { 142 | "gte": "now-{{ctx.metadata.window}}" 143 | } 144 | } 145 | } 146 | ] 147 | } 148 | } 149 | } 150 | } 151 | } 152 | }, 153 | "condition": { 154 | "compare": { 155 | "ctx.payload.hits.total": { 156 | "eq": 0 157 | } 158 | } 159 | }, 160 | "actions": { 161 | "email_admin": { 162 | "throttle_period_in_millis": 300000, 163 | "email": { 164 | "profile": "standard", 165 | "subject": "{{ctx.metadata.subject}}", 166 | "body": { 167 | "text": "{{ctx.metadata.message}}" 168 | } 169 | } 170 | }, 171 | "notify-slack": { 172 | "throttle_period_in_millis": 300000, 173 | "slack": { 174 | "message": { 175 | "text": "{{ctx.metadata.message}}" 176 | } 177 | } 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kuberwatcher 2 | 3 | [![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+kuberwatcher+master.svg)](https://devops-ci.elastic.co/job/elastic+kuberwatcher+master/) 4 | 5 | This is a small python script that will generate separate watches for each group of pods. 6 | If you have a namespace called `test` and a deployment called `nginx` and `elasticsearch` this script will automagically create watches called `test.nginx` and `test.elasticsearch` which will send email and or slack alerts if any of the pods in these deployments are not ready. 7 | 8 | # When is an alert sent? 9 | 10 | ## Deployments/Daemonsets/Statefulsets 11 | 12 | An alert will be sent if a single pod is not ready for 2 minutes in a 5 minute window. This can be increased or decreased by changing the `failures` settings. 13 | 14 | ## Cronjobs 15 | 16 | An alert will be sent if a cronjob fails 2 times in a row. This behavior can be changed by modifying the `job_failures` 17 | 18 | 19 | # Enabling monitoring 20 | 21 | Monitoring can be enabled on a namespace or pod level. To enable monitoring on the a namespace called `infra` you can add the label `watcher=enabled` 22 | ``` 23 | kubectl label namespace infra watcher=enabled 24 | ``` 25 | This can also be enabled directly inside the namespace definition 26 | ``` 27 | apiVersion: v1 28 | kind: Namespace 29 | metadata: 30 | name: infra 31 | labels: 32 | watcher: enabled 33 | ``` 34 | 35 | Enabling watcher for a single deployment in a namespace that isn't enabled 36 | ``` 37 | spec: 38 | template: 39 | metadata: 40 | labels: 41 | watcher: enabled 42 | ``` 43 | 44 | Disabling watcher for a single deployment in a namespace that is enabled 45 | ``` 46 | spec: 47 | template: 48 | metadata: 49 | labels: 50 | watcher: disabled 51 | ``` 52 | 53 | # Overriding monitoring defaults 54 | 55 | All of the defaults (read from `./kuberwatcher.yml`) can be overridden on a namespace and a pod level under the annotations metadata in the Kubernetes object 56 | 57 | Sending all slack alerts in the `michael` namespace to slack user `@michael.russell` 58 | 59 | ``` 60 | apiVersion: v1 61 | kind: Namespace 62 | metadata: 63 | name: michael 64 | labels: 65 | watcher: enabled 66 | annotations: 67 | watcher.alerts.slack: '@michael.russell' 68 | ``` 69 | 70 | Adding alert documentation links for pods in a deployment 71 | ``` 72 | spec: 73 | template: 74 | metadata: 75 | annotations: 76 | watcher.docs: https://github.com/elastic/kuberwatcher/blob/master/README.md 77 | ``` 78 | 79 | 80 | # Configuration 81 | 82 | ``` 83 | watcher.alerts.email: 'username@elastic.co,team@elastic.co' # Comma separated list of email addresses 84 | watcher.alerts.slack: '@michael.russell,infra' # Comma separated list of `@usernames` and `channelnames` 85 | watcher.kibana_url 'https://kibana.elastic.co' # Base URl for generating links to Kibana 86 | watcher.docs: 'https://example.com/HALP.md' # Documentation link that will be included in the alert 87 | watcher.failures: 12 # How many failed attempts need to have happened in the current window per pod (optional) 88 | watcher.job_failures: 2 # How many times a cronjob needs to fail in a row being an alert is sent (optional) 89 | watcher.interval: '30s' # How often the query is run (optional) 90 | watcher.reply_to 'team@elastic.co,reply@elastic.o' # Comma separated list of email addresses to use for the reply_to field in the email alerts 91 | watcher.throttle: 600000 # How often to send the alert in ms (optional) 92 | watcher.window: '300s' # How long the the pods need to be not ready before alerting (optional) 93 | watcher.metricbeat_index_pattern: 'metricbeat-*' # The Index Pattern to search for Metricbeat metrics (optional) 94 | ``` 95 | 96 | # Running Kuberwatcher 97 | 98 | ## Requirements 99 | 100 | * [Metricbeat](https://www.elastic.co/guide/en/beats/metricbeat/current/running-on-kubernetes.html) (with kube-state-metrics) deployed on kubernetes with the [`state_pod`](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-metricset-kubernetes-state_pod.html) metricset enabled. 101 | * Elasticsearch with X-Pack and watcher enabled 102 | * If you want to send emails you will need to configure an [email account](https://www.elastic.co/guide/en/elastic-stack-overview/current/actions-email.html#configuring-email) in X-Pack. 103 | * If you want to send slack alerts you will need to configure a [slack account](https://www.elastic.co/guide/en/elastic-stack-overview/current/actions-slack.html#configuring-slack) in X-Pack. 104 | 105 | ### Try kuberwatcher locally 106 | 107 | If you just want to try out kuberwatcher running locally it will use your current kubectl context to generate alerts. 108 | 109 | * First update the local kuberwatcher.yml 110 | * Set the elasticsearch connection details 111 | ``` 112 | export ES_USERNAME='elastic' 113 | export ES_PASSWORD='changeme' 114 | export ES_HOSTS='http://elasticsearch:9200' 115 | ``` 116 | * Run kuberwatcher in docker 117 | ``` 118 | make run 119 | ``` 120 | 121 | ### Run kuberwatcher in kubernetes 122 | 123 | There is an example in [k8s/kuberwatcher.yml](./k8s/kuberwatcher.yml) for running kuberwatcher as a CronJob in kubernetes. 124 | 125 | * Create the secrets for kuberwatcher to connect to Elasticsearch 126 | ``` 127 | kubectl create secret generic kuberwatcher --from-literal=endpoint=http://elasticsearch:9200 --from-literal=password=changeme --from-literal=username=elastic 128 | ``` 129 | * Modify the configuration in the configmap defined in [k8s/kuberwatcher.yml](./k8s/kuberwatcher.yml) 130 | * Deploy! 131 | ``` 132 | kubectl apply -f k8s/kuberwatcher.yml 133 | ``` 134 | 135 | # Developing kuberwatcher 136 | 137 | Requirements: 138 | * Docker 139 | * Make 140 | 141 | To run all of the tests: 142 | ``` 143 | make test 144 | ``` 145 | 146 | Because this project interacts with two fast moving projects (Elasticsearch and kubernetes) it uses [vcrpy](http://vcrpy.readthedocs.io/en/latest/usage.html) to record interactions with these APIs instead of trying to constantly update mocks for both projects. 147 | 148 | If you are making changes that affect API calls to kubernetes or Elasticsearch you will need to make sure you have a working development environment to record the new API transactions. To start Elasticsearch and Kibana you can run: 149 | ``` 150 | # if you want to test with slack you need to add your slack url too 151 | export SLACK_URL='https://hooks.slack.com/services/SDKLSD/SDFLIJSDF323f3f' 152 | make deps 153 | ``` 154 | 155 | To generate the test pods in a local minikube cluster you can run install the fixtures by running: 156 | 157 | ``` 158 | make fixtures 159 | ``` 160 | This will start metricbeat, kube-state-metrics and a bunch of tests pods which were used to generate the cassette data. 161 | 162 | If you want to refresh the recorded API data in the cassettes you can run: 163 | 164 | ``` 165 | make clean_cassettes 166 | ``` 167 | 168 | When you are finished you can clean up everything by running: 169 | 170 | ``` 171 | make clean 172 | ``` 173 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /kuberwatcher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from kubernetes import client as kube_client, config as kube_config 4 | from collections import defaultdict 5 | import json 6 | import urllib.parse 7 | from elasticsearch import Elasticsearch 8 | from elasticsearch.client.watcher import WatcherClient 9 | from elasticsearch.exceptions import NotFoundError 10 | from template import k8s_template, metricbeat_template 11 | import certifi 12 | import copy 13 | import os 14 | import yaml 15 | 16 | # Recursive defaultdict so we don't need to constantly check if key in dict before creating a nested dict 17 | tree = lambda: defaultdict(tree) 18 | 19 | def merge_defaults(defaults, config): 20 | new = defaults.copy() 21 | new.update(config) 22 | return(new) 23 | 24 | def unflatten(dictionary): 25 | if not dictionary: 26 | return {} 27 | resultDict = dict() 28 | for key, value in dictionary.items(): 29 | parts = key.split(".") 30 | d = resultDict 31 | for part in parts[:-1]: 32 | if part not in d: 33 | d[part] = dict() 34 | d = d[part] 35 | d[parts[-1]] = value 36 | return resultDict 37 | 38 | def render_template( 39 | name, 40 | regex, 41 | namespace, 42 | pod_type, 43 | alerts={}, 44 | docs='', 45 | kibana_url='https://kibana.example.com', 46 | failures=12, 47 | job_failures=2, 48 | interval='30s', 49 | reply_to=None, 50 | throttle=3600000, 51 | window='300s', 52 | metricbeat_index_pattern='metricbeat-*', 53 | ): 54 | 55 | 56 | template = copy.deepcopy(k8s_template) 57 | template['input']['search']['request']['body']['query']['bool']['must'].append({'regexp': {'kubernetes.pod.name': regex }}) 58 | template['input']['search']['request']['body']['query']['bool']['must'].append({'match': {'kubernetes.namespace': namespace}}) 59 | template['input']['search']['request']['body']['query']['bool']['must'].append({'match': {'metricset.name': 'state_pod'}}) 60 | 61 | if pod_type == 'job': 62 | # Only return the amount of failures we care about. With the default amount of 2 we will get 2 results. If all of these results have failed we will send an alert 63 | template['input']['search']['request']['body']['size'] = job_failures 64 | 65 | # Filter out running and pending jobs since we just want to find the latest completed results 66 | template['input']['search']['request']['body']['query']['bool']['must_not'].append({'match': {'kubernetes.pod.status.phase': 'running'}}) 67 | template['input']['search']['request']['body']['query']['bool']['must_not'].append({'match': {'kubernetes.pod.status.phase': 'pending'}}) 68 | 69 | # Cronjob pods contain a unix timestamp in their name. By sorting the pods by name we can get the most recent jobs at the top of the results 70 | template['input']['search']['request']['body']['sort'] = [{"kubernetes.pod.name" : {"order" : "desc", "mode" : "max"}}] 71 | template["condition"] = { 72 | "script": { 73 | "lang": "painless", 74 | # Don't alert if hits.hits does not have any results as this can produce false positives 75 | # If any of the jobs have suceeded don't send an alert. The 76 | # query returns the most recent job_failures amount of pods 77 | # (default 2). So if any of them passed we don't need to alert 78 | # yet. 79 | "source": "if (ctx.payload.hits.hits.size() == 0) return false; " 80 | + "for (h in ctx.payload.hits.hits) { " 81 | + "if (h._source.kubernetes.pod.status.phase == 'succeeded') return false; } " 82 | + "return true;", 83 | } 84 | } 85 | 86 | else: 87 | # Only look at running pods so that we don't accidentally match job pods 88 | template['input']['search']['request']['body']['query']['bool']['must'].append({'match': {'kubernetes.pod.status.phase': 'running'}}) 89 | 90 | template['input']['search']['request']['body']['query']['bool']['must_not'].append({'match': {'kubernetes.pod.status.ready': 'true'}}) 91 | # We want to alert if there are any not ready pods in this group 92 | template['condition'] = { 93 | "compare": { 94 | "ctx.payload.aggregations.not_ready.buckets": { 95 | "not_eq": [] 96 | } 97 | } 98 | } 99 | 100 | template['input']['search']['request']['body']['aggs']['not_ready']['terms']['min_doc_count'] = failures 101 | template['input']['search']['request']['indices'] = [ 102 | metricbeat_index_pattern 103 | ] 104 | template['trigger']['schedule']['interval'] = interval 105 | template['metadata']['name'] = name 106 | template['metadata']['namespace'] = namespace 107 | template['metadata']['window'] = window 108 | template['metadata']['kibana_url'] = kibana_url 109 | template['metadata']['docs'] = docs 110 | template['metadata']['regex'] = urllib.parse.quote_plus(regex) 111 | 112 | return add_alerts(template, alerts, throttle, reply_to) 113 | 114 | 115 | kind_dashes_map = { 116 | 'replicationcontroller': 2, 117 | 'replicaset': 2, 118 | 'statefulset': 1, 119 | 'daemonset': 1, 120 | 'job': 2, 121 | } 122 | 123 | def base_name(name, kind): 124 | if kind not in kind_dashes_map: 125 | return None 126 | return name.rsplit('-', kind_dashes_map[kind])[0] 127 | 128 | def add_alerts(template, alerts, throttle, reply_to=None): 129 | if 'email' in alerts: 130 | template['actions']['email_admin']['email']['to'] = alerts['email'].split(',') 131 | template['actions']['email_admin']['throttle_period_in_millis'] = throttle 132 | if reply_to: 133 | template['actions']['email_admin']['email']['reply_to'] = reply_to.split(',') 134 | else: 135 | del template['actions']['email_admin'] 136 | 137 | if 'slack' in alerts: 138 | template['actions']['notify-slack']['slack']['message']['to'] = alerts['slack'].split(',') 139 | template['actions']['notify-slack']['throttle_period_in_millis'] = throttle 140 | else: 141 | del template['actions']['notify-slack'] 142 | 143 | return template 144 | 145 | def get_all_pods(namespaces, defaults): 146 | v1 = kube_client.CoreV1Api() 147 | kinds = tree() 148 | ret = v1.list_pod_for_all_namespaces(watch=False,label_selector='watcher!=disabled') 149 | for i in ret.items: 150 | namespace = i.metadata.namespace 151 | # If this namespace doesn't have watcher alerts enabled we want to see if they are enabled on the pod itself 152 | if namespace not in namespaces: 153 | if i.metadata.labels and not i.metadata.labels.get('watcher') == 'enabled': 154 | continue 155 | name = i.metadata.name 156 | kind = None 157 | try: 158 | kind = i.metadata.owner_references[0].kind 159 | except: 160 | print('Could not determine the kind for:', name) 161 | 162 | if not kind: 163 | continue 164 | 165 | kind = kind.lower() 166 | pod_group_name = base_name(name, kind) 167 | if not pod_group_name: 168 | continue 169 | 170 | annotations = unflatten(i.metadata.annotations) 171 | config = merge_defaults(namespaces.get(namespace, defaults), annotations.get('watcher', {})) 172 | kinds[kind][namespace][pod_group_name] = config 173 | 174 | return kinds 175 | 176 | 177 | def pod_regex(name, kind): 178 | dashes = kind_dashes_map[kind] 179 | dash_regex = '-[^-]+' * dashes 180 | return '{name}{dash_regex}'.format(**locals()) 181 | 182 | 183 | def full_name(pod, namespace): 184 | return '{0}.{1}'.format(namespace, pod) 185 | 186 | 187 | def generate_watch(pods): 188 | watches = {} 189 | for pod_type, pods in pods.items(): 190 | for namespace, groups in pods.items(): 191 | for pod, config in groups.items(): 192 | name = full_name(pod, namespace) 193 | config = merge_defaults(config, {'name': name, 'namespace': namespace, 'regex': pod_regex(pod, pod_type), 'pod_type': pod_type}) 194 | watch = render_template(**config) 195 | watches[name] = watch 196 | return watches 197 | 198 | 199 | def get_namespaces(defaults): 200 | v1 = kube_client.CoreV1Api() 201 | namespaces = {} 202 | for ns in v1.list_namespace(label_selector='watcher=enabled').items: 203 | if ns.metadata.annotations: 204 | annotations = unflatten(ns.metadata.annotations) 205 | config = merge_defaults(defaults, annotations.get('watcher',{})) 206 | else: 207 | # Older versions of kubernetes don't always have annotations by default 208 | # We are only testing against the latest version so path doesn't occur 209 | # with the latest kubernetes version 210 | config = defaults # pragma: nocover 211 | namespaces[ns.metadata.name] = config 212 | return namespaces 213 | 214 | def load_config(): # pragma: nocover 215 | if os.environ.get('CI') == 'true': 216 | return 217 | elif os.path.exists('/run/secrets/kubernetes.io/serviceaccount'): 218 | kube_config.load_incluster_config() 219 | else: 220 | kube_config.load_kube_config() 221 | 222 | def get_current_watches(es): 223 | watcher_client = WatcherClient(es) 224 | 225 | watches = {} 226 | 227 | for watch in watcher_client.query_watches(body={'size': 1000})['watches']: 228 | watches[watch['_id']] = watch['watch'] 229 | 230 | return watches 231 | 232 | def watch_changed(watch, template, watches): 233 | if watch in watches: 234 | changed = json.dumps(watches[watch], sort_keys=True) != json.dumps(template, sort_keys=True) 235 | if not changed: 236 | print('Skipping: {0}'.format(watch)) 237 | else: 238 | print('Updating {0}'.format(watch)) 239 | return changed 240 | else: 241 | print('Creating: {0}'.format(watch)) 242 | return True 243 | 244 | def send_watches(watches, current_watches, es): 245 | watcher_client = WatcherClient(es) 246 | updated = [] 247 | for watch, template in watches.items(): 248 | if watch_changed(watch, template, current_watches): 249 | watcher_client.put_watch(id=watch, body=template) 250 | updated.append(watch) 251 | return updated 252 | 253 | def delete_watches(watches, es): # pragma: nocover 254 | watcher_client = WatcherClient(es) 255 | for watch in watches: 256 | print('Removing: {0}'.format(watch)) 257 | watcher_client.delete_watch(id=watch) 258 | 259 | def find_old_watches(watches, current_watches): 260 | old_watches = [] 261 | for watch, template in current_watches.items(): 262 | # Skip Watches that we found this run 263 | if watch in watches: 264 | continue 265 | 266 | # Skip any watches that have no metadata 267 | if "metadata" not in template: 268 | continue 269 | 270 | # Remove Kuberwatcher managed Watches that are no longer needed 271 | if template["metadata"].get("kuberwatcher") == "true": 272 | old_watches.append(watch) 273 | continue 274 | 275 | # Fuzzy detection to clean up Watchers managed by older versions of Kuberwatcher 276 | if "regex" not in template["metadata"]: 277 | continue 278 | 279 | try: 280 | if ( 281 | "not ready pod(s) <" 282 | in template["actions"]["notify-slack"]["slack"]["message"]["text"] 283 | ): 284 | old_watches.append(watch) 285 | continue 286 | except KeyError: 287 | pass 288 | 289 | try: 290 | if ( 291 | "not ready pod(s) <" 292 | in template["actions"]["email_admin"]["body"]["html"] 293 | ): 294 | old_watches.append(watch) 295 | continue 296 | except KeyError: 297 | pass 298 | 299 | return old_watches 300 | 301 | def es_connection_config(): 302 | es_hosts = os.environ.get('ES_HOSTS', 'http://elasticsearch:9200') 303 | es_client_cert_path = os.environ.get('ES_CLIENT_CERT_PATH') 304 | ca_cert_path = os.environ.get('ES_CA_CERTS', certifi.where()) 305 | 306 | es_client_kwargs = dict( 307 | http_auth=( 308 | os.environ.get('ES_USERNAME','elastic'), 309 | os.environ.get('ES_PASSWORD','changeme') 310 | ), 311 | ca_certs=ca_cert_path 312 | ) 313 | 314 | if es_client_cert_path: 315 | es_client_kwargs.pop('http_auth') 316 | es_client_kwargs['client_cert'] = es_client_cert_path 317 | es_client_kwargs['client_key'] = os.environ.get('ES_CLIENT_KEY_PATH') 318 | 319 | return es_hosts, es_client_kwargs 320 | 321 | def connect_to_es(): 322 | es_hosts, es_client_kwargs = es_connection_config() 323 | return Elasticsearch([es_hosts], **es_client_kwargs) 324 | 325 | def main(es, defaults): 326 | load_config() 327 | namespaces = get_namespaces(defaults) 328 | pods = get_all_pods(namespaces, defaults) 329 | watches = generate_watch(pods) 330 | 331 | watches['metricbeat'] = add_alerts(metricbeat_template, defaults['alerts'], defaults.get('throttle', 3600000), defaults.get('reply_to', None)) 332 | watches['metricbeat']['metadata']['message'] = 'No metricbeat data has been recieved in the last 5 minutes! <{0}|kibana>'.format(defaults['kibana_url']) 333 | watches['metricbeat']['input']['search']['request']['indices'] = [ 334 | defaults.get('metricbeat_index_pattern', 'metricbeat-*') 335 | ] 336 | return watches 337 | 338 | if __name__ == "__main__": # pragma: nocover 339 | es = connect_to_es() 340 | defaults = unflatten(yaml.full_load(open('kuberwatcher.yml'))) 341 | current_watches = get_current_watches(es) 342 | watches = main(es, defaults) 343 | send_watches(watches, current_watches, es) 344 | old_watches = find_old_watches(watches, current_watches) 345 | delete_watches(old_watches, es) 346 | -------------------------------------------------------------------------------- /tests/kuberwatcher_test.py: -------------------------------------------------------------------------------- 1 | from kuberwatcher import * 2 | import chevron 3 | import pytest 4 | import vcr 5 | import certifi 6 | 7 | my_vcr = vcr.VCR( 8 | cassette_library_dir='tests/cassettes', 9 | record_mode='new_episodes', 10 | filter_headers=['authorization'], 11 | match_on=['path', 'query'] 12 | ) 13 | 14 | def mustache_render(template, event): 15 | return chevron.render(template, {'ctx':{'payload':{'aggregations': {'result': {'hits': {'hits': {'0': {'_source': event }}}}}}}}) 16 | 17 | def test_template_defaults_with_no_outputs(): 18 | watch = { 19 | 'name': 'namespace.podgroup', 20 | 'regex': 'pod-.*', 21 | 'pod_type': 'replicaset', 22 | 'namespace': 'namespace' 23 | } 24 | template = render_template(**watch) 25 | assert template['metadata']['window'] == '300s' 26 | assert template['input']['search']['request']['body']['query']['bool']['must'][0]['regexp']['kubernetes.pod.name'] == 'pod-.*' 27 | assert template['input']['search']['request']['body']['query']['bool']['must'][1]['match']['kubernetes.namespace'] == 'namespace' 28 | 29 | def test_template_with_job_pod_type(): 30 | watch = { 31 | 'name': 'namespace.podgroup', 32 | 'regex': 'pod-.*', 33 | 'pod_type': 'job', 34 | 'namespace': 'namespace' 35 | } 36 | template = render_template(**watch) 37 | assert template['input']['search']['request']['body']['query']['bool']['must_not'][0]['match'] == {'kubernetes.pod.status.phase': 'running'} 38 | assert template['input']['search']['request']['body']['query']['bool']['must_not'][1]['match'] == {'kubernetes.pod.status.phase': 'pending'} 39 | assert template['input']['search']['request']['body']['sort'] == [{"kubernetes.pod.name" : {"order" : "desc", "mode" : "max"}}] 40 | 41 | def test_template_that_isnt_a_job(): 42 | watch = { 43 | 'name': 'namespace.podgroup', 44 | 'regex': 'pod-.*', 45 | 'pod_type': 'replicaset', 46 | 'namespace': 'namespace' 47 | } 48 | template = render_template(**watch) 49 | assert template['input']['search']['request']['body']['query']['bool']['must_not'][0]['match'] == {'kubernetes.pod.status.ready': 'true'} 50 | 51 | def test_unflattening_a_yaml_config_from_kubernetes(): 52 | config = ''' 53 | watcher.hello: 'hello' 54 | watcher.goodbye: 'goodbye' 55 | watcher.alerts.slack: 'slack' 56 | watcher.alerts.email: 'email' 57 | ''' 58 | config = unflatten(yaml.safe_load(config)) 59 | assert config['watcher']['hello'] == 'hello' 60 | assert config['watcher']['goodbye'] == 'goodbye' 61 | assert config['watcher']['alerts']['slack'] == 'slack' 62 | assert config['watcher']['alerts']['email'] == 'email' 63 | 64 | def test_merging_all_defaults(): 65 | defaults = {'hello': 'world'} 66 | new = {} 67 | 68 | assert merge_defaults(defaults, new) == defaults 69 | 70 | def test_overriding_merged_default(): 71 | defaults = {'hello': 'world'} 72 | new = {'name': 'monitor', 'hello': 'mars'} 73 | 74 | assert merge_defaults(defaults, new) == new 75 | 76 | def test_merging_in_a_new_field(): 77 | defaults = {'hello': 'world'} 78 | new = {'name': 'monitor', 'test': 'mars'} 79 | 80 | assert merge_defaults(defaults, new) == {'name': 'monitor', 'test': 'mars', 'hello': 'world'} 81 | 82 | def test_defaults_are_cloned_properly(): 83 | defaults = {'hello': 'world'} 84 | new = {'name': 'monitor', 'test': 'mars'} 85 | assert merge_defaults(defaults, new) == {'name': 'monitor', 'test': 'mars', 'hello': 'world'} 86 | new2 = {'name': 'monitor2'} 87 | assert merge_defaults(defaults, new2) == {'name': 'monitor2', 'hello': 'world'} 88 | 89 | def test_slack_mustache_template(): 90 | watch = { 91 | 'name': 'namespace.podgroup', 92 | 'regex': 'pod-.*', 93 | 'namespace': 'namespace', 94 | 'pod_type': 'replicaset', 95 | 'alerts': { 96 | 'slack': '@username' 97 | } 98 | } 99 | event = { 100 | 'ctx': { 101 | 'metadata': { 102 | 'kibana_url': 'https://kibana.com', 103 | 'name': 'namespace.podgroup' 104 | }, 105 | 'payload': { 106 | 'aggregations': { 107 | 'pods': { 108 | 'value': 1 109 | } 110 | } 111 | } 112 | } 113 | } 114 | template = render_template(**watch) 115 | result = mustache_render(template['actions']['notify-slack']['slack']['message']['text'], event) 116 | assert result == "** has `1` not ready pod(s) " 117 | 118 | def test_conditionally_adding_docs_field_for_slack(): 119 | watch = { 120 | 'name': 'namespace.podgroup', 121 | 'regex': 'pod-.*', 122 | 'namespace': 'namespace', 123 | 'pod_type': 'replicaset', 124 | 'alerts': { 125 | 'slack': '@username' 126 | } 127 | } 128 | event = { 129 | 'ctx': { 130 | 'metadata': { 131 | 'docs': 'https://docs.com/doc' 132 | } 133 | } 134 | } 135 | template = render_template(**watch) 136 | result = mustache_render(template['actions']['notify-slack']['slack']['message']['text'], event) 137 | assert '' in result 138 | 139 | def test_customizing_the_metricbeat_index_pattern(): 140 | watch = { 141 | 'name': 'namespace.podgroup', 142 | 'regex': 'pod-.*', 143 | 'namespace': 'namespace', 144 | 'pod_type': 'replicaset', 145 | 'alerts': { 146 | 'slack': '@username' 147 | }, 148 | 'metricbeat_index_pattern': 'some-other-index-pattern' 149 | } 150 | 151 | template = render_template(**watch) 152 | metricbeat_index_pattern = template['input']['search']['request']['indices'][0] 153 | assert metricbeat_index_pattern == 'some-other-index-pattern' 154 | 155 | def test_conditionally_not_adding_docs_field_for_slack(): 156 | watch = { 157 | 'name': 'namespace.podgroup', 158 | 'regex': 'pod-.*', 159 | 'namespace': 'namespace', 160 | 'pod_type': 'replicaset', 161 | 'alerts': { 162 | 'slack': '@username' 163 | } 164 | } 165 | event = { 166 | 'ctx': { 167 | 'metadata': { 168 | 'docs': '' 169 | } 170 | } 171 | } 172 | template = render_template(**watch) 173 | result = mustache_render(template['actions']['notify-slack']['slack']['message']['text'], event) 174 | assert '[docs]' not in result 175 | 176 | def test_email_mustache_template(): 177 | watch = { 178 | 'name': 'namespace.podgroup', 179 | 'regex': 'pod-.*', 180 | 'namespace': 'namespace', 181 | 'pod_type': 'replicaset', 182 | 'alerts': { 183 | 'email': 'username@email.com' 184 | } 185 | } 186 | event = { 187 | 'ctx': { 188 | 'metadata': { 189 | 'kibana_url': 'https://kibana.com', 190 | 'name': 'namespace.podgroup' 191 | }, 192 | 'payload': { 193 | 'aggregations': { 194 | 'pods': { 195 | 'value': 1 196 | } 197 | } 198 | } 199 | } 200 | } 201 | template = render_template(**watch) 202 | result = mustache_render(template['actions']['email_admin']['email']['body']['html'], event) 203 | expected = '''namespace.podgroup has 1 not ready pod(s) [ack]'''.rstrip() 204 | assert result == expected 205 | 206 | def test_conditionally_adding_docs_field_for_email(): 207 | watch = { 208 | 'name': 'namespace.podgroup', 209 | 'regex': 'pod-.*', 210 | 'namespace': 'namespace', 211 | 'pod_type': 'replicaset', 212 | 'alerts': { 213 | 'email': 'kuberwatcher-alerts@example.com' 214 | } 215 | } 216 | event = { 217 | 'ctx': { 218 | 'metadata': { 219 | 'docs': 'https://docs.com/doc' 220 | } 221 | } 222 | } 223 | template = render_template(**watch) 224 | result = mustache_render(template['actions']['email_admin']['email']['body']['html'], event) 225 | assert '[docs]' in result 226 | 227 | def test_conditionally_not_adding_docs_field_for_email(): 228 | watch = { 229 | 'name': 'namespace.podgroup', 230 | 'regex': 'pod-.*', 231 | 'namespace': 'namespace', 232 | 'pod_type': 'replicaset', 233 | 'alerts': { 234 | 'email': 'kuberwatcher-alerts@example.com' 235 | } 236 | } 237 | event = { 238 | 'ctx': { 239 | 'metadata': { 240 | 'docs': '' 241 | } 242 | } 243 | } 244 | template = render_template(**watch) 245 | result = mustache_render(template['actions']['email_admin']['email']['body']['html'], event) 246 | assert '[docs]' not in result 247 | 248 | def test_kubernetes_base_name(): 249 | assert base_name('test-nginx-b4cc76b85-4f588', 'replicaset') == 'test-nginx' 250 | assert base_name('test-nginx-0', 'statefulset') == 'test-nginx' 251 | assert base_name('metricbeat-jlrb7', 'daemonset') == 'metricbeat' 252 | assert base_name('vault-backup-vault-poc-vault-test-1514995200-c4rch', 'job') == 'vault-backup-vault-poc-vault-test' 253 | 254 | def test_kubernetes_base_name_with_unsupported_kind(): 255 | assert base_name('fakeity-fake-fake', 'challenge') == None 256 | 257 | 258 | def test_generating_pod_regex(): 259 | assert pod_regex('test-nginx', 'replicaset') == 'test-nginx-[^-]+-[^-]+' 260 | assert pod_regex('test-nginx', 'statefulset') == 'test-nginx-[^-]+' 261 | assert pod_regex('metricbeat', 'daemonset') == 'metricbeat-[^-]+' 262 | assert pod_regex('vault-backup-vault-poc-vault-test', 'job') == 'vault-backup-vault-poc-vault-test-[^-]+-[^-]+' 263 | 264 | def test_generating_full_pod_name(): 265 | assert full_name('pod', 'namespace') == 'namespace.pod' 266 | 267 | @my_vcr.use_cassette() 268 | def test_getting_all_namespace(): 269 | load_config() 270 | defaults = {} 271 | namespaces = get_namespaces(defaults) 272 | assert 'test' in namespaces 273 | 274 | @my_vcr.use_cassette() 275 | def test_getting_namespace_with_overriden_config(): 276 | load_config() 277 | defaults = { 278 | 'interval': '30s', 279 | 'throttle': 360000, 280 | 'window': '300s', 281 | 'failures': 3, 282 | 'alerts': { 283 | 'email': 'kuberwatcher-alerts@example.com', 284 | 'slack': '@michael.russell' 285 | } 286 | } 287 | namespaces = get_namespaces(defaults) 288 | assert namespaces['override']['alerts']['slack'] == '@michael.override' 289 | 290 | @my_vcr.use_cassette() 291 | def test_getting_email_alerts_disabled_when_overriding_alerts(): 292 | load_config() 293 | defaults = { 294 | 'alerts': { 295 | 'email': 'kuberwatcher-alerts@example.com', 296 | 'slack': '@michael.russell' 297 | } 298 | } 299 | namespaces = get_namespaces(defaults) 300 | assert namespaces['override']['alerts']['slack'] == '@michael.override' 301 | assert 'email' not in namespaces['override']['alerts'] 302 | 303 | @my_vcr.use_cassette() 304 | def test_get_all_pods(): 305 | namespaces = {'test': {}} 306 | pods = get_all_pods(namespaces, {}) 307 | assert 'nginx' in pods['replicaset']['test'] 308 | 309 | @my_vcr.use_cassette() 310 | def test_get_all_pods_with_unsupported_kind(): 311 | namespaces = {'test': {}} 312 | pods = get_all_pods(namespaces, {}) 313 | assert 'challenge' not in pods 314 | 315 | @my_vcr.use_cassette() 316 | def test_get_all_pods_with_global_defaults(): 317 | defaults = {"kibana_url": "https://kibana.custom.com"} 318 | namespaces = {} 319 | pods = get_all_pods(namespaces, defaults) 320 | assert pods['replicaset']['disabled']['nginx']['kibana_url'] == defaults['kibana_url'] 321 | 322 | @my_vcr.use_cassette() 323 | def test_get_all_pods_including_jobs(): 324 | namespaces = {'test': {}} 325 | pods = get_all_pods(namespaces, {}) 326 | assert 'replicaset' in pods 327 | assert 'job' in pods 328 | 329 | @my_vcr.use_cassette() 330 | def test_get_watcher_enabled_without_namespace_enabled(): 331 | namespaces = {} 332 | pods = get_all_pods(namespaces, {}) 333 | assert 'nginx' in pods['replicaset']['disabled'] 334 | assert 'disabled' not in pods['replicaset']['disabled'] 335 | 336 | @my_vcr.use_cassette() 337 | def test_get_all_pods_with_pods_that_dont_have_created_by(): 338 | namespaces = {'test': {}} 339 | pods = get_all_pods(namespaces, {}) 340 | assert 'nginx-pod' not in pods['replicaset']['test'] 341 | 342 | def test_generate_watch(): 343 | pods = { 344 | 'replicaset': { 345 | 'test': { 346 | 'nginx': {} 347 | } 348 | } 349 | } 350 | watches = generate_watch(pods) 351 | assert watches['test.nginx']['metadata']['name'] == 'test.nginx' 352 | assert watches['test.nginx']['metadata']['namespace'] == 'test' 353 | assert watches['test.nginx']['metadata']['regex'].startswith('nginx') 354 | 355 | def test_watch_that_didnt_change(): 356 | watch = 'test' 357 | template = {'hello': 'world'} 358 | watches = {'test': {'hello': 'world'}} 359 | assert watch_changed(watch, template, watches) == False 360 | 361 | def test_watch_that_changed(): 362 | watch = 'test' 363 | template = {'hello': 'world2'} 364 | watches = {'test': {'hello': 'world'}} 365 | assert watch_changed(watch, template, watches) == True 366 | 367 | def test_watch_that_didnt_exist(): 368 | watch = 'new' 369 | template = {'hello': 'world'} 370 | watches = {'test': {'hello': 'world'}} 371 | assert watch_changed(watch, template, watches) == True 372 | 373 | @my_vcr.use_cassette() 374 | def test_real_watch_that_did_actually_change(): 375 | defaults = { 376 | "kibana_url": "https://kibana.example.com", 377 | "alerts": { 378 | "email": "kuberwatcher-alerts@example.com", 379 | "slack": "@michael.russell", 380 | }, 381 | "failures": 3, 382 | "interval": "30s", 383 | "throttle": 360000, 384 | "window": "300s" 385 | } 386 | es = connect_to_es() 387 | watches = main(es, defaults) 388 | watch = 'test.nginx' 389 | current_watches = get_current_watches(es) 390 | template = watches[watch] 391 | template['metadata']['name'] = 'changed' 392 | assert watch_changed(watch, template, current_watches) == True 393 | 394 | @my_vcr.use_cassette() 395 | def test_sending_a_watch_to_watcher(): 396 | defaults = { 397 | "kibana_url": "https://kibana.example.com", 398 | "alerts": { 399 | "email": "kuberwatcher-alerts@example.com", 400 | "slack": "@michael.russell" 401 | } 402 | } 403 | es = connect_to_es() 404 | current_watches = get_current_watches(es) 405 | watches = main(es, defaults) 406 | updated = send_watches(watches, current_watches, es) 407 | assert len(updated) != 0 408 | assert 'test.nginx' in watches 409 | assert 'metricbeat' in watches 410 | assert watches['metricbeat']['metadata']['message'] == 'No metricbeat data has been recieved in the last 5 minutes! ' 411 | assert watches['metricbeat']['actions']['email_admin']['email']['to'] == ['kuberwatcher-alerts@example.com'] 412 | assert watches['metricbeat']['actions']['email_admin']['throttle_period_in_millis'] == 3600000 413 | 414 | # When sending the watches again they should not be updated 415 | current_watches = get_current_watches(es) 416 | updated = send_watches(watches, current_watches, es) 417 | assert len(updated) == 0 418 | 419 | def test_add_alerts(): 420 | alerts = { 421 | "email": "kuberwatcher-alerts@example.com,kuberwatcher-other-team@example.com", 422 | "slack": "@michael.russell,@micky" 423 | } 424 | result = add_alerts(copy.deepcopy(metricbeat_template), alerts, 0) 425 | assert result['actions']['email_admin']['email']['to'] == ['kuberwatcher-alerts@example.com', 'kuberwatcher-other-team@example.com'] 426 | assert result['actions']['notify-slack']['slack']['message']['to'] == ['@michael.russell', '@micky'] 427 | 428 | def test_add_alerts_with_only_slack(): 429 | alerts = { 430 | "slack": "@michael.russell" 431 | } 432 | result = add_alerts(copy.deepcopy(metricbeat_template), alerts, 0) 433 | assert 'email_admin' not in result['actions'] 434 | assert result['actions']['notify-slack']['slack']['message']['to'] == ['@michael.russell'] 435 | 436 | def test_add_alerts_with_only_email(): 437 | alerts = { 438 | "email": "kuberwatcher-alerts@example.com", 439 | } 440 | result = add_alerts(copy.deepcopy(metricbeat_template), alerts, 0) 441 | assert 'notify-slack' not in result['actions'] 442 | assert result['actions']['email_admin']['email']['to'] == ['kuberwatcher-alerts@example.com'] 443 | 444 | def test_add_alerts_with_reply_to(): 445 | alerts = { 446 | "email": "kuberwatcher-alerts@example.com", 447 | } 448 | result = add_alerts(copy.deepcopy(metricbeat_template), alerts, 0, 'reply@elastic.co') 449 | assert 'notify-slack' not in result['actions'] 450 | assert result['actions']['email_admin']['email']['to'] == ['kuberwatcher-alerts@example.com'] 451 | assert result['actions']['email_admin']['email']['reply_to'] == ['reply@elastic.co'] 452 | 453 | def test_add_alerts_with_overriden_throttle_period(): 454 | alerts = { 455 | "email": "kuberwatcher-alerts@example.com" 456 | } 457 | result = add_alerts(copy.deepcopy(metricbeat_template), alerts, 123456) 458 | assert result['actions']['email_admin']['throttle_period_in_millis'] == 123456 459 | 460 | def test_es_client_config_with_client_cert_path(monkeypatch): 461 | mock_client_cert_path = 'path/to/client.pem' 462 | monkeypatch.setitem(os.environ, 'ES_CLIENT_CERT_PATH', mock_client_cert_path) 463 | es_hosts, es_client_kwargs = es_connection_config() 464 | 465 | assert es_client_kwargs.get('client_cert') == mock_client_cert_path 466 | assert es_client_kwargs.get('client_key') == None 467 | assert es_client_kwargs.get('http_auth') == None 468 | 469 | def test_es_client_config_with_client_cert_and_key_path(monkeypatch): 470 | mock_client_cert_path = '/path/to/client.pem' 471 | mock_client_key_path = '/path/to/client.key' 472 | monkeypatch.setitem(os.environ, 'ES_CLIENT_CERT_PATH', mock_client_cert_path) 473 | monkeypatch.setitem(os.environ, 'ES_CLIENT_KEY_PATH', mock_client_key_path) 474 | es_hosts, es_client_kwargs = es_connection_config() 475 | 476 | assert es_client_kwargs.get('client_cert') == mock_client_cert_path 477 | assert es_client_kwargs.get('client_key') == mock_client_key_path 478 | assert es_client_kwargs.get('http_auth') == None 479 | 480 | def test_es_client_config_without_ca_certs_set(): 481 | expected_ca_cert_path = certifi.where() 482 | es_hosts, es_client_kwargs = es_connection_config() 483 | 484 | assert es_client_kwargs.get('ca_certs') == expected_ca_cert_path 485 | 486 | def test_es_client_config_with_ca_certs_set(monkeypatch): 487 | mock_ca_cert_path = '/path/to/ca.pem' 488 | monkeypatch.setitem(os.environ, 'ES_CA_CERTS', mock_ca_cert_path) 489 | es_hosts, es_client_kwargs = es_connection_config() 490 | 491 | assert es_client_kwargs.get('ca_certs') == mock_ca_cert_path 492 | 493 | def test_find_old_watch(): 494 | watches = {"latest": {}} 495 | current_watches = { 496 | "old": {"metadata": {"kuberwatcher": "true"}}, 497 | "latest": {}, 498 | } 499 | 500 | old_watches = find_old_watches(watches, current_watches) 501 | 502 | assert old_watches == ["old"] 503 | 504 | 505 | def test_find_legacy_old_watch_email(): 506 | watches = {"latest": {}} 507 | current_watches = { 508 | "old": { 509 | "metadata": {"regex": "test"}, 510 | "actions": { 511 | "email_admin": { 512 | "body": {"html": "Kuberwatcher found 3 not ready pod(s) < just now"} 513 | } 514 | }, 515 | }, 516 | "latest": {}, 517 | } 518 | 519 | old_watches = find_old_watches(watches, current_watches) 520 | 521 | assert old_watches == ["old"] 522 | 523 | def test_find_legacy_old_watch_slack(): 524 | watches = {"latest": {}} 525 | current_watches = { 526 | "old": { 527 | "metadata": {"regex": "test"}, 528 | "actions": { 529 | "notify-slack": { 530 | "slack": { 531 | "message": { 532 | "text": "Kuberwatcher found 3 not ready pod(s) < just now" 533 | } 534 | } 535 | } 536 | }, 537 | }, 538 | "latest": {}, 539 | } 540 | 541 | old_watches = find_old_watches(watches, current_watches) 542 | 543 | assert old_watches == ["old"] 544 | 545 | def test_find_old_watch_no_metadata(): 546 | watches = {"latest": {}} 547 | current_watches = { 548 | "old": { 549 | "actions": { 550 | "notify-slack": { 551 | "slack": {"message": {"text": "Important watcher alert!"}} 552 | } 553 | }, 554 | }, 555 | "latest": {}, 556 | } 557 | 558 | old_watches = find_old_watches(watches, current_watches) 559 | 560 | assert old_watches == [] 561 | 562 | def test_find_old_watch_not_our_metadata(): 563 | watches = {"latest": {}} 564 | current_watches = { 565 | "old": { 566 | "metadata": { 567 | "test": "hello", 568 | }, 569 | "actions": { 570 | "notify-slack": { 571 | "slack": {"message": {"text": "Important watcher alert!"}} 572 | } 573 | }, 574 | }, 575 | "latest": {}, 576 | } 577 | 578 | old_watches = find_old_watches(watches, current_watches) 579 | 580 | assert old_watches == [] 581 | 582 | def test_find_old_watch_without_our_alert_actions(): 583 | watches = {"latest": {}} 584 | current_watches = { 585 | "old": { 586 | "metadata": {"regex": "test"}, 587 | }, 588 | "latest": {}, 589 | } 590 | 591 | old_watches = find_old_watches(watches, current_watches) 592 | 593 | assert old_watches == [] 594 | -------------------------------------------------------------------------------- /tests/cassettes/test_get_all_pods: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: null 4 | headers: 5 | Accept: 6 | - application/json 7 | Content-Type: 8 | - application/json 9 | User-Agent: 10 | - OpenAPI-Generator/25.3.0/python 11 | method: GET 12 | uri: https://192.168.49.2:8443/api/v1/pods?labelSelector=watcher%21%3Ddisabled&watch=False 13 | response: 14 | body: 15 | string: '{"kind":"PodList","apiVersion":"v1","metadata":{"resourceVersion":"22851"},"items":[{"metadata":{"name":"disabled-6c56767ff7-mj65p","generateName":"disabled-6c56767ff7-","namespace":"disabled","uid":"962a1bce-ef74-4cce-a45d-3eb7fbc088cc","resourceVersion":"22720","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"pod-template-hash":"6c56767ff7","run":"disabled"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"disabled-6c56767ff7","uid":"ccf37402-64fc-4a16-91ae-238ba10babf4","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:pod-template-hash":{},"f:run":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"ccf37402-64fc-4a16-91ae-238ba10babf4\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:08Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.218\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-hjlhb","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"nginx","image":"nginx","resources":{},"volumeMounts":[{"name":"kube-api-access-hjlhb","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:08Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:08Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.218","podIPs":[{"ip":"10.244.0.218"}],"startTime":"2023-04-25T20:30:03Z","containerStatuses":[{"name":"nginx","state":{"running":{"startedAt":"2023-04-25T20:30:08Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"nginx:latest","imageID":"docker-pullable://nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94","containerID":"docker://cae62637648ed374d96ba67d406c2dff5504fbde215934c1138108c0a0a8f4ae","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"nginx-857cbb7658-xcmgt","generateName":"nginx-857cbb7658-","namespace":"disabled","uid":"7a2c007a-bcf0-4acd-bb70-da6c7bfde728","resourceVersion":"22713","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"pod-template-hash":"857cbb7658","run":"nginx","watcher":"enabled"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"nginx-857cbb7658","uid":"d2e0e2fb-68cd-4877-adc8-fc436870f6af","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:pod-template-hash":{},"f:run":{},"f:watcher":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d2e0e2fb-68cd-4877-adc8-fc436870f6af\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:07Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.216\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-phqlr","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"nginx","image":"nginx","resources":{},"volumeMounts":[{"name":"kube-api-access-phqlr","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:07Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:07Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.216","podIPs":[{"ip":"10.244.0.216"}],"startTime":"2023-04-25T20:30:03Z","containerStatuses":[{"name":"nginx","state":{"running":{"startedAt":"2023-04-25T20:30:07Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"nginx:latest","imageID":"docker-pullable://nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94","containerID":"docker://7eb3b99f2ceabe716d569317d964ea4be011ed72dd621a7aa28ea96fc0f3612a","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"coredns-787d4945fb-jwbw9","generateName":"coredns-787d4945fb-","namespace":"kube-system","uid":"b6156261-f50c-4310-aea2-6a0090b8d2f2","resourceVersion":"365","creationTimestamp":"2023-04-25T16:50:23Z","labels":{"k8s-app":"kube-dns","pod-template-hash":"787d4945fb"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"coredns-787d4945fb","uid":"00f49a5b-a3af-4bb1-b1dd-1eacc7a981dc","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:23Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:k8s-app":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"00f49a5b-a3af-4bb1-b1dd-1eacc7a981dc\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:podAntiAffinity":{".":{},"f:preferredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"coredns\"}":{".":{},"f:args":{},"f:image":{},"f:imagePullPolicy":{},"f:livenessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":53,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:name":{},"f:protocol":{}},"k:{\"containerPort\":53,\"protocol\":\"UDP\"}":{".":{},"f:containerPort":{},"f:name":{},"f:protocol":{}},"k:{\"containerPort\":9153,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:name":{},"f:protocol":{}}},"f:readinessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:limits":{".":{},"f:memory":{}},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:allowPrivilegeEscalation":{},"f:capabilities":{".":{},"f:add":{},"f:drop":{}},"f:readOnlyRootFilesystem":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/coredns\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"config-volume\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:items":{},"f:name":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:26Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"config-volume","configMap":{"name":"coredns","items":[{"key":"Corefile","path":"Corefile"}],"defaultMode":420}},{"name":"kube-api-access-vg2xw","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"coredns","image":"registry.k8s.io/coredns/coredns:v1.9.3","args":["-conf","/etc/coredns/Corefile"],"ports":[{"name":"dns","containerPort":53,"protocol":"UDP"},{"name":"dns-tcp","containerPort":53,"protocol":"TCP"},{"name":"metrics","containerPort":9153,"protocol":"TCP"}],"resources":{"limits":{"memory":"170Mi"},"requests":{"cpu":"100m","memory":"70Mi"}},"volumeMounts":[{"name":"config-volume","readOnly":true,"mountPath":"/etc/coredns"},{"name":"kube-api-access-vg2xw","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"livenessProbe":{"httpGet":{"path":"/health","port":8080,"scheme":"HTTP"},"initialDelaySeconds":60,"timeoutSeconds":5,"periodSeconds":10,"successThreshold":1,"failureThreshold":5},"readinessProbe":{"httpGet":{"path":"/ready","port":8181,"scheme":"HTTP"},"timeoutSeconds":1,"periodSeconds":10,"successThreshold":1,"failureThreshold":3},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"add":["NET_BIND_SERVICE"],"drop":["all"]},"readOnlyRootFilesystem":true,"allowPrivilegeEscalation":false}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"Default","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"coredns","serviceAccount":"coredns","nodeName":"minikube","securityContext":{},"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":100,"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"k8s-app","operator":"In","values":["kube-dns"]}]},"topologyKey":"kubernetes.io/hostname"}}]}},"schedulerName":"default-scheduler","tolerations":[{"key":"CriticalAddonsOnly","operator":"Exists"},{"key":"node-role.kubernetes.io/control-plane","effect":"NoSchedule"},{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priorityClassName":"system-cluster-critical","priority":2000000000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:24Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:24Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.2","podIPs":[{"ip":"10.244.0.2"}],"startTime":"2023-04-25T16:50:23Z","containerStatuses":[{"name":"coredns","state":{"running":{"startedAt":"2023-04-25T16:50:24Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/coredns/coredns:v1.9.3","imageID":"docker-pullable://registry.k8s.io/coredns/coredns@sha256:8e352a029d304ca7431c6507b56800636c321cb52289686a581ab70aaa8a2e2a","containerID":"docker://f0fd7b01742a3f4f65beb797a8b32ad3f586a829780380cdc0d04df9834a955b","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"etcd-minikube","namespace":"kube-system","uid":"c6fc83df-2740-4615-a154-de263f5249b8","resourceVersion":"293","creationTimestamp":"2023-04-25T16:50:10Z","labels":{"component":"etcd","tier":"control-plane"},"annotations":{"kubeadm.kubernetes.io/etcd.advertise-client-urls":"https://192.168.49.2:2379","kubernetes.io/config.hash":"a121e106627e5c6efa9ba48006cc43bf","kubernetes.io/config.mirror":"a121e106627e5c6efa9ba48006cc43bf","kubernetes.io/config.seen":"2023-04-25T16:50:10.549410231Z","kubernetes.io/config.source":"file"},"ownerReferences":[{"apiVersion":"v1","kind":"Node","name":"minikube","uid":"d739d6f0-db76-4187-bb71-ba126b44b4d7","controller":true}],"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeadm.kubernetes.io/etcd.advertise-client-urls":{},"f:kubernetes.io/config.hash":{},"f:kubernetes.io/config.mirror":{},"f:kubernetes.io/config.seen":{},"f:kubernetes.io/config.source":{}},"f:labels":{".":{},"f:component":{},"f:tier":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d739d6f0-db76-4187-bb71-ba126b44b4d7\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"etcd\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:livenessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:startupProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/var/lib/minikube/certs/etcd\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/minikube/etcd\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeName":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{".":{},"f:seccompProfile":{".":{},"f:type":{}}},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"etcd-certs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"etcd-data\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:19Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{".":{},"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"PodScheduled\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"etcd-certs","hostPath":{"path":"/var/lib/minikube/certs/etcd","type":"DirectoryOrCreate"}},{"name":"etcd-data","hostPath":{"path":"/var/lib/minikube/etcd","type":"DirectoryOrCreate"}}],"containers":[{"name":"etcd","image":"registry.k8s.io/etcd:3.5.6-0","command":["etcd","--advertise-client-urls=https://192.168.49.2:2379","--cert-file=/var/lib/minikube/certs/etcd/server.crt","--client-cert-auth=true","--data-dir=/var/lib/minikube/etcd","--experimental-initial-corrupt-check=true","--experimental-watch-progress-notify-interval=5s","--initial-advertise-peer-urls=https://192.168.49.2:2380","--initial-cluster=minikube=https://192.168.49.2:2380","--key-file=/var/lib/minikube/certs/etcd/server.key","--listen-client-urls=https://127.0.0.1:2379,https://192.168.49.2:2379","--listen-metrics-urls=http://127.0.0.1:2381","--listen-peer-urls=https://192.168.49.2:2380","--name=minikube","--peer-cert-file=/var/lib/minikube/certs/etcd/peer.crt","--peer-client-cert-auth=true","--peer-key-file=/var/lib/minikube/certs/etcd/peer.key","--peer-trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt","--proxy-refresh-interval=70000","--snapshot-count=10000","--trusted-ca-file=/var/lib/minikube/certs/etcd/ca.crt"],"resources":{"requests":{"cpu":"100m","memory":"100Mi"}},"volumeMounts":[{"name":"etcd-data","mountPath":"/var/lib/minikube/etcd"},{"name":"etcd-certs","mountPath":"/var/lib/minikube/certs/etcd"}],"livenessProbe":{"httpGet":{"path":"/health?exclude=NOSPACE\u0026serializable=true","port":2381,"host":"127.0.0.1","scheme":"HTTP"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":8},"startupProbe":{"httpGet":{"path":"/health?serializable=false","port":2381,"host":"127.0.0.1","scheme":"HTTP"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":24},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeName":"minikube","hostNetwork":true,"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists","effect":"NoExecute"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:19Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:19Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:11Z","containerStatuses":[{"name":"etcd","state":{"running":{"startedAt":"2023-04-25T16:50:05Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/etcd:3.5.6-0","imageID":"docker-pullable://registry.k8s.io/etcd@sha256:dd75ec974b0a2a6f6bb47001ba09207976e625db898d1b16735528c009cb171c","containerID":"docker://01fc557a450025fbafbdbe26227190c0efe8d133f417566408f05b6917965e8d","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"kube-apiserver-minikube","namespace":"kube-system","uid":"f6bb7440-5ac5-419d-b332-84d184bf7dd3","resourceVersion":"288","creationTimestamp":"2023-04-25T16:50:10Z","labels":{"component":"kube-apiserver","tier":"control-plane"},"annotations":{"kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint":"192.168.49.2:8443","kubernetes.io/config.hash":"5239bb256c1be9f71fd10c884d9299b1","kubernetes.io/config.mirror":"5239bb256c1be9f71fd10c884d9299b1","kubernetes.io/config.seen":"2023-04-25T16:50:10.549411451Z","kubernetes.io/config.source":"file"},"ownerReferences":[{"apiVersion":"v1","kind":"Node","name":"minikube","uid":"d739d6f0-db76-4187-bb71-ba126b44b4d7","controller":true}],"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint":{},"f:kubernetes.io/config.hash":{},"f:kubernetes.io/config.mirror":{},"f:kubernetes.io/config.seen":{},"f:kubernetes.io/config.source":{}},"f:labels":{".":{},"f:component":{},"f:tier":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d739d6f0-db76-4187-bb71-ba126b44b4d7\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"kube-apiserver\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:livenessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:name":{},"f:readinessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{}}},"f:startupProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/etc/ssl/certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/usr/local/share/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/usr/share/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/minikube/certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeName":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{".":{},"f:seccompProfile":{".":{},"f:type":{}}},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"ca-certs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"etc-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"k8s-certs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"usr-local-share-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"usr-share-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{".":{},"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"PodScheduled\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"ca-certs","hostPath":{"path":"/etc/ssl/certs","type":"DirectoryOrCreate"}},{"name":"etc-ca-certificates","hostPath":{"path":"/etc/ca-certificates","type":"DirectoryOrCreate"}},{"name":"k8s-certs","hostPath":{"path":"/var/lib/minikube/certs","type":"DirectoryOrCreate"}},{"name":"usr-local-share-ca-certificates","hostPath":{"path":"/usr/local/share/ca-certificates","type":"DirectoryOrCreate"}},{"name":"usr-share-ca-certificates","hostPath":{"path":"/usr/share/ca-certificates","type":"DirectoryOrCreate"}}],"containers":[{"name":"kube-apiserver","image":"registry.k8s.io/kube-apiserver:v1.26.1","command":["kube-apiserver","--advertise-address=192.168.49.2","--allow-privileged=true","--authorization-mode=Node,RBAC","--client-ca-file=/var/lib/minikube/certs/ca.crt","--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota","--enable-bootstrap-token-auth=true","--etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt","--etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt","--etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key","--etcd-servers=https://127.0.0.1:2379","--kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt","--kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key","--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname","--proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt","--proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key","--requestheader-allowed-names=front-proxy-client","--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt","--requestheader-extra-headers-prefix=X-Remote-Extra-","--requestheader-group-headers=X-Remote-Group","--requestheader-username-headers=X-Remote-User","--secure-port=8443","--service-account-issuer=https://kubernetes.default.svc.cluster.local","--service-account-key-file=/var/lib/minikube/certs/sa.pub","--service-account-signing-key-file=/var/lib/minikube/certs/sa.key","--service-cluster-ip-range=10.96.0.0/12","--tls-cert-file=/var/lib/minikube/certs/apiserver.crt","--tls-private-key-file=/var/lib/minikube/certs/apiserver.key"],"resources":{"requests":{"cpu":"250m"}},"volumeMounts":[{"name":"ca-certs","readOnly":true,"mountPath":"/etc/ssl/certs"},{"name":"etc-ca-certificates","readOnly":true,"mountPath":"/etc/ca-certificates"},{"name":"k8s-certs","readOnly":true,"mountPath":"/var/lib/minikube/certs"},{"name":"usr-local-share-ca-certificates","readOnly":true,"mountPath":"/usr/local/share/ca-certificates"},{"name":"usr-share-ca-certificates","readOnly":true,"mountPath":"/usr/share/ca-certificates"}],"livenessProbe":{"httpGet":{"path":"/livez","port":8443,"host":"192.168.49.2","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":8},"readinessProbe":{"httpGet":{"path":"/readyz","port":8443,"host":"192.168.49.2","scheme":"HTTPS"},"timeoutSeconds":15,"periodSeconds":1,"successThreshold":1,"failureThreshold":3},"startupProbe":{"httpGet":{"path":"/livez","port":8443,"host":"192.168.49.2","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":24},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeName":"minikube","hostNetwork":true,"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists","effect":"NoExecute"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:17Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:17Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:11Z","containerStatuses":[{"name":"kube-apiserver","state":{"running":{"startedAt":"2023-04-25T16:50:05Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/kube-apiserver:v1.26.1","imageID":"docker-pullable://registry.k8s.io/kube-apiserver@sha256:99e1ed9fbc8a8d36a70f148f25130c02e0e366875249906be0bcb2c2d9df0c26","containerID":"docker://6480be90562935e1c64ce2616eb49fad4bf62c96f6f143b3037b280c48bc5755","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"kube-controller-manager-minikube","namespace":"kube-system","uid":"dbc14338-808c-488f-ad28-d76644ea507f","resourceVersion":"289","creationTimestamp":"2023-04-25T16:50:10Z","labels":{"component":"kube-controller-manager","tier":"control-plane"},"annotations":{"kubernetes.io/config.hash":"5175bba984ed52052d891b5a45b584b6","kubernetes.io/config.mirror":"5175bba984ed52052d891b5a45b584b6","kubernetes.io/config.seen":"2023-04-25T16:50:01.128338670Z","kubernetes.io/config.source":"file"},"ownerReferences":[{"apiVersion":"v1","kind":"Node","name":"minikube","uid":"d739d6f0-db76-4187-bb71-ba126b44b4d7","controller":true}],"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:10Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubernetes.io/config.hash":{},"f:kubernetes.io/config.mirror":{},"f:kubernetes.io/config.seen":{},"f:kubernetes.io/config.source":{}},"f:labels":{".":{},"f:component":{},"f:tier":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d739d6f0-db76-4187-bb71-ba126b44b4d7\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"kube-controller-manager\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:livenessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{}}},"f:startupProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/etc/kubernetes/controller-manager.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/etc/ssl/certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/usr/libexec/kubernetes/kubelet-plugins/volume/exec\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/usr/local/share/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/usr/share/ca-certificates\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/var/lib/minikube/certs\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeName":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{".":{},"f:seccompProfile":{".":{},"f:type":{}}},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"ca-certs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"etc-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"flexvolume-dir\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"k8s-certs\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"kubeconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"usr-local-share-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"usr-share-ca-certificates\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{".":{},"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"PodScheduled\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"ca-certs","hostPath":{"path":"/etc/ssl/certs","type":"DirectoryOrCreate"}},{"name":"etc-ca-certificates","hostPath":{"path":"/etc/ca-certificates","type":"DirectoryOrCreate"}},{"name":"flexvolume-dir","hostPath":{"path":"/usr/libexec/kubernetes/kubelet-plugins/volume/exec","type":"DirectoryOrCreate"}},{"name":"k8s-certs","hostPath":{"path":"/var/lib/minikube/certs","type":"DirectoryOrCreate"}},{"name":"kubeconfig","hostPath":{"path":"/etc/kubernetes/controller-manager.conf","type":"FileOrCreate"}},{"name":"usr-local-share-ca-certificates","hostPath":{"path":"/usr/local/share/ca-certificates","type":"DirectoryOrCreate"}},{"name":"usr-share-ca-certificates","hostPath":{"path":"/usr/share/ca-certificates","type":"DirectoryOrCreate"}}],"containers":[{"name":"kube-controller-manager","image":"registry.k8s.io/kube-controller-manager:v1.26.1","command":["kube-controller-manager","--allocate-node-cidrs=true","--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf","--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf","--bind-address=127.0.0.1","--client-ca-file=/var/lib/minikube/certs/ca.crt","--cluster-cidr=10.244.0.0/16","--cluster-name=mk","--cluster-signing-cert-file=/var/lib/minikube/certs/ca.crt","--cluster-signing-key-file=/var/lib/minikube/certs/ca.key","--controllers=*,bootstrapsigner,tokencleaner","--kubeconfig=/etc/kubernetes/controller-manager.conf","--leader-elect=false","--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt","--root-ca-file=/var/lib/minikube/certs/ca.crt","--service-account-private-key-file=/var/lib/minikube/certs/sa.key","--service-cluster-ip-range=10.96.0.0/12","--use-service-account-credentials=true"],"resources":{"requests":{"cpu":"200m"}},"volumeMounts":[{"name":"ca-certs","readOnly":true,"mountPath":"/etc/ssl/certs"},{"name":"etc-ca-certificates","readOnly":true,"mountPath":"/etc/ca-certificates"},{"name":"flexvolume-dir","mountPath":"/usr/libexec/kubernetes/kubelet-plugins/volume/exec"},{"name":"k8s-certs","readOnly":true,"mountPath":"/var/lib/minikube/certs"},{"name":"kubeconfig","readOnly":true,"mountPath":"/etc/kubernetes/controller-manager.conf"},{"name":"usr-local-share-ca-certificates","readOnly":true,"mountPath":"/usr/local/share/ca-certificates"},{"name":"usr-share-ca-certificates","readOnly":true,"mountPath":"/usr/share/ca-certificates"}],"livenessProbe":{"httpGet":{"path":"/healthz","port":10257,"host":"127.0.0.1","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":8},"startupProbe":{"httpGet":{"path":"/healthz","port":10257,"host":"127.0.0.1","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":24},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeName":"minikube","hostNetwork":true,"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists","effect":"NoExecute"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:17Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:17Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:11Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:11Z","containerStatuses":[{"name":"kube-controller-manager","state":{"running":{"startedAt":"2023-04-25T16:50:05Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/kube-controller-manager:v1.26.1","imageID":"docker-pullable://registry.k8s.io/kube-controller-manager@sha256:40adecbe3a40aa147c7d6e9a1f5fbd99b3f6d42d5222483ed3a47337d4f9a10b","containerID":"docker://9d9fb12c1683ebcc46c702c5ad3d163ef1a1c9a4f3d216a203c8e0f409138782","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"kube-proxy-k5rcc","generateName":"kube-proxy-","namespace":"kube-system","uid":"abba49e8-be52-4187-8073-c48bb8c6d206","resourceVersion":"359","creationTimestamp":"2023-04-25T16:50:23Z","labels":{"controller-revision-hash":"6bc4695d8c","k8s-app":"kube-proxy","pod-template-generation":"1"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"kube-proxy","uid":"a6459052-3c93-4bdf-b525-effd315d9229","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:23Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:k8s-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"a6459052-3c93-4bdf-b525-effd315d9229\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"kube-proxy\"}":{".":{},"f:command":{},"f:env":{".":{},"k:{\"name\":\"NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/lib/modules\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/run/xtables.lock\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/var/lib/kube-proxy\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeSelector":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"kube-proxy\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"lib-modules\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"xtables-lock\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-proxy","configMap":{"name":"kube-proxy","defaultMode":420}},{"name":"xtables-lock","hostPath":{"path":"/run/xtables.lock","type":"FileOrCreate"}},{"name":"lib-modules","hostPath":{"path":"/lib/modules","type":""}},{"name":"kube-api-access-5qb9j","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"kube-proxy","image":"registry.k8s.io/kube-proxy:v1.26.1","command":["/usr/local/bin/kube-proxy","--config=/var/lib/kube-proxy/config.conf","--hostname-override=$(NODE_NAME)"],"env":[{"name":"NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{},"volumeMounts":[{"name":"kube-proxy","mountPath":"/var/lib/kube-proxy"},{"name":"xtables-lock","mountPath":"/run/xtables.lock"},{"name":"lib-modules","readOnly":true,"mountPath":"/lib/modules"},{"name":"kube-api-access-5qb9j","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"privileged":true}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"kubernetes.io/os":"linux"},"serviceAccountName":"kube-proxy","serviceAccount":"kube-proxy","nodeName":"minikube","hostNetwork":true,"securityContext":{},"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["minikube"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists"},{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/disk-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/memory-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/pid-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/unschedulable","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/network-unavailable","operator":"Exists","effect":"NoSchedule"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:24Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:24Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:23Z","containerStatuses":[{"name":"kube-proxy","state":{"running":{"startedAt":"2023-04-25T16:50:23Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/kube-proxy:v1.26.1","imageID":"docker-pullable://registry.k8s.io/kube-proxy@sha256:85f705e7d98158a67432c53885b0d470c673b0fad3693440b45d07efebcda1c3","containerID":"docker://d3408759c0b33d75a9a31c6730610bd62ac134261293483a9ff729c4bb54f183","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"kube-scheduler-minikube","namespace":"kube-system","uid":"a41a19fe-8aa9-4f2d-b08c-be5d25ce7ffc","resourceVersion":"358","creationTimestamp":"2023-04-25T16:50:11Z","labels":{"component":"kube-scheduler","tier":"control-plane"},"annotations":{"kubernetes.io/config.hash":"197cd0de602d7cb722d0bd2daf878121","kubernetes.io/config.mirror":"197cd0de602d7cb722d0bd2daf878121","kubernetes.io/config.seen":"2023-04-25T16:50:10.549390531Z","kubernetes.io/config.source":"file"},"ownerReferences":[{"apiVersion":"v1","kind":"Node","name":"minikube","uid":"d739d6f0-db76-4187-bb71-ba126b44b4d7","controller":true}],"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:11Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubernetes.io/config.hash":{},"f:kubernetes.io/config.mirror":{},"f:kubernetes.io/config.seen":{},"f:kubernetes.io/config.source":{}},"f:labels":{".":{},"f:component":{},"f:tier":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"d739d6f0-db76-4187-bb71-ba126b44b4d7\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"kube-scheduler\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:livenessProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:name":{},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{}}},"f:startupProbe":{".":{},"f:failureThreshold":{},"f:httpGet":{".":{},"f:host":{},"f:path":{},"f:port":{},"f:scheme":{}},"f:initialDelaySeconds":{},"f:periodSeconds":{},"f:successThreshold":{},"f:timeoutSeconds":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/kubernetes/scheduler.conf\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:nodeName":{},"f:priorityClassName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{".":{},"f:seccompProfile":{".":{},"f:type":{}}},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"kubeconfig\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:24Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{".":{},"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"PodScheduled\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kubeconfig","hostPath":{"path":"/etc/kubernetes/scheduler.conf","type":"FileOrCreate"}}],"containers":[{"name":"kube-scheduler","image":"registry.k8s.io/kube-scheduler:v1.26.1","command":["kube-scheduler","--authentication-kubeconfig=/etc/kubernetes/scheduler.conf","--authorization-kubeconfig=/etc/kubernetes/scheduler.conf","--bind-address=127.0.0.1","--kubeconfig=/etc/kubernetes/scheduler.conf","--leader-elect=false"],"resources":{"requests":{"cpu":"100m"}},"volumeMounts":[{"name":"kubeconfig","readOnly":true,"mountPath":"/etc/kubernetes/scheduler.conf"}],"livenessProbe":{"httpGet":{"path":"/healthz","port":10259,"host":"127.0.0.1","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":8},"startupProbe":{"httpGet":{"path":"/healthz","port":10259,"host":"127.0.0.1","scheme":"HTTPS"},"initialDelaySeconds":10,"timeoutSeconds":15,"periodSeconds":10,"successThreshold":1,"failureThreshold":24},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeName":"minikube","hostNetwork":true,"securityContext":{"seccompProfile":{"type":"RuntimeDefault"}},"schedulerName":"default-scheduler","tolerations":[{"operator":"Exists","effect":"NoExecute"}],"priorityClassName":"system-node-critical","priority":2000001000,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:10Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:10Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:10Z","containerStatuses":[{"name":"kube-scheduler","state":{"running":{"startedAt":"2023-04-25T16:50:05Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"registry.k8s.io/kube-scheduler:v1.26.1","imageID":"docker-pullable://registry.k8s.io/kube-scheduler@sha256:af0292c2c4fa6d09ee8544445eef373c1c280113cb6c968398a37da3744c41e4","containerID":"docker://5e7269848a77d76ab70b43c3f0a1327ef5bd0c704ad9228882d21dafe874034f","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"metricbeat-g2snm","generateName":"metricbeat-","namespace":"kube-system","uid":"827533de-e14e-48e8-b967-34fe9ddfd488","resourceVersion":"22778","creationTimestamp":"2023-04-25T20:30:36Z","labels":{"controller-revision-hash":"6c44456c5b","k8s-app":"metricbeat","pod-template-generation":"1"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"DaemonSet","name":"metricbeat","uid":"c6a5db20-c9ba-4181-afab-35036bd4354b","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:36Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:controller-revision-hash":{},"f:k8s-app":{},"f:pod-template-generation":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"c6a5db20-c9ba-4181-afab-35036bd4354b\"}":{}}},"f:spec":{"f:affinity":{".":{},"f:nodeAffinity":{".":{},"f:requiredDuringSchedulingIgnoredDuringExecution":{}}},"f:containers":{"k:{\"name\":\"metricbeat\"}":{".":{},"f:args":{},"f:env":{".":{},"k:{\"name\":\"ELASTICSEARCH_HOST\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"ELASTICSEARCH_PASSWORD\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"ELASTICSEARCH_PORT\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"ELASTICSEARCH_USERNAME\"}":{".":{},"f:name":{},"f:value":{}},"k:{\"name\":\"ELASTIC_CLOUD_AUTH\"}":{".":{},"f:name":{}},"k:{\"name\":\"ELASTIC_CLOUD_ID\"}":{".":{},"f:name":{}},"k:{\"name\":\"NODE_NAME\"}":{".":{},"f:name":{},"f:valueFrom":{".":{},"f:fieldRef":{}}}},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{".":{},"f:limits":{".":{},"f:memory":{}},"f:requests":{".":{},"f:cpu":{},"f:memory":{}}},"f:securityContext":{".":{},"f:runAsUser":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/etc/metricbeat.yml\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{},"f:subPath":{}},"k:{\"mountPath\":\"/hostfs/proc\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/hostfs/sys/fs/cgroup\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}},"k:{\"mountPath\":\"/usr/share/metricbeat/data\"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{\"mountPath\":\"/usr/share/metricbeat/modules.d\"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:tolerations":{},"f:volumes":{".":{},"k:{\"name\":\"cgroup\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"config\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"data\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{\"name\":\"modules\"}":{".":{},"f:configMap":{".":{},"f:defaultMode":{},"f:name":{}},"f:name":{}},"k:{\"name\":\"proc\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:38Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"proc","hostPath":{"path":"/proc","type":""}},{"name":"cgroup","hostPath":{"path":"/sys/fs/cgroup","type":""}},{"name":"config","configMap":{"name":"metricbeat-daemonset-config","defaultMode":416}},{"name":"modules","configMap":{"name":"metricbeat-daemonset-modules","defaultMode":416}},{"name":"data","hostPath":{"path":"/var/lib/metricbeat-data","type":"DirectoryOrCreate"}},{"name":"kube-api-access-5dph6","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"metricbeat","image":"docker.elastic.co/beats/metricbeat:7.17.8","args":["-c","/etc/metricbeat.yml","-e","-system.hostfs=/hostfs"],"env":[{"name":"ELASTICSEARCH_HOST","value":"elasticsearch"},{"name":"ELASTICSEARCH_PORT","value":"9200"},{"name":"ELASTICSEARCH_USERNAME","value":"elastic"},{"name":"ELASTICSEARCH_PASSWORD","value":"changeme"},{"name":"ELASTIC_CLOUD_ID"},{"name":"ELASTIC_CLOUD_AUTH"},{"name":"NODE_NAME","valueFrom":{"fieldRef":{"apiVersion":"v1","fieldPath":"spec.nodeName"}}}],"resources":{"limits":{"memory":"200Mi"},"requests":{"cpu":"100m","memory":"100Mi"}},"volumeMounts":[{"name":"config","readOnly":true,"mountPath":"/etc/metricbeat.yml","subPath":"metricbeat.yml"},{"name":"data","mountPath":"/usr/share/metricbeat/data"},{"name":"modules","readOnly":true,"mountPath":"/usr/share/metricbeat/modules.d"},{"name":"proc","readOnly":true,"mountPath":"/hostfs/proc"},{"name":"cgroup","readOnly":true,"mountPath":"/hostfs/sys/fs/cgroup"},{"name":"kube-api-access-5dph6","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"runAsUser":0}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirstWithHostNet","serviceAccountName":"metricbeat","serviceAccount":"metricbeat","nodeName":"minikube","hostNetwork":true,"securityContext":{},"affinity":{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchFields":[{"key":"metadata.name","operator":"In","values":["minikube"]}]}]}}},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute"},{"key":"node.kubernetes.io/disk-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/memory-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/pid-pressure","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/unschedulable","operator":"Exists","effect":"NoSchedule"},{"key":"node.kubernetes.io/network-unavailable","operator":"Exists","effect":"NoSchedule"}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:36Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:38Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:38Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:36Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T20:30:36Z","containerStatuses":[{"name":"metricbeat","state":{"running":{"startedAt":"2023-04-25T20:30:37Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"docker.elastic.co/beats/metricbeat:7.17.8","imageID":"docker-pullable://docker.elastic.co/beats/metricbeat@sha256:1751fa18dcf5bae3a8fb8ee196e8e86f2d64abd2dd80d1c231a15b1b4cb3b790","containerID":"docker://9ff09d1f38518f54c23df22ae01cb90ed8127e4cd9f2eb926cf4c390e4eff315","started":true}],"qosClass":"Burstable"}},{"metadata":{"name":"storage-provisioner","namespace":"kube-system","uid":"753018b5-18f8-4c1f-b8d1-d778b9034ea0","resourceVersion":"386","creationTimestamp":"2023-04-25T16:50:11Z","labels":{"addonmanager.kubernetes.io/mode":"Reconcile","integration-test":"storage-provisioner"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"annotations\":{},\"labels\":{\"addonmanager.kubernetes.io/mode\":\"Reconcile\",\"integration-test\":\"storage-provisioner\"},\"name\":\"storage-provisioner\",\"namespace\":\"kube-system\"},\"spec\":{\"containers\":[{\"command\":[\"/storage-provisioner\"],\"image\":\"gcr.io/k8s-minikube/storage-provisioner:v5\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"storage-provisioner\",\"volumeMounts\":[{\"mountPath\":\"/tmp\",\"name\":\"tmp\"}]}],\"hostNetwork\":true,\"serviceAccountName\":\"storage-provisioner\",\"volumes\":[{\"hostPath\":{\"path\":\"/tmp\",\"type\":\"Directory\"},\"name\":\"tmp\"}]}}\n"},"managedFields":[{"manager":"kube-scheduler","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:11Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{".":{},"k:{\"type\":\"PodScheduled\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:11Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:addonmanager.kubernetes.io/mode":{},"f:integration-test":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"storage-provisioner\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{\"mountPath\":\"/tmp\"}":{".":{},"f:mountPath":{},"f:name":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostNetwork":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:serviceAccount":{},"f:serviceAccountName":{},"f:terminationGracePeriodSeconds":{},"f:volumes":{".":{},"k:{\"name\":\"tmp\"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T16:50:55Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"192.168.49.2\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"tmp","hostPath":{"path":"/tmp","type":"Directory"}},{"name":"kube-api-access-fc6jq","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"storage-provisioner","image":"gcr.io/k8s-minikube/storage-provisioner:v5","command":["/storage-provisioner"],"resources":{},"volumeMounts":[{"name":"tmp","mountPath":"/tmp"},{"name":"kube-api-access-fc6jq","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"storage-provisioner","serviceAccount":"storage-provisioner","nodeName":"minikube","hostNetwork":true,"securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:55Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:55Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T16:50:23Z"}],"hostIP":"192.168.49.2","podIP":"192.168.49.2","podIPs":[{"ip":"192.168.49.2"}],"startTime":"2023-04-25T16:50:23Z","containerStatuses":[{"name":"storage-provisioner","state":{"running":{"startedAt":"2023-04-25T16:50:55Z"}},"lastState":{"terminated":{"exitCode":1,"reason":"Error","startedAt":"2023-04-25T16:50:23Z","finishedAt":"2023-04-25T16:50:53Z","containerID":"docker://f4c996de141e6d8c5ab937b5a77bbc484a199990ae43a5d6c67e9e652814214f"}},"ready":true,"restartCount":1,"image":"gcr.io/k8s-minikube/storage-provisioner:v5","imageID":"docker-pullable://gcr.io/k8s-minikube/storage-provisioner@sha256:18eb69d1418e854ad5a19e399310e52808a8321e4c441c1dddad8977a0d7a944","containerID":"docker://c9c9ef49120bc9627b07af08faa80ad64671ed5b31424d6af625b2f3c45b01db","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"challenge","namespace":"test","uid":"ccaf0aef-f36e-4187-b01d-ff9f0ebb9df4","resourceVersion":"22777","creationTimestamp":"2023-04-25T20:30:36Z","labels":{"watcher":"enabled"},"annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"annotations\":{},\"labels\":{\"watcher\":\"enabled\"},\"name\":\"challenge\",\"namespace\":\"test\",\"ownerReferences\":[{\"apiVersion\":\"apps/v1\",\"kind\":\"Challenge\",\"name\":\"challenge\",\"uid\":\"5e014df1-0c28-451e-97f5-0edde20bab45\"}]},\"spec\":{\"containers\":[{\"image\":\"nginx\",\"name\":\"nginx\"}]}}\n"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"Challenge","name":"challenge","uid":"5e014df1-0c28-451e-97f5-0edde20bab45"}],"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:36Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:watcher":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"5e014df1-0c28-451e-97f5-0edde20bab45\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:38Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.219\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-qhd8s","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"nginx","image":"nginx","resources":{},"volumeMounts":[{"name":"kube-api-access-qhd8s","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:36Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:38Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:38Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:36Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.219","podIPs":[{"ip":"10.244.0.219"}],"startTime":"2023-04-25T20:30:36Z","containerStatuses":[{"name":"nginx","state":{"running":{"startedAt":"2023-04-25T20:30:37Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"nginx:latest","imageID":"docker-pullable://nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94","containerID":"docker://247471cba4eae25dd428d02611c48328eabed7a89fadaab24cbc2f2e4dd390df","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"hello-28040911-7x9gp","generateName":"hello-28040911-","namespace":"test","uid":"7cd97c02-f9ad-4fad-88bf-637470452b31","resourceVersion":"22833","creationTimestamp":"2023-04-25T20:31:00Z","labels":{"controller-uid":"5464b3ec-bd39-480c-bcd7-d58e5589508d","job-name":"hello-28040911"},"ownerReferences":[{"apiVersion":"batch/v1","kind":"Job","name":"hello-28040911","uid":"5464b3ec-bd39-480c-bcd7-d58e5589508d","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:31:00Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:controller-uid":{},"f:job-name":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"5464b3ec-bd39-480c-bcd7-d58e5589508d\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"hello\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:31:04Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:reason":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.220\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-lnkzd","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"hello","image":"hello-world","resources":{},"volumeMounts":[{"name":"kube-api-access-lnkzd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"OnFailure","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Succeeded","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:31:00Z","reason":"PodCompleted"},{"type":"Ready","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:31:00Z","reason":"PodCompleted"},{"type":"ContainersReady","status":"False","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:31:00Z","reason":"PodCompleted"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:31:00Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.220","podIPs":[{"ip":"10.244.0.220"}],"startTime":"2023-04-25T20:31:00Z","containerStatuses":[{"name":"hello","state":{"terminated":{"exitCode":0,"reason":"Completed","startedAt":"2023-04-25T20:31:01Z","finishedAt":"2023-04-25T20:31:01Z","containerID":"docker://86937c13198cc7f928eb76f24a6054e34e13a6cf4a6f9d696e5096df2bf0d654"}},"lastState":{},"ready":false,"restartCount":0,"image":"hello-world:latest","imageID":"docker-pullable://hello-world@sha256:4e83453afed1b4fa1a3500525091dbfca6ce1e66903fd4c01ff015dbcb1ba33e","containerID":"docker://86937c13198cc7f928eb76f24a6054e34e13a6cf4a6f9d696e5096df2bf0d654","started":false}],"qosClass":"BestEffort"}},{"metadata":{"name":"nginx-746dc99457-n4tvk","generateName":"nginx-746dc99457-","namespace":"test","uid":"079a5cb8-6035-4bad-a5f3-0889239cbe0d","resourceVersion":"22701","creationTimestamp":"2023-04-25T20:30:03Z","labels":{"pod-template-hash":"746dc99457","run":"nginx"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"nginx-746dc99457","uid":"8bad806b-a3f2-4747-8acc-793332426466","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:pod-template-hash":{},"f:run":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"8bad806b-a3f2-4747-8acc-793332426466\"}":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:05Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.215\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-mqt7c","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"nginx","image":"nginx","resources":{},"volumeMounts":[{"name":"kube-api-access-mqt7c","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:05Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:05Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.215","podIPs":[{"ip":"10.244.0.215"}],"startTime":"2023-04-25T20:30:03Z","containerStatuses":[{"name":"nginx","state":{"running":{"startedAt":"2023-04-25T20:30:05Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"nginx:latest","imageID":"docker-pullable://nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94","containerID":"docker://0421378ff0eaa14d6e499e4285475ab37cd5020be85bab7b14ccc66daa7167cb","started":true}],"qosClass":"BestEffort"}},{"metadata":{"name":"nginx-pod","namespace":"test","uid":"0e531379-c1b0-4243-9836-2e4996c83272","resourceVersion":"22707","creationTimestamp":"2023-04-25T20:30:03Z","annotations":{"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"annotations\":{},\"name\":\"nginx-pod\",\"namespace\":\"test\"},\"spec\":{\"containers\":[{\"image\":\"nginx\",\"name\":\"nginx\"}]}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:03Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2023-04-25T20:30:06Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.217\"}":{".":{},"f:ip":{}}},"f:startTime":{}}},"subresource":"status"}]},"spec":{"volumes":[{"name":"kube-api-access-j97t8","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}}],"defaultMode":420}}],"containers":[{"name":"nginx","image":"nginx","resources":{},"volumeMounts":[{"name":"kube-api-access-j97t8","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"minikube","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:06Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:06Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2023-04-25T20:30:03Z"}],"hostIP":"192.168.49.2","podIP":"10.244.0.217","podIPs":[{"ip":"10.244.0.217"}],"startTime":"2023-04-25T20:30:03Z","containerStatuses":[{"name":"nginx","state":{"running":{"startedAt":"2023-04-25T20:30:06Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"nginx:latest","imageID":"docker-pullable://nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94","containerID":"docker://0821dce808f8944946b51d16663628e6e078e533df131a01187498c9b8105fb9","started":true}],"qosClass":"BestEffort"}}]} 16 | 17 | ' 18 | headers: 19 | Audit-Id: 20 | - c2b40394-7c08-4bae-872c-f4c935f92fda 21 | Cache-Control: 22 | - no-cache, private 23 | Content-Type: 24 | - application/json 25 | Date: 26 | - Tue, 25 Apr 2023 20:31:15 GMT 27 | Transfer-Encoding: 28 | - chunked 29 | X-Kubernetes-Pf-Flowschema-Uid: 30 | - b6229760-c42a-4dc5-a942-f7612b973d7c 31 | X-Kubernetes-Pf-Prioritylevel-Uid: 32 | - f1a99473-5335-402d-ada4-0498e1d87d9b 33 | status: 34 | code: 200 35 | message: OK 36 | version: 1 37 | --------------------------------------------------------------------------------