├── .dockerignore ├── .github └── workflows │ ├── docker-registry-push.yml │ └── helm-release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── charts └── mongodb-profiler-exporter │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── deployment.yaml │ ├── ingress.yaml │ ├── service.yaml │ ├── serviceaccount.yaml │ └── servicemonitor.yaml │ └── values.yaml ├── images └── image1.png ├── mongodb-profiler-exporter.py ├── requirements.txt └── tests └── docker-compose ├── .env ├── README.md ├── docker-compose.yml ├── grafana ├── dashboards │ ├── dashboards.yaml │ └── mongodb-profiler-exporter.json └── datasources │ └── prometheus-datasource.yaml ├── mongodb └── scripts │ ├── generate_random_data.js │ ├── prepare.sh │ └── query.sh └── prometheus └── prometheus.yml /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | LICENSE 3 | Makefile 4 | README.md 5 | images/ 6 | .git 7 | .github 8 | .gitignore 9 | .dockerignore 10 | .history 11 | -------------------------------------------------------------------------------- /.github/workflows/docker-registry-push.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | env: 8 | REPO: andriik/mongodb-profiler-exporter 9 | 10 | jobs: 11 | push_to_registry: 12 | name: Push Docker image to Docker Hub 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | 17 | - name: Checkout 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: Check Tag 23 | id: check-tag 24 | run: | 25 | if [[ "${{ github.event.release.tag_name }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 26 | echo "match=true" >> $GITHUB_ENV 27 | else 28 | echo "match=false" >> $GITHUB_ENV 29 | fi 30 | 31 | - name: Set up QEMU 32 | uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 33 | if: env.match == 'true' 34 | 35 | - name: Set up Docker Buildx 36 | uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 37 | if: env.match == 'true' 38 | 39 | 40 | - name: Log in to Docker Hub 41 | uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d 42 | if: env.match == 'true' 43 | with: 44 | username: ${{ secrets.DOCKER_USERNAME }} 45 | password: ${{ secrets.DOCKER_PASSWORD }} 46 | 47 | - name: Extract metadata (tags, labels) for Docker 48 | id: meta 49 | uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 50 | if: env.match == 'true' 51 | with: 52 | images: ${{ env.REPO }} 53 | 54 | - name: Build and push Docker image 55 | uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 56 | if: env.match == 'true' 57 | with: 58 | context: . 59 | file: ./Dockerfile 60 | push: true 61 | tags: | 62 | ${{ steps.meta.outputs.tags }} 63 | ${{ env.REPO }}:latest 64 | labels: ${{ steps.meta.outputs.labels }} 65 | platforms: linux/amd64,linux/arm64 66 | -------------------------------------------------------------------------------- /.github/workflows/helm-release.yml: -------------------------------------------------------------------------------- 1 | name: Release Charts 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | env: 8 | CHART_FILE: "charts/mongodb-profiler-exporter/Chart.yaml" 9 | RELEASE_TAG: "${{ github.event.release.tag_name }}" 10 | 11 | jobs: 12 | release: 13 | # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions 14 | # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token 15 | permissions: 16 | contents: write 17 | runs-on: ubuntu-latest 18 | steps: 19 | 20 | - name: Checkout 21 | uses: actions/checkout@v3 22 | 23 | - name: Check Tag 24 | id: check-tag 25 | run: | 26 | if [[ "${{ github.event.release.tag_name }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 27 | echo "match=true" >> $GITHUB_ENV 28 | else 29 | echo "match=false" >> $GITHUB_ENV 30 | fi 31 | 32 | - name: Update version in Chart.yaml 33 | if: env.match == 'true' 34 | run: | 35 | 36 | sed -i.bak "s/^version: .*/version: ${RELEASE_TAG}/" "$CHART_FILE" 37 | sed -i.bak "s/^appVersion: .*/appVersion: ${RELEASE_TAG}/" "$CHART_FILE" 38 | 39 | - name: Commit and push changes 40 | if: env.match == 'true' 41 | run: | 42 | git config user.name "$GITHUB_ACTOR" 43 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 44 | git add "${CHART_FILE}" 45 | git commit -m "Update version to ${RELEASE_TAG}" 46 | git push origin HEAD:main 47 | 48 | - name: Install Helm 49 | uses: azure/setup-helm@v4 50 | if: env.match == 'true' 51 | env: 52 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 53 | 54 | - name: Run chart-releaser 55 | uses: helm/chart-releaser-action@v1.6.0 56 | if: env.match == 'true' 57 | env: 58 | CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | .json 163 | .sql 164 | .history 165 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG TAG=3.12-alpine3.20 2 | FROM python:${TAG} 3 | 4 | ARG USER=app 5 | ARG UID=1000 6 | 7 | RUN adduser -D -s /bin/sh -u ${UID} ${USER} 8 | WORKDIR /app 9 | ADD requirements.txt . 10 | RUN pip install --no-cache-dir -r requirements.txt 11 | COPY . . 12 | RUN chown -R ${USER}:${USER} /app 13 | USER ${USER} 14 | EXPOSE 9179 15 | 16 | ENTRYPOINT ["python", "-u", "mongodb-profiler-exporter.py"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 andrii29 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build run 2 | 3 | IMAGE_NAME = mongodb-profiler-exporter 4 | IMAGE_VERSION = latest 5 | 6 | IMAGE_DOCKERHUB = andriik/$(IMAGE_NAME) 7 | 8 | # dockerhub 9 | docker-build-dockerhub: 10 | docker build -t $(IMAGE_DOCKERHUB):$(IMAGE_VERSION) . 11 | docker-run: 12 | docker run -p 9179:9179 -e MAX_STRING_SIZE=200 -e VERBOSE=true -e MONGODB_URI='mongodb://127.0.0.1:27017/' --network host --rm --name $(IMAGE_NAME) $(IMAGE_DOCKERHUB):$(IMAGE_VERSION) 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## MongoDB Profiler Exporter 2 | ![Grafana Dashboard 20383](images/image1.png) 3 | A Python script that exports MongoDB slow query metrics from system.profile collection for Prometheus monitoring. [Read more](https://medium.com/@andriikrymus/mongodb-profiler-exporter-3abb84b877f1) 4 | 5 | ### Installation 6 | ```bash 7 | pip install -r requirements.txt 8 | python mongodb-profiler-exporter.py 9 | ``` 10 | 11 | ### Docker 12 | ```js 13 | docker run -p 9179:9179 -it --rm --name mongodb-profiler-exporter andriik/mongodb-profiler-exporter 14 | docker run -it --rm --net host --name mongodb-profiler-exporter andriik/mongodb-profiler-exporter // host network 15 | ``` 16 | 17 | ### Helm 18 | ```js 19 | helm repo add mongodb-profiler-exporter https://andrii29.github.io/mongodb-profiler-exporter 20 | helm repo update 21 | helm search repo mongodb-profiler-exporter 22 | helm show values mongodb-profiler-exporter/mongodb-profiler-exporter 23 | helm install -n mongodb-profiler-exporter/mongodb-profiler-exporter 24 | ``` 25 | 26 | ### Usage 27 | ``` 28 | usage: mongodb-profiler-exporter.py [-h] [--mongodb-uri MONGODB_URI] [--wait-interval WAIT_INTERVAL] [--max-string-size MAX_STRING_SIZE] [--listen-ip LISTEN_IP] [--listen-port LISTEN_PORT] [--verbose] 29 | 30 | MongoDB Prometheus Exporter 31 | 32 | options: 33 | -h, --help show this help message and exit 34 | --mongodb-uri MONGODB_URI 35 | MongoDB URI (default: mongodb://127.0.0.1:27017/) 36 | --wait-interval WAIT_INTERVAL 37 | Wait interval between data parsing in seconds (default: 10) 38 | --max-string-size MAX_STRING_SIZE 39 | Maximum string size for Prometheus labels (default: 1000) 40 | --listen-ip LISTEN_IP 41 | IP address to listen on (default: 0.0.0.0) 42 | --listen-port LISTEN_PORT 43 | Port to listen (default: 9179) 44 | --verbose Enable Verbose Mode (default: False) 45 | 46 | ``` 47 | 48 | #### Environment Variables 49 | 50 | You can use environment variables to configure the exporter. If an environment variable is set, it takes precedence over the corresponding command-line argument. 51 | 52 | - `MONGODB_URI`: MongoDB URI (default: `mongodb://127.0.0.1:27017/`) 53 | - `WAIT_INTERVAL`: Wait interval between data parsing in seconds (default: `10`) 54 | - `MAX_STRING_SIZE`: Maximum string size for Prometheus labels (default: `1000`) 55 | - `LISTEN_IP`: IP address to listen on (default: `0.0.0.0`) 56 | - `LISTEN_PORT`: Port to listen (default: `9179`) 57 | 58 | ### Authentication 59 | To set up authentication, follow these steps: 60 | ``` 61 | mongosh 62 | 63 | use admin 64 | db.createUser({ 65 | user: "mongodb-profiler-exporter", 66 | pwd: passwordPrompt(), 67 | roles: [ { role: "clusterMonitor", db: "admin" } ] 68 | }) 69 | 70 | python mongodb-profiler-exporter.py --mongodb-uri "mongodb://mongodb-profiler-exporter:@127.0.0.1:27017/admin?authSource=admin&readPreference=primaryPreferred" 71 | ``` 72 | 73 | ### Enable MongoDB Profiler 74 | There are two ways to enable profiler in mongodb: 75 | #### Per Dababase 76 | ``` 77 | use db_name 78 | db.getProfilingStatus() 79 | db.setProfilingLevel(1, { slowms: 100 }) 80 | ``` 81 | 82 | #### Globally in mongod.conf 83 | ```yaml 84 | operationProfiling: 85 | mode: slowOp 86 | slowOpThresholdMs: 100 87 | ``` 88 | 89 | ### Increase system.profile size 90 | The default size of the system.profile collection is set to 1MB, which can be insufficient for certain scenarios. To address this limitation, you can adjust the size of the collection by recreating it. Note that this process should not be replicated to replicas. 91 | 92 | Below are example commands that can be used to increase the size of the system.profile collection to 50MB: 93 | ```js 94 | db.setProfilingLevel(0) // Disable profiling temporarily 95 | db.system.profile.drop() // Drop the existing system.profile collection 96 | db.createCollection( "system.profile", { capped: true, size: 1024 * 1024 * 50 } ) 97 | db.setProfilingLevel(1, { slowms: 100 }) // Enable profiling again 98 | ``` 99 | 100 | ### Supported MongoDB versions 101 | ``` 102 | 4.4, 5.0, 6.0, 7.0, 8.0 103 | ``` 104 | 105 | ### Grafana Dashboard 106 | You can find example dashboard at id [20387](https://grafana.com/grafana/dashboards/20387) 107 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: mongodb-profiler-exporter 3 | description: Mongodb Profiler Exporter 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 1.15.1 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: 1.15.1 25 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if .Values.ingress.enabled }} 3 | {{- range $host := .Values.ingress.hosts }} 4 | {{- range .paths }} 5 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} 6 | {{- end }} 7 | {{- end }} 8 | {{- else if contains "NodePort" .Values.service.type }} 9 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mongodb-profiler-exporter.fullname" . }}) 10 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 11 | echo http://$NODE_IP:$NODE_PORT 12 | {{- else if contains "LoadBalancer" .Values.service.type }} 13 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 14 | You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mongodb-profiler-exporter.fullname" . }}' 15 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mongodb-profiler-exporter.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") 16 | echo http://$SERVICE_IP:{{ .Values.service.port }} 17 | {{- else if contains "ClusterIP" .Values.service.type }} 18 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mongodb-profiler-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 19 | export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") 20 | echo "Visit http://127.0.0.1:8080 to use your application" 21 | kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT 22 | {{- end }} 23 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "mongodb-profiler-exporter.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "mongodb-profiler-exporter.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "mongodb-profiler-exporter.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "mongodb-profiler-exporter.labels" -}} 37 | helm.sh/chart: {{ include "mongodb-profiler-exporter.chart" . }} 38 | {{ include "mongodb-profiler-exporter.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "mongodb-profiler-exporter.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "mongodb-profiler-exporter.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "mongodb-profiler-exporter.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "mongodb-profiler-exporter.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "mongodb-profiler-exporter.fullname" . }} 5 | labels: 6 | {{- include "mongodb-profiler-exporter.labels" . | nindent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | {{- include "mongodb-profiler-exporter.selectorLabels" . | nindent 6 }} 12 | template: 13 | metadata: 14 | {{- with .Values.podAnnotations }} 15 | annotations: 16 | {{- toYaml . | nindent 8 }} 17 | {{- end }} 18 | labels: 19 | {{- include "mongodb-profiler-exporter.labels" . | nindent 8 }} 20 | {{- with .Values.podLabels }} 21 | {{- toYaml . | nindent 8 }} 22 | {{- end }} 23 | spec: 24 | {{- with .Values.imagePullSecrets }} 25 | imagePullSecrets: 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | serviceAccountName: {{ include "mongodb-profiler-exporter.serviceAccountName" . }} 29 | securityContext: 30 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 31 | containers: 32 | - name: {{ .Chart.Name }} 33 | securityContext: 34 | {{- toYaml .Values.securityContext | nindent 12 }} 35 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 36 | imagePullPolicy: {{ .Values.image.pullPolicy }} 37 | env: 38 | - name: LISTEN_PORT 39 | value: {{ .Values.service.containerPort | quote }} 40 | {{- with .Values.env }} 41 | {{- range $key, $value := . }} 42 | - name: {{ $key }} 43 | value: {{ $value | quote }} 44 | {{- end }} 45 | {{- end }} 46 | envFrom: 47 | {{- if .Values.configmapEnv }} 48 | - configMapRef: 49 | name: {{ .Values.configmapEnv }} 50 | {{- end }} 51 | {{- if .Values.secretEnv }} 52 | - secretRef: 53 | name: {{ .Values.secretEnv }} 54 | {{- end }} 55 | ports: 56 | - name: metrics 57 | containerPort: {{ .Values.service.containerPort }} 58 | protocol: TCP 59 | livenessProbe: 60 | httpGet: 61 | path: /metrics 62 | port: metrics 63 | readinessProbe: 64 | httpGet: 65 | path: /metrics 66 | port: metrics 67 | resources: 68 | {{- toYaml .Values.resources | nindent 12 }} 69 | {{- with .Values.volumeMounts }} 70 | volumeMounts: 71 | {{- toYaml . | nindent 12 }} 72 | {{- end }} 73 | {{- with .Values.volumes }} 74 | volumes: 75 | {{- toYaml . | nindent 8 }} 76 | {{- end }} 77 | {{- with .Values.nodeSelector }} 78 | nodeSelector: 79 | {{- toYaml . | nindent 8 }} 80 | {{- end }} 81 | {{- with .Values.affinity }} 82 | affinity: 83 | {{- toYaml . | nindent 8 }} 84 | {{- end }} 85 | {{- with .Values.tolerations }} 86 | tolerations: 87 | {{- toYaml . | nindent 8 }} 88 | {{- end }} 89 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "mongodb-profiler-exporter.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "mongodb-profiler-exporter.labels" . | nindent 4 }} 21 | {{- with .Values.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.ingress.tls }} 30 | tls: 31 | {{- range .Values.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $fullName }} 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }} 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "mongodb-profiler-exporter.fullname" . }} 5 | labels: 6 | {{- include "mongodb-profiler-exporter.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: metrics 12 | protocol: TCP 13 | name: metrics 14 | selector: 15 | {{- include "mongodb-profiler-exporter.selectorLabels" . | nindent 4 }} 16 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "mongodb-profiler-exporter.serviceAccountName" . }} 6 | labels: 7 | {{- include "mongodb-profiler-exporter.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | automountServiceAccountToken: {{ .Values.serviceAccount.automount }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceMonitor.enabled | default false }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "mongodb-profiler-exporter.fullname" . }} 6 | labels: 7 | {{- include "mongodb-profiler-exporter.labels" . | nindent 4 }} 8 | spec: 9 | selector: 10 | matchLabels: 11 | {{- include "mongodb-profiler-exporter.selectorLabels" . | nindent 6 }} 12 | endpoints: 13 | - port: metrics 14 | interval: {{ .Values.serviceMonitor.scrapeInterval | default "10s" }} 15 | path: {{ .Values.serviceMonitor.metricsPath | default "/metrics" }} 16 | scheme: {{ .Values.serviceMonitor.scheme | default "http" }} 17 | relabelings: 18 | {{- if .Values.serviceMonitor.relabelings }} 19 | {{ toYaml .Values.serviceMonitor.relabelings | nindent 6 }} 20 | {{- end }} 21 | namespaceSelector: 22 | matchNames: 23 | - {{ .Release.Namespace }} 24 | {{- end }} 25 | -------------------------------------------------------------------------------- /charts/mongodb-profiler-exporter/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for mongodb-profiler-exporter. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | repository: andriik/mongodb-profiler-exporter 9 | pullPolicy: IfNotPresent 10 | # Overrides the image tag whose default is the chart appVersion. 11 | tag: "" 12 | 13 | imagePullSecrets: [] 14 | nameOverride: "" 15 | fullnameOverride: "" 16 | 17 | serviceAccount: 18 | # Specifies whether a service account should be created 19 | create: false 20 | # Automatically mount a ServiceAccount's API credentials? 21 | automount: true 22 | # Annotations to add to the service account 23 | annotations: {} 24 | # The name of the service account to use. 25 | # If not set and create is true, a name is generated using the fullname template 26 | name: "" 27 | 28 | podAnnotations: {} 29 | podLabels: {} 30 | 31 | podSecurityContext: {} 32 | # fsGroup: 2000 33 | 34 | securityContext: {} 35 | # capabilities: 36 | # drop: 37 | # - ALL 38 | # readOnlyRootFilesystem: true 39 | # runAsNonRoot: true 40 | # runAsUser: 1000 41 | 42 | service: 43 | type: ClusterIP 44 | port: 80 45 | containerPort: 9179 46 | 47 | serviceMonitor: 48 | enabled: true 49 | # scrapeInterval: "10s" 50 | # metricsPath: "/metrics" 51 | # scheme: "http" 52 | # relabelings: [] 53 | 54 | ingress: 55 | enabled: false 56 | className: "" 57 | annotations: {} 58 | # kubernetes.io/ingress.class: nginx 59 | # kubernetes.io/tls-acme: "true" 60 | hosts: 61 | - host: chart-example.local 62 | paths: 63 | - path: / 64 | pathType: ImplementationSpecific 65 | tls: [] 66 | # - secretName: chart-example-tls 67 | # hosts: 68 | # - chart-example.local 69 | 70 | env: 71 | MONGODB_URI: "mongodb://127.0.0.1:27017/" # !CHANGEME 72 | # WAIT_INTERVAL: "10" 73 | # MAX_STRING_SIZE: "1000" 74 | # LISTEN_IP: "0.0.0.0" 75 | # VERBOSE: "true" 76 | 77 | # name of configmap with environment variables (optional) 78 | configmapEnv: "" 79 | # name of secret with environment variables (optional) 80 | secretEnv: "" 81 | 82 | resources: {} 83 | # We usually recommend not to specify default resources and to leave this as a conscious 84 | # choice for the user. This also increases chances charts run on environments with little 85 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 86 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 87 | # limits: 88 | # cpu: 100m 89 | # memory: 128Mi 90 | # requests: 91 | # cpu: 100m 92 | # memory: 128Mi 93 | 94 | # Additional volumes on the output Deployment definition. 95 | volumes: [] 96 | # - name: foo 97 | # secret: 98 | # secretName: mysecret 99 | # optional: false 100 | 101 | # Additional volumeMounts on the output Deployment definition. 102 | volumeMounts: [] 103 | # - name: foo 104 | # mountPath: "/etc/foo" 105 | # readOnly: true 106 | 107 | nodeSelector: {} 108 | 109 | tolerations: [] 110 | 111 | affinity: {} 112 | -------------------------------------------------------------------------------- /images/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrii29/mongodb-profiler-exporter/bac494ce0132cc7f7aa7ef470c723a35a093f7f9/images/image1.png -------------------------------------------------------------------------------- /mongodb-profiler-exporter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import logging 5 | from prometheus_client import start_http_server, Counter, Gauge 6 | import pymongo 7 | import time 8 | from datetime import datetime, timedelta 9 | from zoneinfo import ZoneInfo 10 | import argparse 11 | 12 | # Configure logging 13 | logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) 14 | 15 | default_labels = ["db", "ns", "query_hash"] 16 | # Prometheus metrics 17 | slow_queries_count_total = Counter('slow_queries_count_total', 'Total number of slow queries', default_labels) 18 | slow_queries_info = Gauge("slow_queries_info", "Information about slow query", 19 | default_labels + ["query_shape", "query_framework", "op", "plan_summary"]) 20 | slow_queries_duration_total = Counter('slow_queries_duration_total', 'Total execution time of slow queries in milliseconds', default_labels) 21 | slow_queries_keys_examined_total = Counter('slow_queries_keys_examined_total', 'Total number of examined keys', default_labels) 22 | slow_queries_docs_examined_total = Counter('slow_queries_docs_examined_total', 'Total number of examined documents', default_labels) 23 | slow_queries_nreturned_total = Counter('slow_queries_nreturned_total', 'Total number of returned documents', default_labels) 24 | 25 | fields_to_metrics_map = { 26 | "millis": slow_queries_duration_total, 27 | "keysExamined": slow_queries_keys_examined_total, 28 | "docsExamined": slow_queries_docs_examined_total, 29 | "nreturned": slow_queries_nreturned_total 30 | } 31 | 32 | def connect_to_mongo(uri): 33 | client = pymongo.MongoClient(uri) 34 | return client 35 | 36 | def get_query_hash_values(db, ns, start_time, end_time): 37 | profile_collection = db.system.profile 38 | # Find unique queryHash values 39 | unique_query_hashes = profile_collection.distinct("queryHash", {"ns": ns ,"ts": {"$gte": start_time, "$lt": end_time}}) 40 | return unique_query_hashes 41 | 42 | def get_ns_values(db, start_time, end_time): 43 | profile_collection = db.system.profile 44 | # Find unique ns values 45 | unique_ns_values = profile_collection.distinct("ns", {"ts": {"$gte": start_time, "$lt": end_time}}) 46 | return unique_ns_values 47 | 48 | def get_slow_queries_count(db, ns, query_hash, start_time, end_time): 49 | profile_collection = db.system.profile 50 | # Find values within the specified time window 51 | query = {"queryHash": query_hash, "ns": ns, "ts": {"$gte": start_time, "$lt": end_time}} 52 | count = profile_collection.count_documents(query) 53 | return count 54 | 55 | def get_slow_queries_value_sum(db, ns, query_hash, start_time, end_time, fields): 56 | profile_collection = db.system.profile 57 | # Find values within the specified time window 58 | match_stage = {"$match": {"queryHash": query_hash, "ns": ns, "ts": {"$gte": start_time, "$lt": end_time}}} 59 | fields_formatted = {f"{field}": {"$sum": f"${field}"} for field in fields} 60 | group_stage = {"$group": {"_id": None, **fields_formatted}} 61 | query = [match_stage, group_stage] 62 | result = list(profile_collection.aggregate(query))[0] 63 | return {field: result.get(field, 0) for field in fields} 64 | 65 | def remove_keys_and_replace(query, keys_to_remove, replace_value="?"): 66 | # Recursively remove keys and replace values in the query. 67 | if isinstance(query, dict): 68 | for key in keys_to_remove: 69 | query.pop(key, None) 70 | for key, value in query.items(): 71 | query[key] = remove_keys_and_replace(value, keys_to_remove, replace_value) 72 | elif isinstance(query, list): 73 | for i, item in enumerate(query): 74 | query[i] = remove_keys_and_replace(item, keys_to_remove, replace_value) 75 | else: 76 | return replace_value 77 | return query 78 | 79 | def get_query_info_values(db, ns, query_hash, start_time, end_time, keys_to_remove): 80 | # Get query information values for Prometheus metric. 81 | profile_collection = db.system.profile 82 | 83 | query = {"queryHash": query_hash,"ns": ns, "ts": {"$gte": start_time, "$lt": end_time}, "command.getMore": {"$exists": False}, "command.explain": {"$exists": False}} 84 | result = list(profile_collection.find(query).limit(1)) 85 | if result: 86 | query = result[0].get("command", "") 87 | query_framework = result[0].get("queryFramework", "") 88 | op = result[0].get("op", "") 89 | plan_summary = result[0].get("planSummary", "") 90 | query_shape = remove_keys_and_replace(query, keys_to_remove) 91 | else: 92 | query_shape, query_framework, op, plan_summary = '', '', '', '' 93 | return [query_shape, query_framework, op, plan_summary] 94 | 95 | def parse_args(): 96 | parser = argparse.ArgumentParser(description='MongoDB Prometheus Exporter', 97 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 98 | # Use environment variables or default values for command-line arguments 99 | parser.add_argument('--mongodb-uri', type=str, default=os.getenv('MONGODB_URI', 'mongodb://127.0.0.1:27017/'), 100 | help='MongoDB URI') 101 | parser.add_argument('--wait-interval', type=int, default=os.getenv('WAIT_INTERVAL', 10), 102 | help='Wait interval between data parsing in seconds') 103 | parser.add_argument('--max-string-size', type=int, default=os.getenv('MAX_STRING_SIZE', 1000), 104 | help='Maximum string size for Prometheus labels') 105 | parser.add_argument('--listen-ip', type=str, default=os.getenv('LISTEN_IP', '0.0.0.0'), 106 | help='IP address to listen on') 107 | parser.add_argument('--listen-port', type=int, default=os.getenv('LISTEN_PORT', 9179), 108 | help='Port to listen') 109 | parser.add_argument('--verbose', action='store_true', default=os.getenv('VERBOSE', False), 110 | help='Enable Verbose Mode') 111 | 112 | return parser.parse_args() 113 | 114 | def main(): 115 | args = parse_args() 116 | verbose = args.verbose 117 | keys_to_remove = ["cursor", "lsid", "projection", "limit", "signature", "$readPreference", "$db", "$clusterTime"] 118 | 119 | # Log important information 120 | logging.info(f"Starting MongoDB Prometheus Exporter with the following parameters:") 121 | logging.info(f"Wait Interval: {args.wait_interval} seconds") 122 | logging.info(f"Maximum String Size: {args.max_string_size}") 123 | logging.info(f"Listen IP: {args.listen_ip}") 124 | logging.info(f"Listen Port: {args.listen_port}") 125 | logging.info(f"Metrics Endpoint: /metrics") 126 | 127 | # Start Prometheus HTTP server 128 | start_http_server(args.listen_port, addr=args.listen_ip) 129 | 130 | slow_queries_info_last_cleared = datetime.now(ZoneInfo("UTC")) 131 | slow_queries_info_last_clear_interval=300 # seconds 132 | 133 | while True: 134 | loop_start = time.time() 135 | try: 136 | # Connect to MongoDB 137 | mongo_client = connect_to_mongo(args.mongodb_uri) 138 | if verbose: 139 | mongo_client.admin.command('ping') 140 | print("MongoDB Connection Status: Connected") 141 | 142 | # Calculate the time window 143 | end_time = datetime.now(ZoneInfo("UTC")) 144 | start_time = end_time - timedelta(seconds=args.wait_interval) 145 | 146 | # Get the list of databases 147 | databases = mongo_client.list_database_names() 148 | 149 | # Remove some dbs 150 | excluded_dbs = ["local", "admin", "config", "test"] 151 | valid_dbs = [db for db in databases if db not in excluded_dbs] 152 | if verbose: print(f"Discovered Databases: {valid_dbs}") 153 | if verbose: print(f"Start Queries Discovery in system.profile") 154 | 155 | # Iterate through valid databases and update metrics 156 | for db_name in valid_dbs: 157 | db = mongo_client[db_name] 158 | ns_values = get_ns_values(db, start_time, end_time) 159 | if ns_values: 160 | if verbose: print(f"Discovered NS in {db_name} Database: {ns_values}") 161 | for ns in ns_values: 162 | query_hash_values = get_query_hash_values(db, ns, start_time, end_time) 163 | if query_hash_values: 164 | if verbose: print(f"Discovered queryHash in {ns}: {query_hash_values}") 165 | for query_hash in query_hash_values: 166 | count = get_slow_queries_count(db, ns, query_hash, start_time, end_time) 167 | slow_queries_count_total.labels(db=db_name, ns=ns, query_hash=query_hash).inc(count) 168 | 169 | slow_queries_values = get_slow_queries_value_sum(db, ns, query_hash, start_time, end_time, fields_to_metrics_map.keys()) 170 | 171 | for field, metric in fields_to_metrics_map.items(): 172 | metric.labels(db=db_name, ns=ns, query_hash=query_hash).inc(slow_queries_values.get(field, 0)) 173 | 174 | query_info = get_query_info_values(db, ns, query_hash, start_time, end_time, keys_to_remove) 175 | 176 | # Clear slow_queries_info metric to handle queries change, such as plan_summary update 177 | if datetime.now(ZoneInfo("UTC")) - slow_queries_info_last_cleared >= timedelta(seconds=slow_queries_info_last_clear_interval): 178 | slow_queries_info._metrics.clear() 179 | slow_queries_info_last_cleared = datetime.now(ZoneInfo("UTC")) 180 | if query_info[0] != '': 181 | slow_queries_info.labels(db=db_name, ns=ns, query_hash=query_hash, 182 | query_shape=str(query_info[0])[:args.max_string_size], 183 | query_framework=query_info[1], op=query_info[2], 184 | plan_summary=str(query_info[3])[:args.max_string_size]).set(1) 185 | else: 186 | if verbose: print(f"Discovered no query_hash_values in {ns}") 187 | else: 188 | if verbose: print(f"No namespaces (NS) found in the 'system.profile' collection of the '{db_name}' database.") 189 | 190 | 191 | # Close MongoDB connection 192 | mongo_client.close() 193 | 194 | except Exception as e: 195 | logging.error(f"Error: {e}") 196 | 197 | # Calculate the time taken to execute the command 198 | elapsed = time.time() - loop_start 199 | if verbose: print(f'Elapsed loop execution time: {elapsed:.2f} seconds\n') 200 | time.sleep(max(0, args.wait_interval - elapsed - 0.0005)) # try to keep loop at interval by extracting actual execution time 201 | 202 | if __name__ == '__main__': 203 | main() 204 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | prometheus_client==0.20.0 2 | pymongo==4.7.3 3 | -------------------------------------------------------------------------------- /tests/docker-compose/.env: -------------------------------------------------------------------------------- 1 | MONGODB_VERSION=8.0 2 | -------------------------------------------------------------------------------- /tests/docker-compose/README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | cat .env 3 | docker network create tests --subnet 172.14.24.0/24 4 | docker compose up -d 5 | docker compose exec mongodb bash /scripts/prepare.sh 6 | docker compose exec mongodb bash /scripts/query.sh 7 | http://127.0.0.1:3000 8 | docker compose logs mongodb-profiler-exporter -f 9 | docker compose down 10 | ``` 11 | -------------------------------------------------------------------------------- /tests/docker-compose/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | prometheus: 3 | image: prom/prometheus:latest 4 | container_name: prometheus 5 | ports: 6 | - 9090:9090 7 | volumes: 8 | - ./prometheus:/etc/prometheus 9 | command: 10 | - '--config.file=/etc/prometheus/prometheus.yml' 11 | restart: always 12 | 13 | grafana: 14 | image: grafana/grafana:9.2.15 15 | container_name: grafana 16 | environment: 17 | - GF_SECURITY_ADMIN_PASSWORD=admin 18 | ports: 19 | - 3000:3000 20 | volumes: 21 | - ./grafana/dashboards:/etc/grafana/provisioning/dashboards 22 | - ./grafana/datasources:/etc/grafana/provisioning/datasources 23 | depends_on: 24 | - prometheus 25 | restart: always 26 | 27 | mongodb: 28 | image: mongo:${MONGODB_VERSION} 29 | container_name: mongodb 30 | ports: 31 | - 27017:27017 32 | volumes: 33 | - ./mongodb/scripts:/scripts 34 | restart: always 35 | 36 | mongodb-profiler-exporter: 37 | image: andriik/mongodb-profiler-exporter 38 | container_name: mongodb-profiler-exporter 39 | environment: 40 | MONGODB_URI: mongodb://mongodb:27017/ 41 | VERBOSE: true 42 | ports: 43 | - 9179:9179 44 | depends_on: 45 | - mongodb 46 | restart: always 47 | 48 | ## docker network create tests --subnet 172.14.24.0/24 49 | networks: 50 | default: 51 | external: true 52 | name: tests 53 | -------------------------------------------------------------------------------- /tests/docker-compose/grafana/dashboards/dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'mongodb-profiler-exporter' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | updateIntervalSeconds: 600 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /tests/docker-compose/grafana/dashboards/mongodb-profiler-exporter.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "description": "Mongo DB Profiler Prometheus Exporter. Check https://github.com/andrii29/mongodb-profiler-exporter for additional info", 25 | "editable": true, 26 | "fiscalYearStartMonth": 0, 27 | "gnetId": 20387, 28 | "graphTooltip": 0, 29 | "id": null, 30 | "links": [], 31 | "liveNow": false, 32 | "panels": [ 33 | { 34 | "collapsed": false, 35 | "gridPos": { 36 | "h": 1, 37 | "w": 24, 38 | "x": 0, 39 | "y": 0 40 | }, 41 | "id": 10, 42 | "panels": [], 43 | "title": "Main", 44 | "type": "row" 45 | }, 46 | { 47 | "datasource": { 48 | "name": "Prometheus", 49 | "type": "Prometheus" 50 | }, 51 | "fieldConfig": { 52 | "defaults": { 53 | "color": { 54 | "mode": "thresholds" 55 | }, 56 | "custom": { 57 | "align": "auto", 58 | "displayMode": "auto", 59 | "inspect": false, 60 | "minWidth": 50 61 | }, 62 | "mappings": [], 63 | "thresholds": { 64 | "mode": "absolute", 65 | "steps": [ 66 | { 67 | "color": "green", 68 | "value": null 69 | }, 70 | { 71 | "color": "red", 72 | "value": 80 73 | } 74 | ] 75 | } 76 | }, 77 | "overrides": [ 78 | { 79 | "matcher": { 80 | "id": "byName", 81 | "options": "Query Rate" 82 | }, 83 | "properties": [ 84 | { 85 | "id": "unit", 86 | "value": "short" 87 | }, 88 | { 89 | "id": "custom.width", 90 | "value": 91 91 | } 92 | ] 93 | }, 94 | { 95 | "matcher": { 96 | "id": "byName", 97 | "options": "Avg Duration" 98 | }, 99 | "properties": [ 100 | { 101 | "id": "unit", 102 | "value": "ms" 103 | }, 104 | { 105 | "id": "custom.width", 106 | "value": 97 107 | } 108 | ] 109 | }, 110 | { 111 | "matcher": { 112 | "id": "byName", 113 | "options": "Plan Summary" 114 | }, 115 | "properties": [ 116 | { 117 | "id": "custom.displayMode", 118 | "value": "color-background-solid" 119 | }, 120 | { 121 | "id": "mappings", 122 | "value": [ 123 | { 124 | "options": { 125 | "COLLSCAN": { 126 | "color": "red", 127 | "index": 0 128 | } 129 | }, 130 | "type": "value" 131 | } 132 | ] 133 | }, 134 | { 135 | "id": "custom.width", 136 | "value": 243 137 | } 138 | ] 139 | }, 140 | { 141 | "matcher": { 142 | "id": "byName", 143 | "options": "Query Hash" 144 | }, 145 | "properties": [ 146 | { 147 | "id": "custom.width", 148 | "value": 95 149 | } 150 | ] 151 | }, 152 | { 153 | "matcher": { 154 | "id": "byName", 155 | "options": "Namespace" 156 | }, 157 | "properties": [ 158 | { 159 | "id": "custom.width", 160 | "value": 138 161 | } 162 | ] 163 | }, 164 | { 165 | "matcher": { 166 | "id": "byName", 167 | "options": "Query Framework" 168 | }, 169 | "properties": [ 170 | { 171 | "id": "custom.width", 172 | "value": 126 173 | } 174 | ] 175 | }, 176 | { 177 | "matcher": { 178 | "id": "byName", 179 | "options": "Operation" 180 | }, 181 | "properties": [ 182 | { 183 | "id": "custom.width", 184 | "value": 86 185 | } 186 | ] 187 | } 188 | ] 189 | }, 190 | "gridPos": { 191 | "h": 13, 192 | "w": 24, 193 | "x": 0, 194 | "y": 1 195 | }, 196 | "id": 7, 197 | "options": { 198 | "footer": { 199 | "enablePagination": false, 200 | "fields": "", 201 | "reducer": [ 202 | "sum" 203 | ], 204 | "show": false 205 | }, 206 | "frameIndex": 5, 207 | "showHeader": true, 208 | "sortBy": [ 209 | { 210 | "desc": true, 211 | "displayName": "Avg Duration" 212 | } 213 | ] 214 | }, 215 | "pluginVersion": "9.2.15", 216 | "targets": [ 217 | { 218 | "datasource": { 219 | "name": "Prometheus", 220 | "type": "Prometheus" 221 | }, 222 | "editorMode": "code", 223 | "exemplar": false, 224 | "expr": "sum(rate(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[$__range])) by (query_hash, ns) * on (query_hash, ns) group_left(op, plan_summary, query_framework, query_shape) topk(1, max_over_time(slow_queries_info[$__range])) by (query_hash)", 225 | "format": "table", 226 | "hide": false, 227 | "instant": true, 228 | "legendFormat": "__auto", 229 | "range": false, 230 | "refId": "A" 231 | }, 232 | { 233 | "datasource": { 234 | "name": "Prometheus", 235 | "type": "Prometheus" 236 | }, 237 | "editorMode": "code", 238 | "exemplar": false, 239 | "expr": "sum (rate(slow_queries_duration_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[$__range])/rate(slow_queries_count_total{job=~\"$job\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[$__range])) by (query_hash, ns) * on (query_hash, ns) group_left(op, plan_summary, query_framework, query_shape) topk(1, max_over_time(slow_queries_info[$__range])) by (query_hash)", 240 | "format": "table", 241 | "hide": false, 242 | "instant": true, 243 | "legendFormat": "__auto", 244 | "range": false, 245 | "refId": "B" 246 | } 247 | ], 248 | "title": "Slow Queries", 249 | "transformations": [ 250 | { 251 | "id": "joinByField", 252 | "options": { 253 | "byField": "query_hash", 254 | "mode": "outer" 255 | } 256 | }, 257 | { 258 | "id": "organize", 259 | "options": { 260 | "excludeByName": { 261 | "Time": true, 262 | "Time 1": true, 263 | "Time 2": true, 264 | "db 1": true, 265 | "db 2": true, 266 | "instance 1": true, 267 | "instance 2": true, 268 | "job 1": true, 269 | "job 2": true, 270 | "ns 1": false, 271 | "ns 2": true, 272 | "op 1": false, 273 | "op 2": true, 274 | "plan_summary 1": false, 275 | "plan_summary 2": true, 276 | "query_framework 1": false, 277 | "query_framework 2": true, 278 | "query_shape 1": false, 279 | "query_shape 2": true 280 | }, 281 | "indexByName": { 282 | "Time 1": 8, 283 | "Time 2": 12, 284 | "Value #A": 6, 285 | "Value #B": 7, 286 | "db 1": 9, 287 | "db 2": 13, 288 | "instance 1": 10, 289 | "instance 2": 14, 290 | "job 1": 11, 291 | "job 2": 15, 292 | "ns 1": 1, 293 | "ns 2": 16, 294 | "op 1": 3, 295 | "op 2": 18, 296 | "plan_summary 1": 4, 297 | "plan_summary 2": 19, 298 | "query_framework 1": 2, 299 | "query_framework 2": 17, 300 | "query_hash": 0, 301 | "query_shape 1": 5, 302 | "query_shape 2": 20 303 | }, 304 | "renameByName": { 305 | "Value": "Query Rate", 306 | "Value #A": "Query Rate", 307 | "Value #B": "Avg Duration", 308 | "db 1": "", 309 | "job 2": "", 310 | "ns": "Namespace", 311 | "ns 1": "Namespace", 312 | "ns 2": "", 313 | "op": "Operation", 314 | "op 1": "Operation", 315 | "op 2": "", 316 | "plan_summary": "Plan Summary", 317 | "plan_summary 1": "Plan Summary", 318 | "plan_summary 2": "", 319 | "query_framework": "Query Framework", 320 | "query_framework 1": "Query Framework", 321 | "query_framework 2": "", 322 | "query_hash": "Query Hash", 323 | "query_shape": "Query Shape", 324 | "query_shape 1": "Query Shape", 325 | "query_shape 2": "" 326 | } 327 | } 328 | } 329 | ], 330 | "type": "table" 331 | }, 332 | { 333 | "datasource": { 334 | "type": "Prometheus", 335 | "name": "Prometheus" 336 | }, 337 | "description": "", 338 | "fieldConfig": { 339 | "defaults": { 340 | "color": { 341 | "mode": "palette-classic" 342 | }, 343 | "custom": { 344 | "axisCenteredZero": false, 345 | "axisColorMode": "text", 346 | "axisLabel": "", 347 | "axisPlacement": "auto", 348 | "barAlignment": 0, 349 | "drawStyle": "line", 350 | "fillOpacity": 0, 351 | "gradientMode": "none", 352 | "hideFrom": { 353 | "legend": false, 354 | "tooltip": false, 355 | "viz": false 356 | }, 357 | "lineInterpolation": "linear", 358 | "lineWidth": 1, 359 | "pointSize": 5, 360 | "scaleDistribution": { 361 | "type": "linear" 362 | }, 363 | "showPoints": "auto", 364 | "spanNulls": false, 365 | "stacking": { 366 | "group": "A", 367 | "mode": "none" 368 | }, 369 | "thresholdsStyle": { 370 | "mode": "off" 371 | } 372 | }, 373 | "mappings": [], 374 | "thresholds": { 375 | "mode": "absolute", 376 | "steps": [ 377 | { 378 | "color": "green", 379 | "value": null 380 | }, 381 | { 382 | "color": "red", 383 | "value": 80 384 | } 385 | ] 386 | }, 387 | "unit": "ms" 388 | }, 389 | "overrides": [] 390 | }, 391 | "gridPos": { 392 | "h": 9, 393 | "w": 12, 394 | "x": 0, 395 | "y": 14 396 | }, 397 | "id": 4, 398 | "options": { 399 | "legend": { 400 | "calcs": [ 401 | "last", 402 | "max" 403 | ], 404 | "displayMode": "table", 405 | "placement": "right", 406 | "showLegend": true, 407 | "sortBy": "Last", 408 | "sortDesc": true 409 | }, 410 | "tooltip": { 411 | "mode": "single", 412 | "sort": "none" 413 | } 414 | }, 415 | "targets": [ 416 | { 417 | "datasource": { 418 | "name": "Prometheus", 419 | "type": "Prometheus" 420 | }, 421 | "editorMode": "code", 422 | "expr": "rate(slow_queries_duration_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m])", 423 | "legendFormat": "{{ns}} - {{query_hash}}", 424 | "range": true, 425 | "refId": "A" 426 | } 427 | ], 428 | "title": "Slow Queries Duration", 429 | "type": "timeseries" 430 | }, 431 | { 432 | "datasource": { 433 | "type": "Prometheus", 434 | "name": "Prometheus" 435 | }, 436 | "description": "", 437 | "fieldConfig": { 438 | "defaults": { 439 | "color": { 440 | "mode": "palette-classic" 441 | }, 442 | "custom": { 443 | "axisCenteredZero": false, 444 | "axisColorMode": "text", 445 | "axisLabel": "", 446 | "axisPlacement": "auto", 447 | "barAlignment": 0, 448 | "drawStyle": "line", 449 | "fillOpacity": 0, 450 | "gradientMode": "none", 451 | "hideFrom": { 452 | "legend": false, 453 | "tooltip": false, 454 | "viz": false 455 | }, 456 | "lineInterpolation": "linear", 457 | "lineWidth": 1, 458 | "pointSize": 5, 459 | "scaleDistribution": { 460 | "type": "linear" 461 | }, 462 | "showPoints": "auto", 463 | "spanNulls": false, 464 | "stacking": { 465 | "group": "A", 466 | "mode": "none" 467 | }, 468 | "thresholdsStyle": { 469 | "mode": "off" 470 | } 471 | }, 472 | "mappings": [], 473 | "thresholds": { 474 | "mode": "absolute", 475 | "steps": [ 476 | { 477 | "color": "green", 478 | "value": null 479 | }, 480 | { 481 | "color": "red", 482 | "value": 80 483 | } 484 | ] 485 | }, 486 | "unit": "ms" 487 | }, 488 | "overrides": [] 489 | }, 490 | "gridPos": { 491 | "h": 9, 492 | "w": 12, 493 | "x": 12, 494 | "y": 14 495 | }, 496 | "id": 5, 497 | "options": { 498 | "legend": { 499 | "calcs": [ 500 | "last", 501 | "max" 502 | ], 503 | "displayMode": "table", 504 | "placement": "right", 505 | "showLegend": true, 506 | "sortBy": "Last", 507 | "sortDesc": true 508 | }, 509 | "tooltip": { 510 | "mode": "single", 511 | "sort": "none" 512 | } 513 | }, 514 | "targets": [ 515 | { 516 | "datasource": { 517 | "name": "Prometheus", 518 | "type": "Prometheus" 519 | }, 520 | "editorMode": "code", 521 | "expr": "rate(slow_queries_duration_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m])/rate(slow_queries_count_total{job=~\"$job\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m])", 522 | "legendFormat": "{{ns}} - {{query_hash}}", 523 | "range": true, 524 | "refId": "A" 525 | } 526 | ], 527 | "title": "Slow Queries Average Duration", 528 | "type": "timeseries" 529 | }, 530 | { 531 | "datasource": { 532 | "type": "Prometheus", 533 | "name": "Prometheus" 534 | }, 535 | "description": "More Is Better", 536 | "fieldConfig": { 537 | "defaults": { 538 | "color": { 539 | "mode": "palette-classic" 540 | }, 541 | "custom": { 542 | "axisCenteredZero": false, 543 | "axisColorMode": "text", 544 | "axisLabel": "", 545 | "axisPlacement": "auto", 546 | "axisSoftMin": 0, 547 | "barAlignment": 0, 548 | "drawStyle": "points", 549 | "fillOpacity": 0, 550 | "gradientMode": "none", 551 | "hideFrom": { 552 | "legend": false, 553 | "tooltip": false, 554 | "viz": false 555 | }, 556 | "lineInterpolation": "linear", 557 | "lineWidth": 1, 558 | "pointSize": 5, 559 | "scaleDistribution": { 560 | "type": "linear" 561 | }, 562 | "showPoints": "auto", 563 | "spanNulls": false, 564 | "stacking": { 565 | "group": "A", 566 | "mode": "none" 567 | }, 568 | "thresholdsStyle": { 569 | "mode": "off" 570 | } 571 | }, 572 | "mappings": [], 573 | "thresholds": { 574 | "mode": "absolute", 575 | "steps": [ 576 | { 577 | "color": "green" 578 | }, 579 | { 580 | "color": "red", 581 | "value": 80 582 | } 583 | ] 584 | }, 585 | "unit": "percentunit" 586 | }, 587 | "overrides": [] 588 | }, 589 | "gridPos": { 590 | "h": 9, 591 | "w": 12, 592 | "x": 0, 593 | "y": 23 594 | }, 595 | "id": 3, 596 | "options": { 597 | "legend": { 598 | "calcs": [ 599 | "last", 600 | "max" 601 | ], 602 | "displayMode": "table", 603 | "placement": "right", 604 | "showLegend": true, 605 | "sortBy": "Last", 606 | "sortDesc": false 607 | }, 608 | "tooltip": { 609 | "mode": "single", 610 | "sort": "none" 611 | } 612 | }, 613 | "targets": [ 614 | { 615 | "datasource": { 616 | "name": "Prometheus", 617 | "type": "Prometheus" 618 | }, 619 | "editorMode": "code", 620 | "expr": "rate(slow_queries_nreturned_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m]) / rate(slow_queries_docs_examined_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m]) < +Inf", 621 | "legendFormat": "{{ns}} - {{query_hash}}", 622 | "range": true, 623 | "refId": "A" 624 | } 625 | ], 626 | "title": "Slow Queries Efficiency", 627 | "type": "timeseries" 628 | }, 629 | { 630 | "datasource": { 631 | "type": "Prometheus", 632 | "name": "Prometheus" 633 | }, 634 | "description": "More Is Better", 635 | "fieldConfig": { 636 | "defaults": { 637 | "color": { 638 | "mode": "palette-classic" 639 | }, 640 | "custom": { 641 | "axisCenteredZero": false, 642 | "axisColorMode": "text", 643 | "axisLabel": "", 644 | "axisPlacement": "auto", 645 | "axisSoftMin": 1, 646 | "barAlignment": 0, 647 | "drawStyle": "points", 648 | "fillOpacity": 0, 649 | "gradientMode": "none", 650 | "hideFrom": { 651 | "legend": false, 652 | "tooltip": false, 653 | "viz": false 654 | }, 655 | "lineInterpolation": "linear", 656 | "lineWidth": 1, 657 | "pointSize": 5, 658 | "scaleDistribution": { 659 | "type": "linear" 660 | }, 661 | "showPoints": "auto", 662 | "spanNulls": false, 663 | "stacking": { 664 | "group": "A", 665 | "mode": "none" 666 | }, 667 | "thresholdsStyle": { 668 | "mode": "off" 669 | } 670 | }, 671 | "mappings": [], 672 | "thresholds": { 673 | "mode": "absolute", 674 | "steps": [ 675 | { 676 | "color": "green" 677 | }, 678 | { 679 | "color": "red", 680 | "value": 80 681 | } 682 | ] 683 | }, 684 | "unit": "short" 685 | }, 686 | "overrides": [] 687 | }, 688 | "gridPos": { 689 | "h": 9, 690 | "w": 12, 691 | "x": 12, 692 | "y": 23 693 | }, 694 | "id": 11, 695 | "options": { 696 | "legend": { 697 | "calcs": [ 698 | "last", 699 | "max" 700 | ], 701 | "displayMode": "table", 702 | "placement": "right", 703 | "showLegend": true, 704 | "sortBy": "Last", 705 | "sortDesc": true 706 | }, 707 | "tooltip": { 708 | "mode": "single", 709 | "sort": "none" 710 | } 711 | }, 712 | "targets": [ 713 | { 714 | "datasource": { 715 | "name": "Prometheus", 716 | "type": "Prometheus" 717 | }, 718 | "editorMode": "code", 719 | "expr": "rate(slow_queries_keys_examined_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m]) / rate(slow_queries_docs_examined_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m]) < +Inf", 720 | "legendFormat": "{{ns}} - {{query_hash}}", 721 | "range": true, 722 | "refId": "A" 723 | } 724 | ], 725 | "title": "Slow Queries Index Efficiency", 726 | "type": "timeseries" 727 | }, 728 | { 729 | "datasource": { 730 | "type": "Prometheus", 731 | "name": "Prometheus" 732 | }, 733 | "description": "", 734 | "fieldConfig": { 735 | "defaults": { 736 | "color": { 737 | "mode": "palette-classic" 738 | }, 739 | "custom": { 740 | "axisCenteredZero": false, 741 | "axisColorMode": "text", 742 | "axisLabel": "", 743 | "axisPlacement": "auto", 744 | "barAlignment": 0, 745 | "drawStyle": "line", 746 | "fillOpacity": 0, 747 | "gradientMode": "none", 748 | "hideFrom": { 749 | "legend": false, 750 | "tooltip": false, 751 | "viz": false 752 | }, 753 | "lineInterpolation": "linear", 754 | "lineWidth": 1, 755 | "pointSize": 5, 756 | "scaleDistribution": { 757 | "type": "linear" 758 | }, 759 | "showPoints": "auto", 760 | "spanNulls": false, 761 | "stacking": { 762 | "group": "A", 763 | "mode": "none" 764 | }, 765 | "thresholdsStyle": { 766 | "mode": "off" 767 | } 768 | }, 769 | "mappings": [], 770 | "thresholds": { 771 | "mode": "absolute", 772 | "steps": [ 773 | { 774 | "color": "green" 775 | }, 776 | { 777 | "color": "red", 778 | "value": 80 779 | } 780 | ] 781 | }, 782 | "unit": "short" 783 | }, 784 | "overrides": [] 785 | }, 786 | "gridPos": { 787 | "h": 9, 788 | "w": 12, 789 | "x": 12, 790 | "y": 32 791 | }, 792 | "id": 2, 793 | "options": { 794 | "legend": { 795 | "calcs": [ 796 | "last", 797 | "max" 798 | ], 799 | "displayMode": "table", 800 | "placement": "right", 801 | "showLegend": true, 802 | "sortBy": "Last", 803 | "sortDesc": true 804 | }, 805 | "tooltip": { 806 | "mode": "single", 807 | "sort": "none" 808 | } 809 | }, 810 | "targets": [ 811 | { 812 | "datasource": { 813 | "name": "Prometheus", 814 | "type": "Prometheus" 815 | }, 816 | "editorMode": "code", 817 | "expr": "rate(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", ns=~\"$ns\", query_hash=~\"$query_hash\" }[1m])", 818 | "legendFormat": "{{ns}} - {{query_hash}}", 819 | "range": true, 820 | "refId": "A" 821 | } 822 | ], 823 | "title": "Slow Queries Rate", 824 | "type": "timeseries" 825 | } 826 | ], 827 | "refresh": "10s", 828 | "schemaVersion": 37, 829 | "style": "dark", 830 | "tags": [], 831 | "templating": { 832 | "list": [ 833 | { 834 | "current": {}, 835 | "datasource": { 836 | "name": "Prometheus", 837 | "type": "Prometheus" 838 | }, 839 | "definition": "label_values(slow_queries_info, job)", 840 | "hide": 0, 841 | "includeAll": false, 842 | "multi": false, 843 | "name": "job", 844 | "options": [], 845 | "query": { 846 | "query": "label_values(slow_queries_info, job)", 847 | "refId": "StandardVariableQuery" 848 | }, 849 | "refresh": 2, 850 | "regex": "", 851 | "skipUrlSync": false, 852 | "sort": 1, 853 | "type": "query" 854 | }, 855 | { 856 | "current": {}, 857 | "datasource": { 858 | "name": "Prometheus", 859 | "type": "Prometheus" 860 | }, 861 | "definition": "label_values(slow_queries_count_total, instance)", 862 | "hide": 0, 863 | "includeAll": false, 864 | "multi": false, 865 | "name": "instance", 866 | "options": [], 867 | "query": { 868 | "query": "label_values(slow_queries_count_total, instance)", 869 | "refId": "StandardVariableQuery" 870 | }, 871 | "refresh": 2, 872 | "regex": "", 873 | "skipUrlSync": false, 874 | "sort": 1, 875 | "type": "query" 876 | }, 877 | { 878 | "current": {}, 879 | "datasource": { 880 | "name": "Prometheus", 881 | "type": "Prometheus" 882 | }, 883 | "definition": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\"}, db)", 884 | "hide": 0, 885 | "includeAll": false, 886 | "multi": false, 887 | "name": "db", 888 | "options": [], 889 | "query": { 890 | "query": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\"}, db)", 891 | "refId": "StandardVariableQuery" 892 | }, 893 | "refresh": 2, 894 | "regex": "", 895 | "skipUrlSync": false, 896 | "sort": 1, 897 | "type": "query" 898 | }, 899 | { 900 | "current": {}, 901 | "datasource": { 902 | "name": "Prometheus", 903 | "type": "Prometheus" 904 | }, 905 | "definition": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", db=~\"$db\"}, ns)", 906 | "hide": 0, 907 | "includeAll": true, 908 | "multi": true, 909 | "name": "ns", 910 | "options": [], 911 | "query": { 912 | "query": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", db=~\"$db\"}, ns)", 913 | "refId": "StandardVariableQuery" 914 | }, 915 | "refresh": 2, 916 | "regex": "", 917 | "skipUrlSync": false, 918 | "sort": 0, 919 | "type": "query" 920 | }, 921 | { 922 | "current": {}, 923 | "datasource": { 924 | "name": "Prometheus", 925 | "type": "Prometheus" 926 | }, 927 | "definition": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", db=~\"$db\", ns=~\"$ns\"}, query_hash)", 928 | "hide": 0, 929 | "includeAll": true, 930 | "multi": true, 931 | "name": "query_hash", 932 | "options": [], 933 | "query": { 934 | "query": "label_values(slow_queries_count_total{job=~\"$job\", instance=~\"$instance\", db=~\"$db\", ns=~\"$ns\"}, query_hash)", 935 | "refId": "StandardVariableQuery" 936 | }, 937 | "refresh": 2, 938 | "regex": "", 939 | "skipUrlSync": false, 940 | "sort": 0, 941 | "type": "query" 942 | } 943 | ] 944 | }, 945 | "time": { 946 | "from": "now-5m", 947 | "to": "now" 948 | }, 949 | "timepicker": {}, 950 | "timezone": "", 951 | "title": "MongoDB Profiler Exporter", 952 | "uid": "r5qsdbpSk", 953 | "version": 2, 954 | "weekStart": "" 955 | } 956 | -------------------------------------------------------------------------------- /tests/docker-compose/grafana/datasources/prometheus-datasource.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | access: proxy 7 | url: http://prometheus:9090 8 | isDefault: true 9 | editable: true 10 | -------------------------------------------------------------------------------- /tests/docker-compose/mongodb/scripts/generate_random_data.js: -------------------------------------------------------------------------------- 1 | var bulkOps = []; 2 | 3 | for (var i = 0; i < 1000000; i++) { 4 | bulkOps.push({ 5 | insertOne: { 6 | document: { app: Math.floor(Math.random() * 10) + 1, host: Math.floor(Math.random() * 10) + 1, guest: Math.floor(Math.random() * 10) + 1 } 7 | } 8 | }); 9 | 10 | // Insert in batches of 1000 documents to avoid overwhelming the server 11 | if (i % 1000 === 0 && i !== 0) { 12 | db.app.bulkWrite(bulkOps); 13 | bulkOps = []; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/docker-compose/mongodb/scripts/prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Generate Random Data" 4 | mongosh "mongodb://127.0.0.1:27017/rto" /scripts/generate_random_data.js 5 | echo "Create Index" 6 | mongosh "mongodb://127.0.0.1:27017/rto" --eval "db.app.createIndex({ guest: 1 })" 7 | echo "Enable Profiler" 8 | mongosh "mongodb://127.0.0.1:27017/rto" --eval "db.setProfilingLevel(2)" 9 | -------------------------------------------------------------------------------- /tests/docker-compose/mongodb/scripts/query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while true; do 4 | # Generate a random number between 1 and 10 for selecting a query 5 | random_query=$((RANDOM % 10 + 1)) 6 | 7 | case $random_query in 8 | 1) 9 | query='db.app.find({"app": 1})' 10 | ;; 11 | 2) 12 | query='db.app.find({"host": {"$gt": 5}})' 13 | ;; 14 | 3) 15 | query='db.app.find({"guest": {"$lt": 15}})' 16 | ;; 17 | 4) 18 | query='db.app.find({"host": {"$eq": 9}}).sort({host:1, app: -1})' 19 | ;; 20 | 5) 21 | query='db.app.aggregate([{"$group": {"_id": "$app", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}])' 22 | ;; 23 | 6) 24 | query='db.app.aggregate([{"$project": {"guest": 1, "host": 1}}, {"$sort": {"host": 1}}])' 25 | ;; 26 | 7) 27 | query='db.app.aggregate([{"$match": {"host": 1}}, {"$group": {"_id": "$host", "avg_host": {"$avg": "$host"}}}])' 28 | ;; 29 | 8) 30 | query='db.app.find().limit(5)' 31 | ;; 32 | 9) 33 | query='db.app.aggregate([{"$group": {"_id": null, "total_guest": {"$sum": "$guest"}}}])' 34 | ;; 35 | 10) 36 | query='db.app.find({"host": {"$gte": 5}, "app": {"$lt": 15}})' 37 | ;; 38 | *) 39 | echo "Invalid random_query number" 40 | continue 41 | ;; 42 | esac 43 | 44 | mongosh "mongodb://127.0.0.1:27017/rto" --eval "$query" 45 | 46 | sleep 0.1 47 | done 48 | -------------------------------------------------------------------------------- /tests/docker-compose/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 10s 3 | 4 | scrape_configs: 5 | - job_name: 'mongodb-profiler-exporter' 6 | static_configs: 7 | - targets: ['mongodb-profiler-exporter:9179'] 8 | --------------------------------------------------------------------------------