├── .github └── workflows │ ├── generate-previews.yml │ └── publish-dashboards.yml ├── .gitignore ├── LICENSE ├── README.md ├── dashboards └── emojivoto │ ├── emoji-popularity.jsonnet │ ├── emoji-service-grpc.jsonnet │ └── voting-service-grpc.jsonnet ├── docker ├── Dockerfile └── cleanup.sh ├── github-actions ├── preview-dashboards │ ├── Dockerfile │ ├── action.yml │ └── run.sh └── publish-dashboards │ ├── Dockerfile │ ├── action.yml │ └── run.sh ├── scripts └── preview.sh └── templates └── grpc.libsonnet /.github/workflows/generate-previews.yml: -------------------------------------------------------------------------------- 1 | name: 'Generate Previews for PR' 2 | on: pull_request 3 | jobs: 4 | generate-previews: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@master 8 | - name: generate previews 9 | id: generate 10 | uses: ./github-actions/preview-dashboards 11 | with: 12 | grafana-url: 'http://promcon-grafana.adamwg.com' 13 | env: 14 | TOKEN: ${{ secrets.GRAFANA_TOKEN }} 15 | - name: post comment 16 | uses: rytswd/respost@v0.1.0 17 | with: 18 | title: 'Generated the following dashboard previews for this PR' 19 | body: ${{ steps.generate.outputs.links }} 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /.github/workflows/publish-dashboards.yml: -------------------------------------------------------------------------------- 1 | name: 'Publish Dashboards on Merge' 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | publish-dashboards: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - name: publish dashboards 12 | uses: ./github-actions/publish-dashboards 13 | with: 14 | grafana-url: 'http://promcon-grafana.adamwg.com' 15 | env: 16 | TOKEN: ${{ secrets.GRAFANA_TOKEN }} 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | config.env 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Adam Wolfe Gordon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Managing Grafana Dashboards with grafonnet and git 2 | 3 | This repository is the demo/example used in my PromCon 2019 talk, "Managing 4 | Grafana Dashboards with grafonnet and git". 5 | -------------------------------------------------------------------------------- /dashboards/emojivoto/emoji-popularity.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local prometheus = grafana.prometheus; 4 | local graph = grafana.graphPanel; 5 | local table = grafana.tablePanel; 6 | 7 | local tableStyles = [ 8 | # Hide the time column. 9 | { 10 | alias: 'Time', 11 | pattern: 'Time', 12 | type: 'hidden', 13 | }, 14 | # Rename Metric -> Emoji and format it as a string. 15 | { 16 | alias: 'Emoji', 17 | pattern: 'Metric', 18 | type: 'string', 19 | }, 20 | # Rename Value -> Votes and format it as an integer. 21 | { 22 | alias: 'Votes', 23 | pattern: 'Value', 24 | type: 'number', 25 | unit: 'locale', 26 | decimals: 0, 27 | }, 28 | ]; 29 | 30 | # Sort the table by votes. 31 | local tableSort = { 32 | sort: { 33 | col: 2, 34 | desc: true, 35 | } 36 | }; 37 | 38 | dashboard.new( 39 | 'Emoji Popularity', 40 | tags=['emojivoto'], 41 | timezone='utc', 42 | schemaVersion=16, 43 | time_from='now-1h', 44 | ) 45 | .addPanel( 46 | graph.new( 47 | 'Emoji Popularity Over Time', 48 | format='opm', 49 | fill=0, 50 | min=0, 51 | datasource='Prometheus', 52 | legend_rightSide=true, 53 | legend_alignAsTable=true, 54 | legend_values=true, 55 | legend_current=true, 56 | legend_max=true, 57 | legend_min=true, 58 | legend_hideZero=true, 59 | legend_sort='current', 60 | legend_sortDesc=true, 61 | ) 62 | .addTarget( 63 | prometheus.target( 64 | '60*sum(rate(emojivoto_votes_total[5m])) by (emoji)', 65 | legendFormat='{{emoji}}', 66 | ) 67 | ), 68 | gridPos={x: 0, y: 0, w: 24, h: 10} 69 | ) 70 | .addPanel( 71 | table.new( 72 | 'Top 20 Emojis (last 24h)', 73 | datasource='Prometheus', 74 | styles=tableStyles, 75 | ) 76 | .addTarget( 77 | prometheus.target( 78 | 'topk(20, sum(increase(emojivoto_votes_total[24h])) by (emoji))', 79 | legendFormat='{{emoji}}', 80 | instant=true, 81 | ) 82 | ) + tableSort, 83 | gridPos={x: 0, y: 1, w: 12, h: 20} 84 | ) 85 | .addPanel( 86 | table.new( 87 | 'Top 20 Emojis (all time)', 88 | datasource='Prometheus', 89 | styles=tableStyles, 90 | ) 91 | .addTarget( 92 | prometheus.target( 93 | 'topk(20, sum(emojivoto_votes_total) by (emoji))', 94 | legendFormat='{{emoji}}', 95 | instant=true, 96 | ) 97 | ) + tableSort, 98 | gridPos={x: 12, y: 1, w: 12, h: 20} 99 | ) 100 | -------------------------------------------------------------------------------- /dashboards/emojivoto/emoji-service-grpc.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local grpc = import 'templates/grpc.libsonnet'; 4 | 5 | dashboard.new( 6 | 'Emoji Service gRPC', 7 | tags=['emojivoto','grpc','emoji-svc'], 8 | timezone='utc', 9 | schemaVersion=14, 10 | time_from='now-1h', 11 | ) 12 | .addRows( 13 | grpc.new('Prometheus', 'emojivoto.v1.EmojiService') 14 | ) 15 | -------------------------------------------------------------------------------- /dashboards/emojivoto/voting-service-grpc.jsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local dashboard = grafana.dashboard; 3 | local grpc = import 'templates/grpc.libsonnet'; 4 | 5 | dashboard.new( 6 | 'Voting Service gRPC', 7 | tags=['emojivoto','grpc','voting-svc'], 8 | timezone='utc', 9 | schemaVersion=14, 10 | time_from='now-1h', 11 | ) 12 | .addRows( 13 | grpc.new('Prometheus', 'emojivoto.v1.VotingService') 14 | ) 15 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stretch-slim 2 | MAINTAINER Adam Wolfe Gordon 3 | ARG jsonnet_version=v0.14.0 4 | 5 | ADD cleanup.sh /cleanup.sh 6 | 7 | ADD https://github.com/google/jsonnet/releases/download/${jsonnet_version}/jsonnet-bin-${jsonnet_version}-linux.tar.gz /jsonnet.tar.gz 8 | ADD https://github.com/grafana/grafonnet-lib/archive/master.tar.gz /grafonnet.tar.gz 9 | 10 | RUN apt-get update && apt-get install -y curl jq git && apt-get clean && /cleanup.sh && \ 11 | tar -C /bin -xvf /jsonnet.tar.gz && \ 12 | tar -C / -xvf /grafonnet.tar.gz && \ 13 | mv /grafonnet-lib-master /grafonnet-lib && \ 14 | rm -f /jsonnet.tar.gz /grafonnet.tar.gz 15 | -------------------------------------------------------------------------------- /docker/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | rm -rf /usr/share/doc 6 | rm -rf /usr/share/man 7 | rm -rf /usr/share/info 8 | rm -rf /usr/share/i18n 9 | rm -rf /usr/share/locale 10 | rm -rf /lib/udev 11 | rm -rf /lib/systemd 12 | rm -rf /var/lib/apt/lists/* 13 | rm -rf /var/cache/apt/archives/* 14 | rm -rf /var/cache/debconf/*old 15 | -------------------------------------------------------------------------------- /github-actions/preview-dashboards/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM adamwg/grafonnet:latest 2 | COPY run.sh /run.sh 3 | ENTRYPOINT ["/run.sh"] 4 | -------------------------------------------------------------------------------- /github-actions/preview-dashboards/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Preview Dashboards' 2 | author: 'Adam Wolfe Gordon ' 3 | description: 'Generates Grafana dashboard previews for a pull request.' 4 | inputs: 5 | grafana-url: 6 | description: 'URL of the Grafana server' 7 | required: true 8 | outputs: 9 | urls: 10 | description: 'URLs of the generated previews, comma-separated' 11 | links: 12 | description: 'Markdown-formatted links to generated previews' 13 | runs: 14 | using: 'docker' 15 | image: Dockerfile 16 | args: 17 | - ${{ inputs.grafana-url }} 18 | -------------------------------------------------------------------------------- /github-actions/preview-dashboards/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # Get the base branch for the PR, so we can determine which files are changed. 6 | base_branch=$(jq -r '.pull_request.base.ref' "${GITHUB_EVENT_PATH}") 7 | 8 | # Determine which dashboards are affected by this change. 9 | files=($(git log --format='' --name-status "origin/${base_branch}".. -- '**/*.jsonnet' | awk '/^[^D]/ {print $2}')) 10 | libfiles=($(git log --format='' --name-status "origin/${base_branch}".. -- '**/*.libsonnet' | awk '{print $2}')) 11 | dependencyFiles=() 12 | for l in "${libfiles[@]}"; do 13 | dependencies=($(grep -R --include='*.jsonnet' -l -F "import '${l}'" . || true)) 14 | dependencyFiles+=("${dependencies[@]}") 15 | done 16 | for d in "${dependencyFiles[@]}"; do 17 | found='false' 18 | for f in "${files[@]}"; do 19 | if [[ "${d}" == "./${f}" ]]; then 20 | found='true' 21 | break 22 | fi 23 | done 24 | 25 | if [[ "${found}" == 'false' ]]; then 26 | dd=$(echo "${d}" | sed -e 's/^..//') 27 | files+=("${dd}") 28 | fi 29 | done 30 | 31 | function generate_previews() { 32 | export GRAFANA="${1}" 33 | export NO_DOCKER='true' 34 | export EXPIRY='259200' # 3 days 35 | for f in "${files[@]}"; do 36 | folder=$(dirname "$f" | sed 's/^dashboards\///') 37 | dbname=$(basename "$f" | sed 's/\.jsonnet$//') 38 | 39 | url=$(scripts/preview.sh "${f}") 40 | echo "${folder}/${dbname}::${url}" 41 | done 42 | } 43 | 44 | urls='' 45 | links='' 46 | for preview in $(generate_previews "${1}"); do 47 | dbname=$(echo "${preview}" | awk -F:: '{print $1}') 48 | url=$(echo "${preview}" | awk -F:: '{print $2}') 49 | urls="${url},${urls}" 50 | links="[Preview for dashboard ${dbname}](${url})
${links}" 51 | done 52 | 53 | echo "::set-output name=links::${links}" 54 | echo "::set-output name=urls::${urls}" 55 | -------------------------------------------------------------------------------- /github-actions/publish-dashboards/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM adamwg/grafonnet:latest 2 | COPY run.sh /run.sh 3 | ENTRYPOINT ["/run.sh"] 4 | -------------------------------------------------------------------------------- /github-actions/publish-dashboards/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Publish Dashboards' 2 | author: 'Adam Wolfe Gordon ' 3 | description: 'Publishes Grafana dashboards.' 4 | inputs: 5 | grafana-url: 6 | description: 'URL of the Grafana server' 7 | required: true 8 | runs: 9 | using: 'docker' 10 | image: Dockerfile 11 | args: 12 | - ${{ inputs.grafana-url }} 13 | -------------------------------------------------------------------------------- /github-actions/publish-dashboards/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | GRAFANA="${1}" 6 | 7 | function get_folder_id() { 8 | fname="\"${1}\"" 9 | existing=$(curl -sf -X GET \ 10 | -H "Authorization: Bearer ${TOKEN}" \ 11 | -H 'Content-type: application/json' \ 12 | -H 'Accept: application/json' \ 13 | "${GRAFANA}/api/folders" | \ 14 | jq ".[] | select(.title == ${fname}) | .id") 15 | if [[ -n "${existing}" ]]; then 16 | echo "${existing}" 17 | return 0 18 | fi 19 | 20 | new=$(curl -sf -X POST \ 21 | -H "Authorization: Bearer ${TOKEN}" \ 22 | -H 'Content-type: application/json' \ 23 | -H 'Accept: application/json' \ 24 | "${GRAFANA}/api/folders" \ 25 | --data-binary "{ \"title\": ${fname} }" | \ 26 | jq '.id') 27 | echo "${new}" 28 | return 0 29 | } 30 | 31 | bad=0 32 | for f in dashboards/**/*.jsonnet ; do 33 | folder=$(dirname "$f" | sed 's/^dashboards\///') 34 | folder_id=$(get_folder_id "${folder}") 35 | dbname=$(basename "$f" | sed 's/\.jsonnet$//') 36 | 37 | echo "Updating dashboard ${folder}/${dbname}" 38 | 39 | if ! jsonnet -J /grafonnet-lib -J . "${f}" | \ 40 | jq "{ \"dashboard\": ., \"folderId\": ${folder_id}, \"overwrite\": true }" > \ 41 | "/tmp/${dbname}.json" ; then 42 | 43 | echo "Failed to build ${folder}/${dbname}" 44 | bad=$((bad+1)) 45 | continue 46 | fi 47 | 48 | if ! curl -sf -X POST \ 49 | -H "Authorization: Bearer ${TOKEN}" \ 50 | -H 'Content-type: application/json' \ 51 | -H 'Accept: application/json' \ 52 | "${GRAFANA}/api/dashboards/db" \ 53 | --data-binary "@/tmp/${dbname}.json" ; then 54 | 55 | echo "Failed to update ${folder}/${dbname}" 56 | bad=$((bad+1)) 57 | fi 58 | done 59 | 60 | exit ${bad} 61 | -------------------------------------------------------------------------------- /scripts/preview.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 5 | 6 | # Read local configuration from config.env in the repository root if it 7 | # exists. It's handy to set GRAFANA and TOKEN here so you don't have to provide 8 | # them on every invocation. 9 | source "${SCRIPTDIR}/../config.env" &> /dev/null || true 10 | 11 | function make_preview() { 12 | # EXPIRY is how many seconds a preview will be available for. Default it to 5 13 | # minutes; this can be overridden in config.env. 14 | : "${EXPIRY:=300}" 15 | 16 | # Make sure we have the necessary parameters. DASHBOARD can be provided as an 17 | # envvar or an argument to the script. 18 | : "${DASHBOARD:=${1:-}}" 19 | if [[ -z "${DASHBOARD}" ]]; then 20 | echo 'Must provide DASHBOARD' 21 | exit 1 22 | fi 23 | if [[ -z "${GRAFANA:-}" ]]; then 24 | echo 'Must provide GRAFANA' 25 | exit 1 26 | fi 27 | 28 | # Compile first so that we see any jsonnet errors. 29 | dbjson=$(mktemp) 30 | jsonnet -J /grafonnet-lib -J . "${DASHBOARD}" > "${dbjson}" 31 | 32 | # Generate the snapshot JSON in to a temporary file. 33 | json=$(mktemp) 34 | jq "{ \"dashboard\": ., \"expires\": ${EXPIRY} }" < "${dbjson}" > "${json}" 35 | 36 | # Use token authentication if we have a token. 37 | CURL=(curl -fsSL) 38 | if [[ ! -z "${TOKEN:-}" ]]; then 39 | CURL+=(-H "Authorization: Bearer ${TOKEN}") 40 | fi 41 | 42 | # Create the snapshot and fix up the URL we get back - when running grafana 43 | # under Kubernetes it thinks its URL is localhost:3000. 44 | resp=$("${CURL[@]}" -X POST -H 'Content-type: application/json' -H 'Accept: application/json' \ 45 | "${GRAFANA}/api/snapshots" --data-binary "@${json}") 46 | url=$(echo "${resp}" | jq -r ".url | sub(\"http://localhost:3000\"; \"${GRAFANA}\")") 47 | 48 | echo "${url}" 49 | } 50 | 51 | # Set NO_DOCKER to run directly on the host - otherwise we'll run under docker. 52 | : "${NO_DOCKER:=}" 53 | # Set OPEN_CMD to specify what command to use to open the preview - otherwise 54 | # we'll guess. 55 | : "${OPEN_CMD:=}" 56 | 57 | if [[ ! -z "${OPEN_CMD}" ]]; then 58 | # Use caller-provided OPEN_CMD. 59 | true 60 | elif command -v open &> /dev/null; then 61 | OPEN_CMD='open' 62 | elif command -v xdg-open &> /dev/null; then 63 | OPEN_CMD='xdg-open' 64 | else 65 | OPEN_CMD='echo' 66 | fi 67 | 68 | url="" 69 | if [[ "${NO_DOCKER}" ]]; then 70 | url=$(make_preview "${@}") 71 | else 72 | url=$(docker run -it --rm \ 73 | -v "${SCRIPTDIR}/..:/dashboards" \ 74 | -w /dashboards \ 75 | -e NO_DOCKER=true \ 76 | adamwg/grafonnet:latest \ 77 | scripts/preview.sh "${@}") 78 | fi 79 | 80 | "${OPEN_CMD}" "${url}" 81 | -------------------------------------------------------------------------------- /templates/grpc.libsonnet: -------------------------------------------------------------------------------- 1 | local grafana = import 'grafonnet/grafana.libsonnet'; 2 | local prometheus = grafana.prometheus; 3 | local graph = grafana.graphPanel; 4 | local row = grafana.row; 5 | 6 | { 7 | new(datasource, service):: 8 | [ 9 | row.new( 10 | title='Request Rates', 11 | height='300px', 12 | ) 13 | .addPanels([ 14 | graph.new( 15 | 'Request Rate by Method', 16 | span=6, 17 | format='opm', 18 | fill=0, 19 | min=0, 20 | datasource=datasource, 21 | legend_values=true, 22 | legend_current=true, 23 | legend_avg=true, 24 | legend_hideZero=true, 25 | legend_rightSide=true, 26 | legend_alignAsTable=true, 27 | ) 28 | .addTarget( 29 | prometheus.target( 30 | '60 * sum(rate(grpc_server_handled_total{grpc_service="%s"}[5m])) by (grpc_method)' % service, 31 | legendFormat='{{grpc_method}}', 32 | ) 33 | ), 34 | graph.new( 35 | 'Request Rate by Response Code', 36 | span=6, 37 | format='opm', 38 | fill=0, 39 | min=0, 40 | datasource=datasource, 41 | legend_values=true, 42 | legend_current=true, 43 | legend_avg=true, 44 | legend_hideZero=true, 45 | legend_rightSide=true, 46 | legend_alignAsTable=true, 47 | ) 48 | .addTarget( 49 | prometheus.target( 50 | '60 * sum(rate(grpc_server_handled_total{grpc_service="%s"}[5m])) by (grpc_code)' % service, 51 | legendFormat='{{grpc_code}}', 52 | ) 53 | ), 54 | ]), 55 | row.new( 56 | title='Error Rates', 57 | height='300px', 58 | ) 59 | .addPanels([ 60 | graph.new( 61 | 'Total Error Rate', 62 | span=6, 63 | format='percentunit', 64 | fill=0, 65 | min=0, 66 | datasource=datasource, 67 | legend_values=true, 68 | legend_current=true, 69 | legend_avg=true, 70 | legend_rightSide=true, 71 | legend_alignAsTable=true, 72 | ) 73 | .addTarget( 74 | prometheus.target( 75 | 'sum(rate(grpc_server_handled_total{grpc_service="%s",grpc_code!="OK"}[5m])) / sum(rate(grpc_server_handled_total{grpc_service="%s"}[5m]))' % [service, service], 76 | legendFormat='Error Rate', 77 | ) 78 | ), 79 | graph.new( 80 | 'Error Rate by Method', 81 | span=6, 82 | format='percentunit', 83 | fill=0, 84 | min=0, 85 | datasource=datasource, 86 | legend_values=true, 87 | legend_current=true, 88 | legend_avg=true, 89 | legend_hideZero=true, 90 | legend_rightSide=true, 91 | legend_alignAsTable=true, 92 | ) 93 | .addTarget( 94 | prometheus.target( 95 | 'sum(rate(grpc_server_handled_total{grpc_service="%s",grpc_code!="OK"}[5m])) by (grpc_method) / sum(rate(grpc_server_handled_total{grpc_service="%s"}[5m])) by (grpc_method)' % [service, service], 96 | legendFormat='{{grpc_method}}', 97 | ) 98 | ), 99 | ]), 100 | row.new( 101 | title='Request Durations', 102 | height='300px', 103 | ) 104 | .addPanels([ 105 | graph.new( 106 | 'p50 Duration by Method', 107 | span=4, 108 | format='s', 109 | fill=0, 110 | min=0, 111 | datasource=datasource, 112 | legend_values=true, 113 | legend_current=true, 114 | legend_avg=true, 115 | legend_rightSide=true, 116 | legend_alignAsTable=true, 117 | ) 118 | .addTarget( 119 | prometheus.target( 120 | 'histogram_quantile(0.5, sum(rate(grpc_server_handling_seconds_bucket{grpc_service="%s"}[5m])) by (le, grpc_method))' % service, 121 | legendFormat='{{grpc_method}}', 122 | ) 123 | ), 124 | graph.new( 125 | 'p90 Duration by Method', 126 | span=4, 127 | format='s', 128 | fill=0, 129 | min=0, 130 | datasource=datasource, 131 | legend_values=true, 132 | legend_current=true, 133 | legend_avg=true, 134 | legend_rightSide=true, 135 | legend_alignAsTable=true, 136 | ) 137 | .addTarget( 138 | prometheus.target( 139 | 'histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{grpc_service="%s"}[5m])) by (le, grpc_method))' % service, 140 | legendFormat='{{grpc_method}}', 141 | ) 142 | ), 143 | graph.new( 144 | 'p99 Duration by Method', 145 | span=4, 146 | format='s', 147 | fill=0, 148 | min=0, 149 | datasource=datasource, 150 | legend_values=true, 151 | legend_current=true, 152 | legend_avg=true, 153 | legend_rightSide=true, 154 | legend_alignAsTable=true, 155 | ) 156 | .addTarget( 157 | prometheus.target( 158 | 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service="%s"}[5m])) by (le, grpc_method))' % service, 159 | legendFormat='{{grpc_method}}', 160 | ) 161 | ), 162 | ]), 163 | ], 164 | } 165 | --------------------------------------------------------------------------------