├── .gitignore ├── .gitlab-ci.yml ├── Dockerfile ├── Elasticsearch Cluster Monitoring.json ├── LICENSE ├── README.md ├── elasticsearch.monitoring ├── __init__.py ├── fetch_stats.py └── templates │ └── monitoring-es.json ├── requirements.txt └── test-cluster ├── Dockerfile ├── README.md └── elasticsearch.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | .idea/ 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: docker:stable 2 | 3 | stages: 4 | - package 5 | - release 6 | 7 | variables: 8 | STAGING_REGISTRY: "registry.gitlab.com" 9 | CONTAINER_TEST_IMAGE: ${STAGING_REGISTRY}/bigdataboutique/${CI_PROJECT_NAME}:commit-${CI_COMMIT_SHA} 10 | 11 | package: 12 | stage: package 13 | services: 14 | - docker:dind 15 | before_script: 16 | - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $STAGING_REGISTRY 17 | script: 18 | - docker build --tag $CONTAINER_TEST_IMAGE . 19 | - docker push $CONTAINER_TEST_IMAGE 20 | 21 | release: 22 | stage: release 23 | cache: {} 24 | services: 25 | - docker:dind 26 | variables: 27 | GIT_STRATEGY: none 28 | before_script: 29 | - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $STAGING_REGISTRY 30 | script: 31 | - IMAGE_NAME=${STAGING_REGISTRY}/bigdataboutique/${CI_PROJECT_NAME}:`date '+%Y-%m-%d-%H%M'`-${CI_COMMIT_SHA:0:9} 32 | - echo $IMAGE_NAME 33 | - docker pull $CONTAINER_TEST_IMAGE 34 | - docker tag $CONTAINER_TEST_IMAGE $IMAGE_NAME 35 | - docker push $IMAGE_NAME 36 | - docker tag $CONTAINER_TEST_IMAGE ${STAGING_REGISTRY}/bigdataboutique/${CI_PROJECT_NAME}:latest 37 | - docker push ${STAGING_REGISTRY}/bigdataboutique/${CI_PROJECT_NAME}:latest 38 | only: 39 | - master 40 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.5.5-slim 2 | 3 | WORKDIR /app 4 | COPY requirements.txt ./ 5 | 6 | ENV PYTHONPATH=/app 7 | 8 | RUN pip install --no-cache-dir -r requirements.txt 9 | 10 | COPY . . -------------------------------------------------------------------------------- /Elasticsearch Cluster Monitoring.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 1, 19 | "iteration": 1581274544943, 20 | "links": [], 21 | "panels": [ 22 | { 23 | "cacheTimeout": null, 24 | "colorBackground": false, 25 | "colorValue": false, 26 | "colors": [ 27 | "#299c46", 28 | "rgba(237, 129, 40, 0.89)", 29 | "#d44a3a" 30 | ], 31 | "datasource": "Elasticsearch Monitor", 32 | "format": "none", 33 | "gauge": { 34 | "maxValue": 100, 35 | "minValue": 0, 36 | "show": false, 37 | "thresholdLabels": false, 38 | "thresholdMarkers": true 39 | }, 40 | "gridPos": { 41 | "h": 3, 42 | "w": 3, 43 | "x": 0, 44 | "y": 0 45 | }, 46 | "id": 16, 47 | "interval": null, 48 | "links": [], 49 | "mappingType": 1, 50 | "mappingTypes": [ 51 | { 52 | "name": "value to text", 53 | "value": 1 54 | }, 55 | { 56 | "name": "range to text", 57 | "value": 2 58 | } 59 | ], 60 | "maxDataPoints": 100, 61 | "nullPointMode": "connected", 62 | "nullText": null, 63 | "options": {}, 64 | "postfix": "", 65 | "postfixFontSize": "50%", 66 | "prefix": "", 67 | "prefixFontSize": "50%", 68 | "rangeMaps": [ 69 | { 70 | "from": "null", 71 | "text": "N/A", 72 | "to": "null" 73 | } 74 | ], 75 | "sparkline": { 76 | "fillColor": "rgba(31, 118, 189, 0.18)", 77 | "full": false, 78 | "lineColor": "rgb(31, 120, 193)", 79 | "show": true 80 | }, 81 | "tableColumn": "", 82 | "targets": [ 83 | { 84 | "bucketAggs": [ 85 | { 86 | "field": "timestamp", 87 | "id": "2", 88 | "settings": { 89 | "interval": "auto", 90 | "min_doc_count": 0, 91 | "trimEdges": 0 92 | }, 93 | "type": "date_histogram" 94 | } 95 | ], 96 | "metrics": [ 97 | { 98 | "field": "source_node.host", 99 | "id": "1", 100 | "meta": {}, 101 | "settings": {}, 102 | "type": "cardinality" 103 | } 104 | ], 105 | "query": "cluster_uuid:$cluster", 106 | "refId": "A", 107 | "timeField": "timestamp" 108 | } 109 | ], 110 | "thresholds": "", 111 | "title": "Total nodes", 112 | "type": "singlestat", 113 | "valueFontSize": "80%", 114 | "valueMaps": [ 115 | { 116 | "op": "=", 117 | "text": "N/A", 118 | "value": "null" 119 | } 120 | ], 121 | "valueName": "avg" 122 | }, 123 | { 124 | "cacheTimeout": null, 125 | "colorBackground": false, 126 | "colorValue": false, 127 | "colors": [ 128 | "#299c46", 129 | "rgba(237, 129, 40, 0.89)", 130 | "#d44a3a" 131 | ], 132 | "datasource": "Elasticsearch Monitor", 133 | "format": "none", 134 | "gauge": { 135 | "maxValue": 100, 136 | "minValue": 0, 137 | "show": false, 138 | "thresholdLabels": false, 139 | "thresholdMarkers": true 140 | }, 141 | "gridPos": { 142 | "h": 3, 143 | "w": 3, 144 | "x": 3, 145 | "y": 0 146 | }, 147 | "id": 18, 148 | "interval": null, 149 | "links": [], 150 | "mappingType": 1, 151 | "mappingTypes": [ 152 | { 153 | "name": "value to text", 154 | "value": 1 155 | }, 156 | { 157 | "name": "range to text", 158 | "value": 2 159 | } 160 | ], 161 | "maxDataPoints": 100, 162 | "nullPointMode": "connected", 163 | "nullText": null, 164 | "options": {}, 165 | "postfix": "", 166 | "postfixFontSize": "50%", 167 | "prefix": "", 168 | "prefixFontSize": "50%", 169 | "rangeMaps": [ 170 | { 171 | "from": "null", 172 | "text": "N/A", 173 | "to": "null" 174 | } 175 | ], 176 | "sparkline": { 177 | "fillColor": "rgba(31, 118, 189, 0.18)", 178 | "full": false, 179 | "lineColor": "rgb(31, 120, 193)", 180 | "show": true 181 | }, 182 | "tableColumn": "", 183 | "targets": [ 184 | { 185 | "bucketAggs": [ 186 | { 187 | "field": "timestamp", 188 | "id": "2", 189 | "settings": { 190 | "interval": "auto", 191 | "min_doc_count": 0, 192 | "trimEdges": 0 193 | }, 194 | "type": "date_histogram" 195 | } 196 | ], 197 | "metrics": [ 198 | { 199 | "field": "source_node.name", 200 | "id": "1", 201 | "meta": {}, 202 | "settings": {}, 203 | "type": "cardinality" 204 | } 205 | ], 206 | "query": "node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND cluster_uuid:$cluster", 207 | "refId": "A", 208 | "timeField": "timestamp" 209 | } 210 | ], 211 | "thresholds": "", 212 | "title": "Data nodes", 213 | "type": "singlestat", 214 | "valueFontSize": "80%", 215 | "valueMaps": [ 216 | { 217 | "op": "=", 218 | "text": "N/A", 219 | "value": "null" 220 | } 221 | ], 222 | "valueName": "avg" 223 | }, 224 | { 225 | "columns": [ 226 | { 227 | "text": "cluster_stats.status", 228 | "value": "cluster_stats.status" 229 | } 230 | ], 231 | "datasource": null, 232 | "fontSize": "100%", 233 | "gridPos": { 234 | "h": 3, 235 | "w": 3, 236 | "x": 6, 237 | "y": 0 238 | }, 239 | "id": 43, 240 | "options": {}, 241 | "pageSize": null, 242 | "showHeader": true, 243 | "sort": { 244 | "col": 0, 245 | "desc": true 246 | }, 247 | "styles": [], 248 | "targets": [ 249 | { 250 | "bucketAggs": [], 251 | "metrics": [ 252 | { 253 | "field": "select field", 254 | "id": "1", 255 | "meta": {}, 256 | "settings": { 257 | "size": 1 258 | }, 259 | "type": "raw_document" 260 | } 261 | ], 262 | "query": "_exists_:cluster_state.status AND cluster_uuid:$cluster", 263 | "refId": "A", 264 | "timeField": "timestamp" 265 | } 266 | ], 267 | "timeFrom": null, 268 | "timeShift": null, 269 | "title": "Cluster Status", 270 | "transform": "json", 271 | "type": "table" 272 | }, 273 | { 274 | "collapsed": false, 275 | "datasource": null, 276 | "gridPos": { 277 | "h": 1, 278 | "w": 24, 279 | "x": 0, 280 | "y": 3 281 | }, 282 | "id": 31, 283 | "panels": [], 284 | "title": "Operations", 285 | "type": "row" 286 | }, 287 | { 288 | "aliasColors": {}, 289 | "bars": false, 290 | "dashLength": 10, 291 | "dashes": false, 292 | "datasource": "Elasticsearch Monitor", 293 | "fill": 1, 294 | "fillGradient": 0, 295 | "gridPos": { 296 | "h": 9, 297 | "w": 12, 298 | "x": 0, 299 | "y": 4 300 | }, 301 | "hiddenSeries": false, 302 | "id": 32, 303 | "legend": { 304 | "avg": false, 305 | "current": false, 306 | "max": true, 307 | "min": true, 308 | "show": true, 309 | "total": false, 310 | "values": true 311 | }, 312 | "lines": true, 313 | "linewidth": 1, 314 | "links": [], 315 | "nullPointMode": "null", 316 | "options": { 317 | "dataLinks": [] 318 | }, 319 | "percentage": false, 320 | "pointradius": 5, 321 | "points": false, 322 | "renderer": "flot", 323 | "seriesOverrides": [ 324 | { 325 | "alias": "/indexing time/", 326 | "yaxis": 2 327 | } 328 | ], 329 | "spaceLength": 10, 330 | "stack": false, 331 | "steppedLine": false, 332 | "targets": [ 333 | { 334 | "alias": "{{source_node.name}} indexing ops", 335 | "bucketAggs": [ 336 | { 337 | "fake": true, 338 | "field": "source_node.name", 339 | "id": "4", 340 | "settings": { 341 | "min_doc_count": 1, 342 | "order": "desc", 343 | "orderBy": "_term", 344 | "size": "0" 345 | }, 346 | "type": "terms" 347 | }, 348 | { 349 | "field": "timestamp", 350 | "id": "2", 351 | "settings": { 352 | "interval": "auto", 353 | "min_doc_count": 0, 354 | "trimEdges": 0 355 | }, 356 | "type": "date_histogram" 357 | } 358 | ], 359 | "metrics": [ 360 | { 361 | "field": "node_stats.indices.indexing.index_total", 362 | "hide": true, 363 | "id": "1", 364 | "meta": {}, 365 | "settings": {}, 366 | "type": "max" 367 | }, 368 | { 369 | "field": "1", 370 | "id": "5", 371 | "meta": {}, 372 | "pipelineAgg": "1", 373 | "settings": {}, 374 | "type": "derivative" 375 | } 376 | ], 377 | "query": "type:node_stats AND node_stats.node_master:false AND source_node.name:($host) AND cluster_uuid:$cluster", 378 | "refId": "A", 379 | "timeField": "timestamp" 380 | }, 381 | { 382 | "alias": "{{source_node.name}} indexing time", 383 | "bucketAggs": [ 384 | { 385 | "fake": true, 386 | "field": "source_node.name", 387 | "id": "4", 388 | "settings": { 389 | "min_doc_count": 1, 390 | "order": "desc", 391 | "orderBy": "_term", 392 | "size": "0" 393 | }, 394 | "type": "terms" 395 | }, 396 | { 397 | "field": "timestamp", 398 | "id": "2", 399 | "settings": { 400 | "interval": "auto", 401 | "min_doc_count": 0, 402 | "trimEdges": 0 403 | }, 404 | "type": "date_histogram" 405 | } 406 | ], 407 | "metrics": [ 408 | { 409 | "field": "node_stats.indices.indexing.index_time_in_millis", 410 | "hide": true, 411 | "id": "1", 412 | "meta": {}, 413 | "settings": {}, 414 | "type": "max" 415 | }, 416 | { 417 | "field": "1", 418 | "id": "5", 419 | "meta": {}, 420 | "pipelineAgg": "1", 421 | "settings": {}, 422 | "type": "derivative" 423 | } 424 | ], 425 | "query": "type:node_stats AND node_stats.node_master:false AND source_node.name:($host) AND cluster_uuid:$cluster", 426 | "refId": "B", 427 | "timeField": "timestamp" 428 | } 429 | ], 430 | "thresholds": [], 431 | "timeFrom": null, 432 | "timeRegions": [], 433 | "timeShift": null, 434 | "title": "Indexing", 435 | "tooltip": { 436 | "shared": true, 437 | "sort": 0, 438 | "value_type": "individual" 439 | }, 440 | "type": "graph", 441 | "xaxis": { 442 | "buckets": null, 443 | "mode": "time", 444 | "name": null, 445 | "show": true, 446 | "values": [] 447 | }, 448 | "yaxes": [ 449 | { 450 | "format": "reqs", 451 | "label": null, 452 | "logBase": 1, 453 | "max": null, 454 | "min": "0", 455 | "show": true 456 | }, 457 | { 458 | "format": "ms", 459 | "label": null, 460 | "logBase": 1, 461 | "max": null, 462 | "min": "0", 463 | "show": true 464 | } 465 | ], 466 | "yaxis": { 467 | "align": false, 468 | "alignLevel": null 469 | } 470 | }, 471 | { 472 | "aliasColors": {}, 473 | "bars": false, 474 | "dashLength": 10, 475 | "dashes": false, 476 | "datasource": "Elasticsearch Monitor", 477 | "fill": 1, 478 | "fillGradient": 0, 479 | "gridPos": { 480 | "h": 9, 481 | "w": 12, 482 | "x": 12, 483 | "y": 4 484 | }, 485 | "hiddenSeries": false, 486 | "id": 33, 487 | "legend": { 488 | "avg": false, 489 | "current": false, 490 | "hideEmpty": false, 491 | "hideZero": false, 492 | "max": true, 493 | "min": true, 494 | "show": true, 495 | "total": false, 496 | "values": true 497 | }, 498 | "lines": true, 499 | "linewidth": 1, 500 | "links": [], 501 | "nullPointMode": "connected", 502 | "options": { 503 | "dataLinks": [] 504 | }, 505 | "percentage": false, 506 | "pointradius": 5, 507 | "points": false, 508 | "renderer": "flot", 509 | "seriesOverrides": [ 510 | { 511 | "alias": "/queries time/", 512 | "yaxis": 2 513 | } 514 | ], 515 | "spaceLength": 10, 516 | "stack": false, 517 | "steppedLine": false, 518 | "targets": [ 519 | { 520 | "alias": "{{source_node.name}} queries", 521 | "bucketAggs": [ 522 | { 523 | "fake": true, 524 | "field": "source_node.name", 525 | "id": "4", 526 | "settings": { 527 | "min_doc_count": 1, 528 | "missing": null, 529 | "order": "desc", 530 | "orderBy": "1", 531 | "size": "10" 532 | }, 533 | "type": "terms" 534 | }, 535 | { 536 | "field": "timestamp", 537 | "id": "2", 538 | "settings": { 539 | "interval": "auto", 540 | "min_doc_count": 0, 541 | "trimEdges": 0 542 | }, 543 | "type": "date_histogram" 544 | } 545 | ], 546 | "metrics": [ 547 | { 548 | "field": "node_stats.indices.search.query_total", 549 | "hide": true, 550 | "id": "1", 551 | "inlineScript": null, 552 | "meta": {}, 553 | "settings": {}, 554 | "type": "max" 555 | }, 556 | { 557 | "field": "1", 558 | "id": "5", 559 | "meta": {}, 560 | "pipelineAgg": "1", 561 | "settings": {}, 562 | "type": "derivative" 563 | } 564 | ], 565 | "query": "type:node_stats AND node_stats.node_master:false AND source_node.name:($host) AND cluster_uuid:$cluster", 566 | "refId": "B", 567 | "timeField": "timestamp" 568 | }, 569 | { 570 | "alias": "{{source_node.name}} queries time", 571 | "bucketAggs": [ 572 | { 573 | "fake": true, 574 | "field": "source_node.name", 575 | "id": "4", 576 | "settings": { 577 | "min_doc_count": 1, 578 | "order": "desc", 579 | "orderBy": "1", 580 | "size": "0" 581 | }, 582 | "type": "terms" 583 | }, 584 | { 585 | "field": "timestamp", 586 | "id": "2", 587 | "settings": { 588 | "interval": "auto", 589 | "min_doc_count": 0, 590 | "trimEdges": 0 591 | }, 592 | "type": "date_histogram" 593 | } 594 | ], 595 | "metrics": [ 596 | { 597 | "field": "node_stats.indices.search.query_time_in_millis", 598 | "hide": true, 599 | "id": "1", 600 | "meta": {}, 601 | "settings": {}, 602 | "type": "max" 603 | }, 604 | { 605 | "field": "1", 606 | "id": "5", 607 | "meta": {}, 608 | "pipelineAgg": "1", 609 | "settings": {}, 610 | "type": "derivative" 611 | } 612 | ], 613 | "query": "type:node_stats AND node_stats.node_master:false AND source_node.name:($host) AND cluster_uuid:$cluster", 614 | "refId": "A", 615 | "timeField": "timestamp" 616 | } 617 | ], 618 | "thresholds": [], 619 | "timeFrom": null, 620 | "timeRegions": [], 621 | "timeShift": null, 622 | "title": "Queries", 623 | "tooltip": { 624 | "shared": true, 625 | "sort": 0, 626 | "value_type": "individual" 627 | }, 628 | "type": "graph", 629 | "xaxis": { 630 | "buckets": null, 631 | "mode": "time", 632 | "name": null, 633 | "show": true, 634 | "values": [] 635 | }, 636 | "yaxes": [ 637 | { 638 | "format": "reqs", 639 | "label": null, 640 | "logBase": 1, 641 | "max": null, 642 | "min": "0", 643 | "show": true 644 | }, 645 | { 646 | "format": "ms", 647 | "label": null, 648 | "logBase": 1, 649 | "max": null, 650 | "min": "0", 651 | "show": true 652 | } 653 | ], 654 | "yaxis": { 655 | "align": false, 656 | "alignLevel": null 657 | } 658 | }, 659 | { 660 | "aliasColors": {}, 661 | "bars": false, 662 | "dashLength": 10, 663 | "dashes": false, 664 | "datasource": "Elasticsearch Monitor", 665 | "fill": 1, 666 | "fillGradient": 0, 667 | "gridPos": { 668 | "h": 9, 669 | "w": 12, 670 | "x": 0, 671 | "y": 13 672 | }, 673 | "hiddenSeries": false, 674 | "id": 34, 675 | "legend": { 676 | "avg": false, 677 | "current": false, 678 | "max": true, 679 | "min": true, 680 | "show": true, 681 | "total": false, 682 | "values": true 683 | }, 684 | "lines": true, 685 | "linewidth": 1, 686 | "links": [], 687 | "nullPointMode": "connected", 688 | "options": { 689 | "dataLinks": [] 690 | }, 691 | "percentage": false, 692 | "pointradius": 5, 693 | "points": false, 694 | "renderer": "flot", 695 | "seriesOverrides": [ 696 | { 697 | "alias": "/queries time/", 698 | "yaxis": 2 699 | } 700 | ], 701 | "spaceLength": 10, 702 | "stack": false, 703 | "steppedLine": false, 704 | "targets": [ 705 | { 706 | "alias": "{{index_stats.index}} merges", 707 | "bucketAggs": [ 708 | { 709 | "fake": true, 710 | "field": "index_stats.index", 711 | "id": "4", 712 | "settings": { 713 | "min_doc_count": 1, 714 | "missing": "0", 715 | "order": "desc", 716 | "orderBy": "_term", 717 | "size": "0" 718 | }, 719 | "type": "terms" 720 | }, 721 | { 722 | "field": "timestamp", 723 | "id": "2", 724 | "settings": { 725 | "interval": "auto", 726 | "min_doc_count": 0, 727 | "trimEdges": 0 728 | }, 729 | "type": "date_histogram" 730 | } 731 | ], 732 | "metrics": [ 733 | { 734 | "field": "node_stats.indices.query_cache.evictions", 735 | "id": "1", 736 | "meta": {}, 737 | "settings": {}, 738 | "type": "max" 739 | }, 740 | { 741 | "field": "1", 742 | "id": "5", 743 | "meta": {}, 744 | "pipelineAgg": "1", 745 | "settings": {}, 746 | "type": "derivative" 747 | } 748 | ], 749 | "query": "cluster_uuid:$cluster", 750 | "refId": "B", 751 | "timeField": "timestamp" 752 | } 753 | ], 754 | "thresholds": [], 755 | "timeFrom": null, 756 | "timeRegions": [], 757 | "timeShift": null, 758 | "title": "Merges", 759 | "tooltip": { 760 | "shared": true, 761 | "sort": 0, 762 | "value_type": "individual" 763 | }, 764 | "type": "graph", 765 | "xaxis": { 766 | "buckets": null, 767 | "mode": "time", 768 | "name": null, 769 | "show": true, 770 | "values": [] 771 | }, 772 | "yaxes": [ 773 | { 774 | "format": "decbytes", 775 | "label": null, 776 | "logBase": 1, 777 | "max": null, 778 | "min": "0", 779 | "show": true 780 | }, 781 | { 782 | "format": "ms", 783 | "label": null, 784 | "logBase": 1, 785 | "max": null, 786 | "min": "0", 787 | "show": false 788 | } 789 | ], 790 | "yaxis": { 791 | "align": false, 792 | "alignLevel": null 793 | } 794 | }, 795 | { 796 | "aliasColors": {}, 797 | "bars": false, 798 | "dashLength": 10, 799 | "dashes": false, 800 | "datasource": "Elasticsearch Monitor", 801 | "fill": 1, 802 | "fillGradient": 0, 803 | "gridPos": { 804 | "h": 9, 805 | "w": 12, 806 | "x": 12, 807 | "y": 13 808 | }, 809 | "hiddenSeries": false, 810 | "id": 39, 811 | "legend": { 812 | "avg": false, 813 | "current": false, 814 | "max": true, 815 | "min": true, 816 | "show": true, 817 | "total": false, 818 | "values": true 819 | }, 820 | "lines": true, 821 | "linewidth": 1, 822 | "links": [], 823 | "nullPointMode": "connected", 824 | "options": { 825 | "dataLinks": [] 826 | }, 827 | "percentage": false, 828 | "pointradius": 5, 829 | "points": false, 830 | "renderer": "flot", 831 | "seriesOverrides": [ 832 | { 833 | "alias": "/queries time/", 834 | "yaxis": 2 835 | } 836 | ], 837 | "spaceLength": 10, 838 | "stack": false, 839 | "steppedLine": false, 840 | "targets": [ 841 | { 842 | "alias": "{{index_stats.index}} merges", 843 | "bucketAggs": [ 844 | { 845 | "fake": true, 846 | "field": "index_stats.index", 847 | "id": "4", 848 | "settings": { 849 | "min_doc_count": 1, 850 | "order": "desc", 851 | "orderBy": "_term", 852 | "size": "0" 853 | }, 854 | "type": "terms" 855 | }, 856 | { 857 | "field": "timestamp", 858 | "id": "2", 859 | "settings": { 860 | "interval": "auto", 861 | "min_doc_count": 0, 862 | "trimEdges": 0 863 | }, 864 | "type": "date_histogram" 865 | } 866 | ], 867 | "metrics": [ 868 | { 869 | "field": "index_stats.primaries.segments.count", 870 | "id": "1", 871 | "meta": {}, 872 | "settings": {}, 873 | "type": "max" 874 | }, 875 | { 876 | "field": "1", 877 | "hide": true, 878 | "id": "5", 879 | "meta": {}, 880 | "pipelineAgg": "1", 881 | "settings": {}, 882 | "type": "derivative" 883 | } 884 | ], 885 | "query": "cluster_uuid:$cluster", 886 | "refId": "B", 887 | "timeField": "timestamp" 888 | } 889 | ], 890 | "thresholds": [], 891 | "timeFrom": null, 892 | "timeRegions": [], 893 | "timeShift": null, 894 | "title": "Segments Count", 895 | "tooltip": { 896 | "shared": true, 897 | "sort": 0, 898 | "value_type": "individual" 899 | }, 900 | "type": "graph", 901 | "xaxis": { 902 | "buckets": null, 903 | "mode": "time", 904 | "name": null, 905 | "show": true, 906 | "values": [] 907 | }, 908 | "yaxes": [ 909 | { 910 | "format": "short", 911 | "label": null, 912 | "logBase": 1, 913 | "max": null, 914 | "min": "0", 915 | "show": true 916 | }, 917 | { 918 | "format": "ms", 919 | "label": null, 920 | "logBase": 1, 921 | "max": null, 922 | "min": "0", 923 | "show": false 924 | } 925 | ], 926 | "yaxis": { 927 | "align": false, 928 | "alignLevel": null 929 | } 930 | }, 931 | { 932 | "collapsed": false, 933 | "datasource": null, 934 | "gridPos": { 935 | "h": 1, 936 | "w": 24, 937 | "x": 0, 938 | "y": 22 939 | }, 940 | "id": 7, 941 | "panels": [], 942 | "title": "Machine stats", 943 | "type": "row" 944 | }, 945 | { 946 | "aliasColors": {}, 947 | "bars": false, 948 | "dashLength": 10, 949 | "dashes": false, 950 | "datasource": "Elasticsearch Monitor", 951 | "fill": 1, 952 | "fillGradient": 0, 953 | "gridPos": { 954 | "h": 9, 955 | "w": 12, 956 | "x": 0, 957 | "y": 23 958 | }, 959 | "hiddenSeries": false, 960 | "id": 40, 961 | "legend": { 962 | "avg": false, 963 | "current": false, 964 | "max": true, 965 | "min": true, 966 | "show": true, 967 | "total": false, 968 | "values": true 969 | }, 970 | "lines": true, 971 | "linewidth": 1, 972 | "links": [], 973 | "nullPointMode": "connected", 974 | "options": { 975 | "dataLinks": [] 976 | }, 977 | "percentage": false, 978 | "pointradius": 5, 979 | "points": false, 980 | "renderer": "flot", 981 | "seriesOverrides": [], 982 | "spaceLength": 10, 983 | "stack": false, 984 | "steppedLine": false, 985 | "targets": [ 986 | { 987 | "alias": "{{source_node.name}} load avg 1m", 988 | "bucketAggs": [ 989 | { 990 | "fake": true, 991 | "field": "source_node.name", 992 | "id": "4", 993 | "settings": { 994 | "min_doc_count": 1, 995 | "order": "desc", 996 | "orderBy": "_term", 997 | "size": "0" 998 | }, 999 | "type": "terms" 1000 | }, 1001 | { 1002 | "field": "timestamp", 1003 | "id": "2", 1004 | "settings": { 1005 | "interval": "auto", 1006 | "min_doc_count": 0, 1007 | "trimEdges": 0 1008 | }, 1009 | "type": "date_histogram" 1010 | } 1011 | ], 1012 | "hide": true, 1013 | "metrics": [ 1014 | { 1015 | "field": "node_stats.os.cpu.load_average.1m", 1016 | "id": "1", 1017 | "meta": {}, 1018 | "settings": {}, 1019 | "type": "max" 1020 | } 1021 | ], 1022 | "query": "source_node.name:($host) AND cluster_uuid:$cluster", 1023 | "refId": "A", 1024 | "timeField": "timestamp" 1025 | }, 1026 | { 1027 | "alias": "{{source_node.name}} load avg 5m", 1028 | "bucketAggs": [ 1029 | { 1030 | "fake": true, 1031 | "field": "source_node.name", 1032 | "id": "4", 1033 | "settings": { 1034 | "min_doc_count": 1, 1035 | "order": "desc", 1036 | "orderBy": "_term", 1037 | "size": "0" 1038 | }, 1039 | "type": "terms" 1040 | }, 1041 | { 1042 | "field": "timestamp", 1043 | "id": "2", 1044 | "settings": { 1045 | "interval": "auto", 1046 | "min_doc_count": 0, 1047 | "trimEdges": 0 1048 | }, 1049 | "type": "date_histogram" 1050 | } 1051 | ], 1052 | "hide": false, 1053 | "metrics": [ 1054 | { 1055 | "field": "node_stats.os.cpu.load_average.5m", 1056 | "id": "1", 1057 | "meta": {}, 1058 | "settings": {}, 1059 | "type": "max" 1060 | } 1061 | ], 1062 | "query": "source_node.name:($host) AND cluster_uuid:$cluster", 1063 | "refId": "B", 1064 | "timeField": "timestamp" 1065 | }, 1066 | { 1067 | "alias": "{{source_node.name}} load avg 15m", 1068 | "bucketAggs": [ 1069 | { 1070 | "fake": true, 1071 | "field": "source_node.name", 1072 | "id": "4", 1073 | "settings": { 1074 | "min_doc_count": 1, 1075 | "order": "desc", 1076 | "orderBy": "_term", 1077 | "size": "0" 1078 | }, 1079 | "type": "terms" 1080 | }, 1081 | { 1082 | "field": "timestamp", 1083 | "id": "2", 1084 | "settings": { 1085 | "interval": "auto", 1086 | "min_doc_count": 0, 1087 | "trimEdges": 0 1088 | }, 1089 | "type": "date_histogram" 1090 | } 1091 | ], 1092 | "metrics": [ 1093 | { 1094 | "field": "node_stats.os.cpu.load_average.15m", 1095 | "id": "1", 1096 | "meta": {}, 1097 | "settings": {}, 1098 | "type": "max" 1099 | } 1100 | ], 1101 | "query": "source_node.name:($host) AND cluster_uuid:$cluster", 1102 | "refId": "C", 1103 | "timeField": "timestamp" 1104 | } 1105 | ], 1106 | "thresholds": [], 1107 | "timeFrom": null, 1108 | "timeRegions": [], 1109 | "timeShift": null, 1110 | "title": "Load Average", 1111 | "tooltip": { 1112 | "shared": true, 1113 | "sort": 0, 1114 | "value_type": "individual" 1115 | }, 1116 | "type": "graph", 1117 | "xaxis": { 1118 | "buckets": null, 1119 | "mode": "time", 1120 | "name": null, 1121 | "show": true, 1122 | "values": [] 1123 | }, 1124 | "yaxes": [ 1125 | { 1126 | "format": "short", 1127 | "label": null, 1128 | "logBase": 1, 1129 | "max": null, 1130 | "min": "0", 1131 | "show": true 1132 | }, 1133 | { 1134 | "format": "short", 1135 | "label": null, 1136 | "logBase": 1, 1137 | "max": null, 1138 | "min": null, 1139 | "show": true 1140 | } 1141 | ], 1142 | "yaxis": { 1143 | "align": false, 1144 | "alignLevel": null 1145 | } 1146 | }, 1147 | { 1148 | "aliasColors": {}, 1149 | "bars": false, 1150 | "dashLength": 10, 1151 | "dashes": false, 1152 | "datasource": "Elasticsearch Monitor", 1153 | "fill": 1, 1154 | "fillGradient": 0, 1155 | "gridPos": { 1156 | "h": 9, 1157 | "w": 12, 1158 | "x": 12, 1159 | "y": 23 1160 | }, 1161 | "hiddenSeries": false, 1162 | "id": 28, 1163 | "legend": { 1164 | "avg": false, 1165 | "current": false, 1166 | "max": true, 1167 | "min": true, 1168 | "show": true, 1169 | "total": false, 1170 | "values": true 1171 | }, 1172 | "lines": true, 1173 | "linewidth": 1, 1174 | "links": [], 1175 | "nullPointMode": "connected", 1176 | "options": { 1177 | "dataLinks": [] 1178 | }, 1179 | "percentage": false, 1180 | "pointradius": 5, 1181 | "points": false, 1182 | "renderer": "flot", 1183 | "seriesOverrides": [], 1184 | "spaceLength": 10, 1185 | "stack": false, 1186 | "steppedLine": false, 1187 | "targets": [ 1188 | { 1189 | "alias": "{{source_node.name}} writes / sec", 1190 | "bucketAggs": [ 1191 | { 1192 | "fake": true, 1193 | "field": "source_node.name", 1194 | "id": "4", 1195 | "settings": { 1196 | "min_doc_count": 1, 1197 | "order": "desc", 1198 | "orderBy": "_term", 1199 | "size": "0" 1200 | }, 1201 | "type": "terms" 1202 | }, 1203 | { 1204 | "field": "timestamp", 1205 | "id": "2", 1206 | "settings": { 1207 | "interval": "auto", 1208 | "min_doc_count": 0, 1209 | "trimEdges": 0 1210 | }, 1211 | "type": "date_histogram" 1212 | } 1213 | ], 1214 | "metrics": [ 1215 | { 1216 | "field": "node_stats.process.cpu.percent", 1217 | "id": "1", 1218 | "meta": {}, 1219 | "settings": {}, 1220 | "type": "max" 1221 | } 1222 | ], 1223 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1224 | "refId": "A", 1225 | "timeField": "timestamp" 1226 | } 1227 | ], 1228 | "thresholds": [], 1229 | "timeFrom": null, 1230 | "timeRegions": [], 1231 | "timeShift": null, 1232 | "title": "CPU Percent", 1233 | "tooltip": { 1234 | "shared": true, 1235 | "sort": 0, 1236 | "value_type": "individual" 1237 | }, 1238 | "type": "graph", 1239 | "xaxis": { 1240 | "buckets": null, 1241 | "mode": "time", 1242 | "name": null, 1243 | "show": true, 1244 | "values": [] 1245 | }, 1246 | "yaxes": [ 1247 | { 1248 | "format": "percent", 1249 | "label": null, 1250 | "logBase": 1, 1251 | "max": null, 1252 | "min": "0", 1253 | "show": true 1254 | }, 1255 | { 1256 | "format": "short", 1257 | "label": null, 1258 | "logBase": 1, 1259 | "max": null, 1260 | "min": null, 1261 | "show": false 1262 | } 1263 | ], 1264 | "yaxis": { 1265 | "align": false, 1266 | "alignLevel": null 1267 | } 1268 | }, 1269 | { 1270 | "aliasColors": {}, 1271 | "bars": false, 1272 | "dashLength": 10, 1273 | "dashes": false, 1274 | "datasource": "Elasticsearch Monitor", 1275 | "fill": 1, 1276 | "fillGradient": 0, 1277 | "gridPos": { 1278 | "h": 9, 1279 | "w": 12, 1280 | "x": 0, 1281 | "y": 32 1282 | }, 1283 | "hiddenSeries": false, 1284 | "id": 2, 1285 | "legend": { 1286 | "avg": false, 1287 | "current": false, 1288 | "max": true, 1289 | "min": true, 1290 | "show": true, 1291 | "total": false, 1292 | "values": true 1293 | }, 1294 | "lines": true, 1295 | "linewidth": 1, 1296 | "links": [], 1297 | "nullPointMode": "connected", 1298 | "options": { 1299 | "dataLinks": [] 1300 | }, 1301 | "percentage": false, 1302 | "pointradius": 5, 1303 | "points": false, 1304 | "renderer": "flot", 1305 | "seriesOverrides": [], 1306 | "spaceLength": 10, 1307 | "stack": false, 1308 | "steppedLine": false, 1309 | "targets": [ 1310 | { 1311 | "bucketAggs": [ 1312 | { 1313 | "fake": true, 1314 | "field": "source_node.name", 1315 | "id": "4", 1316 | "settings": { 1317 | "min_doc_count": 1, 1318 | "order": "desc", 1319 | "orderBy": "_term", 1320 | "size": "0" 1321 | }, 1322 | "type": "terms" 1323 | }, 1324 | { 1325 | "field": "timestamp", 1326 | "id": "2", 1327 | "settings": { 1328 | "interval": "auto", 1329 | "min_doc_count": 0, 1330 | "trimEdges": 0 1331 | }, 1332 | "type": "date_histogram" 1333 | } 1334 | ], 1335 | "metrics": [ 1336 | { 1337 | "field": "node_stats.fs.io_stats.total.read_kilobytes", 1338 | "hide": true, 1339 | "id": "1", 1340 | "meta": {}, 1341 | "settings": {}, 1342 | "type": "max" 1343 | }, 1344 | { 1345 | "field": "1", 1346 | "id": "5", 1347 | "meta": {}, 1348 | "pipelineAgg": "1", 1349 | "settings": {}, 1350 | "type": "derivative" 1351 | } 1352 | ], 1353 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1354 | "refId": "A", 1355 | "timeField": "timestamp" 1356 | } 1357 | ], 1358 | "thresholds": [], 1359 | "timeFrom": null, 1360 | "timeRegions": [], 1361 | "timeShift": null, 1362 | "title": "iostat reads / sec", 1363 | "tooltip": { 1364 | "shared": true, 1365 | "sort": 0, 1366 | "value_type": "individual" 1367 | }, 1368 | "type": "graph", 1369 | "xaxis": { 1370 | "buckets": null, 1371 | "mode": "time", 1372 | "name": null, 1373 | "show": true, 1374 | "values": [] 1375 | }, 1376 | "yaxes": [ 1377 | { 1378 | "format": "rps", 1379 | "label": null, 1380 | "logBase": 1, 1381 | "max": null, 1382 | "min": null, 1383 | "show": true 1384 | }, 1385 | { 1386 | "format": "short", 1387 | "label": null, 1388 | "logBase": 1, 1389 | "max": null, 1390 | "min": null, 1391 | "show": true 1392 | } 1393 | ], 1394 | "yaxis": { 1395 | "align": false, 1396 | "alignLevel": null 1397 | } 1398 | }, 1399 | { 1400 | "aliasColors": {}, 1401 | "bars": false, 1402 | "dashLength": 10, 1403 | "dashes": false, 1404 | "datasource": "Elasticsearch Monitor", 1405 | "fill": 1, 1406 | "fillGradient": 0, 1407 | "gridPos": { 1408 | "h": 9, 1409 | "w": 12, 1410 | "x": 12, 1411 | "y": 32 1412 | }, 1413 | "hiddenSeries": false, 1414 | "id": 3, 1415 | "legend": { 1416 | "avg": false, 1417 | "current": false, 1418 | "max": true, 1419 | "min": true, 1420 | "show": true, 1421 | "total": false, 1422 | "values": true 1423 | }, 1424 | "lines": true, 1425 | "linewidth": 1, 1426 | "links": [], 1427 | "nullPointMode": "null", 1428 | "options": { 1429 | "dataLinks": [] 1430 | }, 1431 | "percentage": false, 1432 | "pointradius": 5, 1433 | "points": false, 1434 | "renderer": "flot", 1435 | "seriesOverrides": [], 1436 | "spaceLength": 10, 1437 | "stack": false, 1438 | "steppedLine": false, 1439 | "targets": [ 1440 | { 1441 | "alias": "{{source_node.name}} writes / sec", 1442 | "bucketAggs": [ 1443 | { 1444 | "fake": true, 1445 | "field": "source_node.name", 1446 | "id": "4", 1447 | "settings": { 1448 | "min_doc_count": 1, 1449 | "order": "desc", 1450 | "orderBy": "_term", 1451 | "size": "0" 1452 | }, 1453 | "type": "terms" 1454 | }, 1455 | { 1456 | "field": "timestamp", 1457 | "id": "2", 1458 | "settings": { 1459 | "interval": "auto", 1460 | "min_doc_count": 0, 1461 | "trimEdges": 0 1462 | }, 1463 | "type": "date_histogram" 1464 | } 1465 | ], 1466 | "metrics": [ 1467 | { 1468 | "field": "node_stats.fs.io_stats.total.write_operations", 1469 | "hide": true, 1470 | "id": "1", 1471 | "meta": {}, 1472 | "settings": {}, 1473 | "type": "max" 1474 | }, 1475 | { 1476 | "field": "1", 1477 | "id": "5", 1478 | "meta": {}, 1479 | "pipelineAgg": "1", 1480 | "settings": {}, 1481 | "type": "derivative" 1482 | } 1483 | ], 1484 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1485 | "refId": "A", 1486 | "timeField": "timestamp" 1487 | } 1488 | ], 1489 | "thresholds": [], 1490 | "timeFrom": null, 1491 | "timeRegions": [], 1492 | "timeShift": null, 1493 | "title": "iostat writes / sec", 1494 | "tooltip": { 1495 | "shared": true, 1496 | "sort": 0, 1497 | "value_type": "individual" 1498 | }, 1499 | "type": "graph", 1500 | "xaxis": { 1501 | "buckets": null, 1502 | "mode": "time", 1503 | "name": null, 1504 | "show": true, 1505 | "values": [] 1506 | }, 1507 | "yaxes": [ 1508 | { 1509 | "format": "rps", 1510 | "label": null, 1511 | "logBase": 1, 1512 | "max": null, 1513 | "min": null, 1514 | "show": true 1515 | }, 1516 | { 1517 | "format": "short", 1518 | "label": null, 1519 | "logBase": 1, 1520 | "max": null, 1521 | "min": null, 1522 | "show": true 1523 | } 1524 | ], 1525 | "yaxis": { 1526 | "align": false, 1527 | "alignLevel": null 1528 | } 1529 | }, 1530 | { 1531 | "collapsed": false, 1532 | "datasource": null, 1533 | "gridPos": { 1534 | "h": 1, 1535 | "w": 24, 1536 | "x": 0, 1537 | "y": 41 1538 | }, 1539 | "id": 20, 1540 | "panels": [], 1541 | "title": "JVM", 1542 | "type": "row" 1543 | }, 1544 | { 1545 | "aliasColors": {}, 1546 | "bars": false, 1547 | "dashLength": 10, 1548 | "dashes": false, 1549 | "datasource": "Elasticsearch Monitor", 1550 | "fill": 1, 1551 | "fillGradient": 0, 1552 | "gridPos": { 1553 | "h": 9, 1554 | "w": 12, 1555 | "x": 0, 1556 | "y": 42 1557 | }, 1558 | "hiddenSeries": false, 1559 | "id": 26, 1560 | "legend": { 1561 | "avg": false, 1562 | "current": false, 1563 | "max": false, 1564 | "min": false, 1565 | "show": true, 1566 | "total": false, 1567 | "values": false 1568 | }, 1569 | "lines": true, 1570 | "linewidth": 1, 1571 | "links": [], 1572 | "nullPointMode": "null", 1573 | "options": { 1574 | "dataLinks": [] 1575 | }, 1576 | "percentage": false, 1577 | "pointradius": 5, 1578 | "points": false, 1579 | "renderer": "flot", 1580 | "seriesOverrides": [], 1581 | "spaceLength": 10, 1582 | "stack": false, 1583 | "steppedLine": false, 1584 | "targets": [ 1585 | { 1586 | "alias": "{{source_node.name}} old GC", 1587 | "bucketAggs": [ 1588 | { 1589 | "fake": true, 1590 | "field": "source_node.name", 1591 | "id": "4", 1592 | "settings": { 1593 | "min_doc_count": 1, 1594 | "order": "desc", 1595 | "orderBy": "_term", 1596 | "size": "0" 1597 | }, 1598 | "type": "terms" 1599 | }, 1600 | { 1601 | "field": "timestamp", 1602 | "id": "2", 1603 | "settings": { 1604 | "interval": "auto", 1605 | "min_doc_count": 0, 1606 | "trimEdges": 0 1607 | }, 1608 | "type": "date_histogram" 1609 | } 1610 | ], 1611 | "metrics": [ 1612 | { 1613 | "field": "node_stats.jvm.gc.collectors.old.collection_count", 1614 | "hide": true, 1615 | "id": "1", 1616 | "meta": {}, 1617 | "settings": {}, 1618 | "type": "max" 1619 | }, 1620 | { 1621 | "field": "1", 1622 | "id": "5", 1623 | "meta": {}, 1624 | "pipelineAgg": "1", 1625 | "settings": {}, 1626 | "type": "derivative" 1627 | } 1628 | ], 1629 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1630 | "refId": "A", 1631 | "timeField": "timestamp" 1632 | }, 1633 | { 1634 | "alias": "{{source_node.name}} young GC", 1635 | "bucketAggs": [ 1636 | { 1637 | "fake": true, 1638 | "field": "source_node.name", 1639 | "id": "3", 1640 | "settings": { 1641 | "min_doc_count": 1, 1642 | "order": "desc", 1643 | "orderBy": "_term", 1644 | "size": "0" 1645 | }, 1646 | "type": "terms" 1647 | }, 1648 | { 1649 | "field": "timestamp", 1650 | "id": "2", 1651 | "settings": { 1652 | "interval": "auto", 1653 | "min_doc_count": 0, 1654 | "trimEdges": 0 1655 | }, 1656 | "type": "date_histogram" 1657 | } 1658 | ], 1659 | "hide": false, 1660 | "metrics": [ 1661 | { 1662 | "field": "node_stats.jvm.gc.collectors.young.collection_count", 1663 | "hide": true, 1664 | "id": "1", 1665 | "meta": {}, 1666 | "settings": {}, 1667 | "type": "max" 1668 | }, 1669 | { 1670 | "field": "1", 1671 | "id": "4", 1672 | "meta": {}, 1673 | "pipelineAgg": "1", 1674 | "settings": {}, 1675 | "type": "derivative" 1676 | } 1677 | ], 1678 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1679 | "refId": "B", 1680 | "timeField": "timestamp" 1681 | } 1682 | ], 1683 | "thresholds": [], 1684 | "timeFrom": null, 1685 | "timeRegions": [], 1686 | "timeShift": null, 1687 | "title": "GC count", 1688 | "tooltip": { 1689 | "shared": true, 1690 | "sort": 0, 1691 | "value_type": "individual" 1692 | }, 1693 | "type": "graph", 1694 | "xaxis": { 1695 | "buckets": null, 1696 | "mode": "time", 1697 | "name": null, 1698 | "show": true, 1699 | "values": [] 1700 | }, 1701 | "yaxes": [ 1702 | { 1703 | "format": "short", 1704 | "label": null, 1705 | "logBase": 1, 1706 | "max": null, 1707 | "min": "0", 1708 | "show": true 1709 | }, 1710 | { 1711 | "format": "short", 1712 | "label": null, 1713 | "logBase": 1, 1714 | "max": null, 1715 | "min": "0", 1716 | "show": true 1717 | } 1718 | ], 1719 | "yaxis": { 1720 | "align": false, 1721 | "alignLevel": null 1722 | } 1723 | }, 1724 | { 1725 | "aliasColors": {}, 1726 | "bars": false, 1727 | "dashLength": 10, 1728 | "dashes": false, 1729 | "datasource": "Elasticsearch Monitor", 1730 | "fill": 1, 1731 | "fillGradient": 0, 1732 | "gridPos": { 1733 | "h": 9, 1734 | "w": 12, 1735 | "x": 12, 1736 | "y": 42 1737 | }, 1738 | "hiddenSeries": false, 1739 | "id": 27, 1740 | "legend": { 1741 | "avg": false, 1742 | "current": false, 1743 | "max": false, 1744 | "min": false, 1745 | "show": true, 1746 | "total": false, 1747 | "values": false 1748 | }, 1749 | "lines": true, 1750 | "linewidth": 1, 1751 | "links": [], 1752 | "nullPointMode": "null", 1753 | "options": { 1754 | "dataLinks": [] 1755 | }, 1756 | "percentage": false, 1757 | "pointradius": 5, 1758 | "points": false, 1759 | "renderer": "flot", 1760 | "seriesOverrides": [], 1761 | "spaceLength": 10, 1762 | "stack": false, 1763 | "steppedLine": false, 1764 | "targets": [ 1765 | { 1766 | "alias": "{{source_node.name}} old GC", 1767 | "bucketAggs": [ 1768 | { 1769 | "fake": true, 1770 | "field": "source_node.name", 1771 | "id": "4", 1772 | "settings": { 1773 | "min_doc_count": 1, 1774 | "order": "desc", 1775 | "orderBy": "_term", 1776 | "size": "0" 1777 | }, 1778 | "type": "terms" 1779 | }, 1780 | { 1781 | "field": "timestamp", 1782 | "id": "2", 1783 | "settings": { 1784 | "interval": "auto", 1785 | "min_doc_count": 0, 1786 | "trimEdges": 0 1787 | }, 1788 | "type": "date_histogram" 1789 | } 1790 | ], 1791 | "metrics": [ 1792 | { 1793 | "field": "node_stats.jvm.gc.collectors.old.collection_time_in_millis", 1794 | "hide": true, 1795 | "id": "1", 1796 | "meta": {}, 1797 | "settings": {}, 1798 | "type": "max" 1799 | }, 1800 | { 1801 | "field": "1", 1802 | "id": "5", 1803 | "meta": {}, 1804 | "pipelineAgg": "1", 1805 | "settings": {}, 1806 | "type": "derivative" 1807 | } 1808 | ], 1809 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1810 | "refId": "A", 1811 | "timeField": "timestamp" 1812 | }, 1813 | { 1814 | "alias": "{{source_node.name}} young GC", 1815 | "bucketAggs": [ 1816 | { 1817 | "fake": true, 1818 | "field": "source_node.name", 1819 | "id": "3", 1820 | "settings": { 1821 | "min_doc_count": 1, 1822 | "order": "desc", 1823 | "orderBy": "_term", 1824 | "size": "0" 1825 | }, 1826 | "type": "terms" 1827 | }, 1828 | { 1829 | "field": "timestamp", 1830 | "id": "2", 1831 | "settings": { 1832 | "interval": "auto", 1833 | "min_doc_count": 0, 1834 | "trimEdges": 0 1835 | }, 1836 | "type": "date_histogram" 1837 | } 1838 | ], 1839 | "hide": false, 1840 | "metrics": [ 1841 | { 1842 | "field": "node_stats.jvm.gc.collectors.young.collection_time_in_millis", 1843 | "hide": true, 1844 | "id": "1", 1845 | "meta": {}, 1846 | "settings": {}, 1847 | "type": "max" 1848 | }, 1849 | { 1850 | "field": "1", 1851 | "id": "4", 1852 | "meta": {}, 1853 | "pipelineAgg": "1", 1854 | "settings": {}, 1855 | "type": "derivative" 1856 | } 1857 | ], 1858 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 1859 | "refId": "B", 1860 | "timeField": "timestamp" 1861 | } 1862 | ], 1863 | "thresholds": [], 1864 | "timeFrom": null, 1865 | "timeRegions": [], 1866 | "timeShift": null, 1867 | "title": "GC time", 1868 | "tooltip": { 1869 | "shared": true, 1870 | "sort": 0, 1871 | "value_type": "individual" 1872 | }, 1873 | "type": "graph", 1874 | "xaxis": { 1875 | "buckets": null, 1876 | "mode": "time", 1877 | "name": null, 1878 | "show": true, 1879 | "values": [] 1880 | }, 1881 | "yaxes": [ 1882 | { 1883 | "format": "ms", 1884 | "label": null, 1885 | "logBase": 1, 1886 | "max": null, 1887 | "min": "0", 1888 | "show": true 1889 | }, 1890 | { 1891 | "format": "short", 1892 | "label": null, 1893 | "logBase": 1, 1894 | "max": null, 1895 | "min": "0", 1896 | "show": false 1897 | } 1898 | ], 1899 | "yaxis": { 1900 | "align": false, 1901 | "alignLevel": null 1902 | } 1903 | }, 1904 | { 1905 | "aliasColors": {}, 1906 | "bars": false, 1907 | "dashLength": 10, 1908 | "dashes": false, 1909 | "datasource": "Elasticsearch Monitor", 1910 | "fill": 1, 1911 | "fillGradient": 0, 1912 | "gridPos": { 1913 | "h": 9, 1914 | "w": 12, 1915 | "x": 0, 1916 | "y": 51 1917 | }, 1918 | "hiddenSeries": false, 1919 | "id": 29, 1920 | "legend": { 1921 | "avg": false, 1922 | "current": false, 1923 | "max": false, 1924 | "min": false, 1925 | "show": true, 1926 | "total": false, 1927 | "values": false 1928 | }, 1929 | "lines": true, 1930 | "linewidth": 1, 1931 | "links": [], 1932 | "nullPointMode": "connected", 1933 | "options": { 1934 | "dataLinks": [] 1935 | }, 1936 | "percentage": false, 1937 | "pointradius": 5, 1938 | "points": false, 1939 | "renderer": "flot", 1940 | "seriesOverrides": [], 1941 | "spaceLength": 10, 1942 | "stack": false, 1943 | "steppedLine": false, 1944 | "targets": [ 1945 | { 1946 | "alias": "{{source_node.name}}", 1947 | "bucketAggs": [ 1948 | { 1949 | "fake": true, 1950 | "field": "source_node.name", 1951 | "id": "4", 1952 | "settings": { 1953 | "min_doc_count": 1, 1954 | "order": "desc", 1955 | "orderBy": "_term", 1956 | "size": "0" 1957 | }, 1958 | "type": "terms" 1959 | }, 1960 | { 1961 | "field": "timestamp", 1962 | "id": "2", 1963 | "settings": { 1964 | "interval": "auto", 1965 | "min_doc_count": 0, 1966 | "trimEdges": 0 1967 | }, 1968 | "type": "date_histogram" 1969 | } 1970 | ], 1971 | "metrics": [ 1972 | { 1973 | "field": "node_stats.jvm.mem.heap_used_in_bytes", 1974 | "id": "1", 1975 | "meta": {}, 1976 | "settings": {}, 1977 | "type": "max" 1978 | } 1979 | ], 1980 | "query": "type:node_stats AND source_node.name:($host) AND cluster_uuid:$cluster", 1981 | "refId": "A", 1982 | "timeField": "timestamp" 1983 | } 1984 | ], 1985 | "thresholds": [], 1986 | "timeFrom": null, 1987 | "timeRegions": [], 1988 | "timeShift": null, 1989 | "title": "Heap Usage", 1990 | "tooltip": { 1991 | "shared": true, 1992 | "sort": 0, 1993 | "value_type": "individual" 1994 | }, 1995 | "type": "graph", 1996 | "xaxis": { 1997 | "buckets": null, 1998 | "mode": "time", 1999 | "name": null, 2000 | "show": true, 2001 | "values": [] 2002 | }, 2003 | "yaxes": [ 2004 | { 2005 | "format": "decbytes", 2006 | "label": null, 2007 | "logBase": 1, 2008 | "max": null, 2009 | "min": "0", 2010 | "show": true 2011 | }, 2012 | { 2013 | "format": "short", 2014 | "label": null, 2015 | "logBase": 1, 2016 | "max": null, 2017 | "min": "0", 2018 | "show": false 2019 | } 2020 | ], 2021 | "yaxis": { 2022 | "align": false, 2023 | "alignLevel": null 2024 | } 2025 | }, 2026 | { 2027 | "collapsed": false, 2028 | "datasource": null, 2029 | "gridPos": { 2030 | "h": 1, 2031 | "w": 24, 2032 | "x": 0, 2033 | "y": 60 2034 | }, 2035 | "id": 22, 2036 | "panels": [], 2037 | "title": "Thread pools", 2038 | "type": "row" 2039 | }, 2040 | { 2041 | "aliasColors": {}, 2042 | "bars": false, 2043 | "dashLength": 10, 2044 | "dashes": false, 2045 | "datasource": "Elasticsearch Monitor", 2046 | "fill": 1, 2047 | "fillGradient": 0, 2048 | "gridPos": { 2049 | "h": 9, 2050 | "w": 12, 2051 | "x": 0, 2052 | "y": 61 2053 | }, 2054 | "hiddenSeries": false, 2055 | "id": 24, 2056 | "legend": { 2057 | "avg": false, 2058 | "current": false, 2059 | "max": false, 2060 | "min": false, 2061 | "show": true, 2062 | "total": false, 2063 | "values": false 2064 | }, 2065 | "lines": true, 2066 | "linewidth": 1, 2067 | "links": [], 2068 | "nullPointMode": "null", 2069 | "options": { 2070 | "dataLinks": [] 2071 | }, 2072 | "percentage": false, 2073 | "pointradius": 5, 2074 | "points": false, 2075 | "renderer": "flot", 2076 | "seriesOverrides": [ 2077 | { 2078 | "alias": "/rejections/", 2079 | "yaxis": 2 2080 | } 2081 | ], 2082 | "spaceLength": 10, 2083 | "stack": false, 2084 | "steppedLine": false, 2085 | "targets": [ 2086 | { 2087 | "alias": "{{source_node.name}} queue", 2088 | "bucketAggs": [ 2089 | { 2090 | "fake": true, 2091 | "field": "source_node.name", 2092 | "id": "6", 2093 | "settings": { 2094 | "min_doc_count": 1, 2095 | "order": "desc", 2096 | "orderBy": "_term", 2097 | "size": "0" 2098 | }, 2099 | "type": "terms" 2100 | }, 2101 | { 2102 | "field": "timestamp", 2103 | "id": "2", 2104 | "settings": { 2105 | "interval": "auto", 2106 | "min_doc_count": 0, 2107 | "trimEdges": 0 2108 | }, 2109 | "type": "date_histogram" 2110 | } 2111 | ], 2112 | "metrics": [ 2113 | { 2114 | "field": "node_stats.thread_pool.index.queue", 2115 | "id": "5", 2116 | "meta": {}, 2117 | "settings": {}, 2118 | "type": "max" 2119 | } 2120 | ], 2121 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2122 | "refId": "A", 2123 | "timeField": "timestamp" 2124 | }, 2125 | { 2126 | "alias": "{{source_node.name}} rejections", 2127 | "bucketAggs": [ 2128 | { 2129 | "fake": true, 2130 | "field": "source_node.name", 2131 | "id": "6", 2132 | "settings": { 2133 | "min_doc_count": 1, 2134 | "order": "desc", 2135 | "orderBy": "_term", 2136 | "size": "0" 2137 | }, 2138 | "type": "terms" 2139 | }, 2140 | { 2141 | "field": "timestamp", 2142 | "id": "2", 2143 | "settings": { 2144 | "interval": "auto", 2145 | "min_doc_count": 0, 2146 | "trimEdges": 0 2147 | }, 2148 | "type": "date_histogram" 2149 | } 2150 | ], 2151 | "metrics": [ 2152 | { 2153 | "field": "node_stats.thread_pool.index.rejected", 2154 | "id": "1", 2155 | "meta": {}, 2156 | "settings": {}, 2157 | "type": "max" 2158 | } 2159 | ], 2160 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2161 | "refId": "B", 2162 | "timeField": "timestamp" 2163 | } 2164 | ], 2165 | "thresholds": [], 2166 | "timeFrom": null, 2167 | "timeRegions": [], 2168 | "timeShift": null, 2169 | "title": "Thread pool: Index", 2170 | "tooltip": { 2171 | "shared": true, 2172 | "sort": 0, 2173 | "value_type": "individual" 2174 | }, 2175 | "type": "graph", 2176 | "xaxis": { 2177 | "buckets": null, 2178 | "mode": "time", 2179 | "name": null, 2180 | "show": true, 2181 | "values": [] 2182 | }, 2183 | "yaxes": [ 2184 | { 2185 | "format": "short", 2186 | "label": "Queue", 2187 | "logBase": 1, 2188 | "max": null, 2189 | "min": "0", 2190 | "show": true 2191 | }, 2192 | { 2193 | "format": "short", 2194 | "label": "Rejections", 2195 | "logBase": 1, 2196 | "max": null, 2197 | "min": "0", 2198 | "show": true 2199 | } 2200 | ], 2201 | "yaxis": { 2202 | "align": false, 2203 | "alignLevel": null 2204 | } 2205 | }, 2206 | { 2207 | "aliasColors": {}, 2208 | "bars": false, 2209 | "dashLength": 10, 2210 | "dashes": false, 2211 | "datasource": "Elasticsearch Monitor", 2212 | "fill": 1, 2213 | "fillGradient": 0, 2214 | "gridPos": { 2215 | "h": 9, 2216 | "w": 12, 2217 | "x": 12, 2218 | "y": 61 2219 | }, 2220 | "hiddenSeries": false, 2221 | "id": 25, 2222 | "legend": { 2223 | "avg": false, 2224 | "current": false, 2225 | "max": false, 2226 | "min": false, 2227 | "show": true, 2228 | "total": false, 2229 | "values": false 2230 | }, 2231 | "lines": true, 2232 | "linewidth": 1, 2233 | "links": [], 2234 | "nullPointMode": "connected", 2235 | "options": { 2236 | "dataLinks": [] 2237 | }, 2238 | "percentage": false, 2239 | "pointradius": 5, 2240 | "points": false, 2241 | "renderer": "flot", 2242 | "seriesOverrides": [ 2243 | { 2244 | "alias": "/rejections/", 2245 | "yaxis": 2 2246 | } 2247 | ], 2248 | "spaceLength": 10, 2249 | "stack": false, 2250 | "steppedLine": false, 2251 | "targets": [ 2252 | { 2253 | "alias": "{{source_node.name}} queue", 2254 | "bucketAggs": [ 2255 | { 2256 | "fake": true, 2257 | "field": "source_node.name", 2258 | "id": "6", 2259 | "settings": { 2260 | "min_doc_count": 1, 2261 | "order": "desc", 2262 | "orderBy": "_term", 2263 | "size": "0" 2264 | }, 2265 | "type": "terms" 2266 | }, 2267 | { 2268 | "field": "timestamp", 2269 | "id": "2", 2270 | "settings": { 2271 | "interval": "auto", 2272 | "min_doc_count": 0, 2273 | "trimEdges": 0 2274 | }, 2275 | "type": "date_histogram" 2276 | } 2277 | ], 2278 | "metrics": [ 2279 | { 2280 | "field": "node_stats.thread_pool.search.queue", 2281 | "id": "5", 2282 | "meta": {}, 2283 | "settings": {}, 2284 | "type": "max" 2285 | } 2286 | ], 2287 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2288 | "refId": "A", 2289 | "timeField": "timestamp" 2290 | }, 2291 | { 2292 | "alias": "{{source_node.name}} rejections", 2293 | "bucketAggs": [ 2294 | { 2295 | "fake": true, 2296 | "field": "source_node.name", 2297 | "id": "6", 2298 | "settings": { 2299 | "min_doc_count": 1, 2300 | "order": "desc", 2301 | "orderBy": "_term", 2302 | "size": "0" 2303 | }, 2304 | "type": "terms" 2305 | }, 2306 | { 2307 | "field": "timestamp", 2308 | "id": "2", 2309 | "settings": { 2310 | "interval": "auto", 2311 | "min_doc_count": 0, 2312 | "trimEdges": 0 2313 | }, 2314 | "type": "date_histogram" 2315 | } 2316 | ], 2317 | "metrics": [ 2318 | { 2319 | "field": "node_stats.thread_pool.search.rejected", 2320 | "hide": true, 2321 | "id": "1", 2322 | "meta": {}, 2323 | "settings": {}, 2324 | "type": "max" 2325 | }, 2326 | { 2327 | "field": "1", 2328 | "id": "7", 2329 | "meta": {}, 2330 | "pipelineAgg": "1", 2331 | "settings": {}, 2332 | "type": "derivative" 2333 | } 2334 | ], 2335 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2336 | "refId": "B", 2337 | "timeField": "timestamp" 2338 | } 2339 | ], 2340 | "thresholds": [], 2341 | "timeFrom": null, 2342 | "timeRegions": [], 2343 | "timeShift": null, 2344 | "title": "Thread pool: Search", 2345 | "tooltip": { 2346 | "shared": true, 2347 | "sort": 0, 2348 | "value_type": "individual" 2349 | }, 2350 | "type": "graph", 2351 | "xaxis": { 2352 | "buckets": null, 2353 | "mode": "time", 2354 | "name": null, 2355 | "show": true, 2356 | "values": [] 2357 | }, 2358 | "yaxes": [ 2359 | { 2360 | "format": "short", 2361 | "label": "Queue", 2362 | "logBase": 1, 2363 | "max": null, 2364 | "min": "0", 2365 | "show": true 2366 | }, 2367 | { 2368 | "format": "short", 2369 | "label": "Rejections", 2370 | "logBase": 1, 2371 | "max": null, 2372 | "min": "0", 2373 | "show": true 2374 | } 2375 | ], 2376 | "yaxis": { 2377 | "align": false, 2378 | "alignLevel": null 2379 | } 2380 | }, 2381 | { 2382 | "aliasColors": {}, 2383 | "bars": false, 2384 | "dashLength": 10, 2385 | "dashes": false, 2386 | "datasource": "Elasticsearch Monitor", 2387 | "fill": 1, 2388 | "fillGradient": 0, 2389 | "gridPos": { 2390 | "h": 9, 2391 | "w": 12, 2392 | "x": 0, 2393 | "y": 70 2394 | }, 2395 | "hiddenSeries": false, 2396 | "id": 41, 2397 | "legend": { 2398 | "avg": false, 2399 | "current": false, 2400 | "max": false, 2401 | "min": false, 2402 | "show": true, 2403 | "total": false, 2404 | "values": false 2405 | }, 2406 | "lines": true, 2407 | "linewidth": 1, 2408 | "links": [], 2409 | "nullPointMode": "null", 2410 | "options": { 2411 | "dataLinks": [] 2412 | }, 2413 | "percentage": false, 2414 | "pointradius": 5, 2415 | "points": false, 2416 | "renderer": "flot", 2417 | "seriesOverrides": [ 2418 | { 2419 | "alias": "/rejections/", 2420 | "yaxis": 2 2421 | } 2422 | ], 2423 | "spaceLength": 10, 2424 | "stack": false, 2425 | "steppedLine": false, 2426 | "targets": [ 2427 | { 2428 | "alias": "{{source_node.name}} queue", 2429 | "bucketAggs": [ 2430 | { 2431 | "fake": true, 2432 | "field": "source_node.name", 2433 | "id": "6", 2434 | "settings": { 2435 | "min_doc_count": 1, 2436 | "order": "desc", 2437 | "orderBy": "_term", 2438 | "size": "0" 2439 | }, 2440 | "type": "terms" 2441 | }, 2442 | { 2443 | "field": "timestamp", 2444 | "id": "2", 2445 | "settings": { 2446 | "interval": "auto", 2447 | "min_doc_count": 0, 2448 | "trimEdges": 0 2449 | }, 2450 | "type": "date_histogram" 2451 | } 2452 | ], 2453 | "metrics": [ 2454 | { 2455 | "field": "node_stats.thread_pool.write.queue", 2456 | "id": "5", 2457 | "meta": {}, 2458 | "settings": {}, 2459 | "type": "max" 2460 | } 2461 | ], 2462 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2463 | "refId": "A", 2464 | "timeField": "timestamp" 2465 | }, 2466 | { 2467 | "alias": "{{source_node.name}} rejections", 2468 | "bucketAggs": [ 2469 | { 2470 | "fake": true, 2471 | "field": "source_node.name", 2472 | "id": "6", 2473 | "settings": { 2474 | "min_doc_count": 1, 2475 | "order": "desc", 2476 | "orderBy": "_term", 2477 | "size": "0" 2478 | }, 2479 | "type": "terms" 2480 | }, 2481 | { 2482 | "field": "timestamp", 2483 | "id": "2", 2484 | "settings": { 2485 | "interval": "auto", 2486 | "min_doc_count": 0, 2487 | "trimEdges": 0 2488 | }, 2489 | "type": "date_histogram" 2490 | } 2491 | ], 2492 | "metrics": [ 2493 | { 2494 | "field": "node_stats.thread_pool.write.rejected", 2495 | "hide": true, 2496 | "id": "1", 2497 | "meta": {}, 2498 | "settings": {}, 2499 | "type": "max" 2500 | }, 2501 | { 2502 | "field": "1", 2503 | "id": "7", 2504 | "meta": {}, 2505 | "pipelineAgg": "1", 2506 | "settings": {}, 2507 | "type": "derivative" 2508 | } 2509 | ], 2510 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2511 | "refId": "B", 2512 | "timeField": "timestamp" 2513 | } 2514 | ], 2515 | "thresholds": [], 2516 | "timeFrom": null, 2517 | "timeRegions": [], 2518 | "timeShift": null, 2519 | "title": "Thread pool: Write", 2520 | "tooltip": { 2521 | "shared": true, 2522 | "sort": 0, 2523 | "value_type": "individual" 2524 | }, 2525 | "type": "graph", 2526 | "xaxis": { 2527 | "buckets": null, 2528 | "mode": "time", 2529 | "name": null, 2530 | "show": true, 2531 | "values": [] 2532 | }, 2533 | "yaxes": [ 2534 | { 2535 | "format": "short", 2536 | "label": "Queue", 2537 | "logBase": 1, 2538 | "max": null, 2539 | "min": "0", 2540 | "show": true 2541 | }, 2542 | { 2543 | "format": "short", 2544 | "label": "Rejections", 2545 | "logBase": 1, 2546 | "max": null, 2547 | "min": "0", 2548 | "show": true 2549 | } 2550 | ], 2551 | "yaxis": { 2552 | "align": false, 2553 | "alignLevel": null 2554 | } 2555 | }, 2556 | { 2557 | "aliasColors": {}, 2558 | "bars": false, 2559 | "dashLength": 10, 2560 | "dashes": false, 2561 | "datasource": "Elasticsearch Monitor", 2562 | "fill": 1, 2563 | "fillGradient": 0, 2564 | "gridPos": { 2565 | "h": 9, 2566 | "w": 12, 2567 | "x": 12, 2568 | "y": 70 2569 | }, 2570 | "hiddenSeries": false, 2571 | "id": 44, 2572 | "legend": { 2573 | "avg": false, 2574 | "current": false, 2575 | "max": false, 2576 | "min": false, 2577 | "show": true, 2578 | "total": false, 2579 | "values": false 2580 | }, 2581 | "lines": true, 2582 | "linewidth": 1, 2583 | "links": [], 2584 | "nullPointMode": "connected", 2585 | "options": { 2586 | "dataLinks": [] 2587 | }, 2588 | "percentage": false, 2589 | "pointradius": 5, 2590 | "points": false, 2591 | "renderer": "flot", 2592 | "seriesOverrides": [ 2593 | { 2594 | "alias": "/rejections/", 2595 | "yaxis": 2 2596 | } 2597 | ], 2598 | "spaceLength": 10, 2599 | "stack": false, 2600 | "steppedLine": false, 2601 | "targets": [ 2602 | { 2603 | "alias": "{{source_node.name}} queue", 2604 | "bucketAggs": [ 2605 | { 2606 | "fake": true, 2607 | "field": "source_node.name", 2608 | "id": "6", 2609 | "settings": { 2610 | "min_doc_count": 1, 2611 | "order": "desc", 2612 | "orderBy": "_term", 2613 | "size": "0" 2614 | }, 2615 | "type": "terms" 2616 | }, 2617 | { 2618 | "field": "timestamp", 2619 | "id": "2", 2620 | "settings": { 2621 | "interval": "auto", 2622 | "min_doc_count": 0, 2623 | "trimEdges": 0 2624 | }, 2625 | "type": "date_histogram" 2626 | } 2627 | ], 2628 | "metrics": [ 2629 | { 2630 | "field": "node_stats.thread_pool.get.queue", 2631 | "id": "5", 2632 | "meta": {}, 2633 | "settings": {}, 2634 | "type": "max" 2635 | } 2636 | ], 2637 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2638 | "refId": "A", 2639 | "timeField": "timestamp" 2640 | }, 2641 | { 2642 | "alias": "{{source_node.name}} rejections", 2643 | "bucketAggs": [ 2644 | { 2645 | "fake": true, 2646 | "field": "source_node.name", 2647 | "id": "6", 2648 | "settings": { 2649 | "min_doc_count": 1, 2650 | "order": "desc", 2651 | "orderBy": "_term", 2652 | "size": "0" 2653 | }, 2654 | "type": "terms" 2655 | }, 2656 | { 2657 | "field": "timestamp", 2658 | "id": "2", 2659 | "settings": { 2660 | "interval": "auto", 2661 | "min_doc_count": 0, 2662 | "trimEdges": 0 2663 | }, 2664 | "type": "date_histogram" 2665 | } 2666 | ], 2667 | "metrics": [ 2668 | { 2669 | "field": "node_stats.thread_pool.get.rejected", 2670 | "hide": true, 2671 | "id": "1", 2672 | "meta": {}, 2673 | "settings": {}, 2674 | "type": "max" 2675 | }, 2676 | { 2677 | "field": "1", 2678 | "id": "7", 2679 | "meta": {}, 2680 | "pipelineAgg": "1", 2681 | "settings": {}, 2682 | "type": "derivative" 2683 | } 2684 | ], 2685 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2686 | "refId": "B", 2687 | "timeField": "timestamp" 2688 | } 2689 | ], 2690 | "thresholds": [], 2691 | "timeFrom": null, 2692 | "timeRegions": [], 2693 | "timeShift": null, 2694 | "title": "Thread pool: Get", 2695 | "tooltip": { 2696 | "shared": true, 2697 | "sort": 0, 2698 | "value_type": "individual" 2699 | }, 2700 | "type": "graph", 2701 | "xaxis": { 2702 | "buckets": null, 2703 | "mode": "time", 2704 | "name": null, 2705 | "show": true, 2706 | "values": [] 2707 | }, 2708 | "yaxes": [ 2709 | { 2710 | "format": "short", 2711 | "label": "Queue", 2712 | "logBase": 1, 2713 | "max": null, 2714 | "min": "0", 2715 | "show": true 2716 | }, 2717 | { 2718 | "format": "short", 2719 | "label": "Rejections", 2720 | "logBase": 1, 2721 | "max": null, 2722 | "min": "0", 2723 | "show": true 2724 | } 2725 | ], 2726 | "yaxis": { 2727 | "align": false, 2728 | "alignLevel": null 2729 | } 2730 | }, 2731 | { 2732 | "collapsed": false, 2733 | "datasource": null, 2734 | "gridPos": { 2735 | "h": 1, 2736 | "w": 24, 2737 | "x": 0, 2738 | "y": 79 2739 | }, 2740 | "id": 9, 2741 | "panels": [], 2742 | "title": "Caches", 2743 | "type": "row" 2744 | }, 2745 | { 2746 | "aliasColors": {}, 2747 | "bars": false, 2748 | "dashLength": 10, 2749 | "dashes": false, 2750 | "datasource": "Elasticsearch Monitor", 2751 | "fill": 1, 2752 | "fillGradient": 0, 2753 | "gridPos": { 2754 | "h": 9, 2755 | "w": 12, 2756 | "x": 0, 2757 | "y": 80 2758 | }, 2759 | "hiddenSeries": false, 2760 | "id": 5, 2761 | "legend": { 2762 | "avg": false, 2763 | "current": false, 2764 | "max": false, 2765 | "min": false, 2766 | "show": true, 2767 | "total": false, 2768 | "values": false 2769 | }, 2770 | "lines": true, 2771 | "linewidth": 1, 2772 | "links": [], 2773 | "nullPointMode": "null", 2774 | "options": { 2775 | "dataLinks": [] 2776 | }, 2777 | "percentage": false, 2778 | "pointradius": 5, 2779 | "points": false, 2780 | "renderer": "flot", 2781 | "seriesOverrides": [], 2782 | "spaceLength": 10, 2783 | "stack": false, 2784 | "steppedLine": false, 2785 | "targets": [ 2786 | { 2787 | "alias": "", 2788 | "bucketAggs": [ 2789 | { 2790 | "fake": true, 2791 | "field": "source_node.name", 2792 | "id": "4", 2793 | "settings": { 2794 | "min_doc_count": 1, 2795 | "order": "desc", 2796 | "orderBy": "_term", 2797 | "size": "0" 2798 | }, 2799 | "type": "terms" 2800 | }, 2801 | { 2802 | "field": "timestamp", 2803 | "id": "2", 2804 | "settings": { 2805 | "interval": "auto", 2806 | "min_doc_count": 0, 2807 | "trimEdges": 0 2808 | }, 2809 | "type": "date_histogram" 2810 | } 2811 | ], 2812 | "metrics": [ 2813 | { 2814 | "field": "node_stats.indices.fielddata.evictions", 2815 | "hide": true, 2816 | "id": "1", 2817 | "meta": {}, 2818 | "settings": {}, 2819 | "type": "max" 2820 | }, 2821 | { 2822 | "field": "1", 2823 | "id": "3", 2824 | "meta": {}, 2825 | "pipelineAgg": "1", 2826 | "settings": {}, 2827 | "type": "derivative" 2828 | } 2829 | ], 2830 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2831 | "refId": "A", 2832 | "timeField": "timestamp" 2833 | } 2834 | ], 2835 | "thresholds": [], 2836 | "timeFrom": null, 2837 | "timeRegions": [], 2838 | "timeShift": null, 2839 | "title": "Fielddata evictions", 2840 | "tooltip": { 2841 | "shared": true, 2842 | "sort": 0, 2843 | "value_type": "individual" 2844 | }, 2845 | "type": "graph", 2846 | "xaxis": { 2847 | "buckets": null, 2848 | "mode": "time", 2849 | "name": null, 2850 | "show": true, 2851 | "values": [] 2852 | }, 2853 | "yaxes": [ 2854 | { 2855 | "format": "short", 2856 | "label": null, 2857 | "logBase": 1, 2858 | "max": null, 2859 | "min": "0", 2860 | "show": true 2861 | }, 2862 | { 2863 | "format": "short", 2864 | "label": null, 2865 | "logBase": 1, 2866 | "max": null, 2867 | "min": null, 2868 | "show": true 2869 | } 2870 | ], 2871 | "yaxis": { 2872 | "align": false, 2873 | "alignLevel": null 2874 | } 2875 | }, 2876 | { 2877 | "aliasColors": {}, 2878 | "bars": false, 2879 | "dashLength": 10, 2880 | "dashes": false, 2881 | "datasource": "Elasticsearch Monitor", 2882 | "fill": 1, 2883 | "fillGradient": 0, 2884 | "gridPos": { 2885 | "h": 9, 2886 | "w": 12, 2887 | "x": 12, 2888 | "y": 80 2889 | }, 2890 | "hiddenSeries": false, 2891 | "id": 12, 2892 | "legend": { 2893 | "avg": false, 2894 | "current": false, 2895 | "max": false, 2896 | "min": false, 2897 | "show": true, 2898 | "total": false, 2899 | "values": false 2900 | }, 2901 | "lines": true, 2902 | "linewidth": 1, 2903 | "links": [], 2904 | "nullPointMode": "connected", 2905 | "options": { 2906 | "dataLinks": [] 2907 | }, 2908 | "percentage": false, 2909 | "pointradius": 5, 2910 | "points": false, 2911 | "renderer": "flot", 2912 | "seriesOverrides": [], 2913 | "spaceLength": 10, 2914 | "stack": false, 2915 | "steppedLine": false, 2916 | "targets": [ 2917 | { 2918 | "alias": "", 2919 | "bucketAggs": [ 2920 | { 2921 | "fake": true, 2922 | "field": "source_node.name", 2923 | "id": "4", 2924 | "settings": { 2925 | "min_doc_count": 1, 2926 | "order": "desc", 2927 | "orderBy": "_term", 2928 | "size": "0" 2929 | }, 2930 | "type": "terms" 2931 | }, 2932 | { 2933 | "field": "timestamp", 2934 | "id": "2", 2935 | "settings": { 2936 | "interval": "auto", 2937 | "min_doc_count": 0, 2938 | "trimEdges": 0 2939 | }, 2940 | "type": "date_histogram" 2941 | } 2942 | ], 2943 | "metrics": [ 2944 | { 2945 | "field": "node_stats.indices.fielddata.memory_size_in_bytes", 2946 | "id": "1", 2947 | "meta": {}, 2948 | "settings": {}, 2949 | "type": "max" 2950 | } 2951 | ], 2952 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 2953 | "refId": "A", 2954 | "timeField": "timestamp" 2955 | } 2956 | ], 2957 | "thresholds": [], 2958 | "timeFrom": null, 2959 | "timeRegions": [], 2960 | "timeShift": null, 2961 | "title": "Fielddata memory size", 2962 | "tooltip": { 2963 | "shared": true, 2964 | "sort": 0, 2965 | "value_type": "individual" 2966 | }, 2967 | "type": "graph", 2968 | "xaxis": { 2969 | "buckets": null, 2970 | "mode": "time", 2971 | "name": null, 2972 | "show": true, 2973 | "values": [] 2974 | }, 2975 | "yaxes": [ 2976 | { 2977 | "format": "decbytes", 2978 | "label": null, 2979 | "logBase": 1, 2980 | "max": null, 2981 | "min": "0", 2982 | "show": true 2983 | }, 2984 | { 2985 | "format": "short", 2986 | "label": null, 2987 | "logBase": 1, 2988 | "max": null, 2989 | "min": null, 2990 | "show": true 2991 | } 2992 | ], 2993 | "yaxis": { 2994 | "align": false, 2995 | "alignLevel": null 2996 | } 2997 | }, 2998 | { 2999 | "aliasColors": {}, 3000 | "bars": false, 3001 | "dashLength": 10, 3002 | "dashes": false, 3003 | "datasource": "Elasticsearch Monitor", 3004 | "fill": 1, 3005 | "fillGradient": 0, 3006 | "gridPos": { 3007 | "h": 9, 3008 | "w": 12, 3009 | "x": 0, 3010 | "y": 89 3011 | }, 3012 | "hiddenSeries": false, 3013 | "id": 11, 3014 | "legend": { 3015 | "avg": false, 3016 | "current": false, 3017 | "max": false, 3018 | "min": false, 3019 | "show": true, 3020 | "total": false, 3021 | "values": false 3022 | }, 3023 | "lines": true, 3024 | "linewidth": 1, 3025 | "links": [], 3026 | "nullPointMode": "null", 3027 | "options": { 3028 | "dataLinks": [] 3029 | }, 3030 | "percentage": false, 3031 | "pointradius": 5, 3032 | "points": false, 3033 | "renderer": "flot", 3034 | "seriesOverrides": [], 3035 | "spaceLength": 10, 3036 | "stack": false, 3037 | "steppedLine": false, 3038 | "targets": [ 3039 | { 3040 | "alias": "", 3041 | "bucketAggs": [ 3042 | { 3043 | "fake": true, 3044 | "field": "source_node.name", 3045 | "id": "4", 3046 | "settings": { 3047 | "min_doc_count": 1, 3048 | "order": "desc", 3049 | "orderBy": "_term", 3050 | "size": "0" 3051 | }, 3052 | "type": "terms" 3053 | }, 3054 | { 3055 | "field": "timestamp", 3056 | "id": "2", 3057 | "settings": { 3058 | "interval": "auto", 3059 | "min_doc_count": 0, 3060 | "trimEdges": 0 3061 | }, 3062 | "type": "date_histogram" 3063 | } 3064 | ], 3065 | "metrics": [ 3066 | { 3067 | "field": "node_stats.indices.request_cache.evictions", 3068 | "hide": true, 3069 | "id": "1", 3070 | "meta": {}, 3071 | "settings": {}, 3072 | "type": "max" 3073 | }, 3074 | { 3075 | "field": "1", 3076 | "id": "3", 3077 | "meta": {}, 3078 | "pipelineAgg": "1", 3079 | "settings": {}, 3080 | "type": "derivative" 3081 | } 3082 | ], 3083 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 3084 | "refId": "A", 3085 | "timeField": "timestamp" 3086 | } 3087 | ], 3088 | "thresholds": [], 3089 | "timeFrom": null, 3090 | "timeRegions": [], 3091 | "timeShift": null, 3092 | "title": "Request cache evictions", 3093 | "tooltip": { 3094 | "shared": true, 3095 | "sort": 0, 3096 | "value_type": "individual" 3097 | }, 3098 | "type": "graph", 3099 | "xaxis": { 3100 | "buckets": null, 3101 | "mode": "time", 3102 | "name": null, 3103 | "show": true, 3104 | "values": [] 3105 | }, 3106 | "yaxes": [ 3107 | { 3108 | "format": "short", 3109 | "label": null, 3110 | "logBase": 1, 3111 | "max": null, 3112 | "min": "0", 3113 | "show": true 3114 | }, 3115 | { 3116 | "decimals": null, 3117 | "format": "short", 3118 | "label": null, 3119 | "logBase": 1, 3120 | "max": null, 3121 | "min": null, 3122 | "show": true 3123 | } 3124 | ], 3125 | "yaxis": { 3126 | "align": false, 3127 | "alignLevel": null 3128 | } 3129 | }, 3130 | { 3131 | "aliasColors": {}, 3132 | "bars": false, 3133 | "dashLength": 10, 3134 | "dashes": false, 3135 | "datasource": "Elasticsearch Monitor", 3136 | "fill": 1, 3137 | "fillGradient": 0, 3138 | "gridPos": { 3139 | "h": 9, 3140 | "w": 12, 3141 | "x": 12, 3142 | "y": 89 3143 | }, 3144 | "hiddenSeries": false, 3145 | "id": 14, 3146 | "legend": { 3147 | "avg": false, 3148 | "current": false, 3149 | "max": false, 3150 | "min": false, 3151 | "show": true, 3152 | "total": false, 3153 | "values": false 3154 | }, 3155 | "lines": true, 3156 | "linewidth": 1, 3157 | "links": [], 3158 | "nullPointMode": "connected", 3159 | "options": { 3160 | "dataLinks": [] 3161 | }, 3162 | "percentage": false, 3163 | "pointradius": 5, 3164 | "points": false, 3165 | "renderer": "flot", 3166 | "seriesOverrides": [], 3167 | "spaceLength": 10, 3168 | "stack": false, 3169 | "steppedLine": false, 3170 | "targets": [ 3171 | { 3172 | "alias": "", 3173 | "bucketAggs": [ 3174 | { 3175 | "fake": true, 3176 | "field": "source_node.name", 3177 | "id": "4", 3178 | "settings": { 3179 | "min_doc_count": 1, 3180 | "order": "desc", 3181 | "orderBy": "_term", 3182 | "size": "0" 3183 | }, 3184 | "type": "terms" 3185 | }, 3186 | { 3187 | "field": "timestamp", 3188 | "id": "2", 3189 | "settings": { 3190 | "interval": "auto", 3191 | "min_doc_count": 0, 3192 | "trimEdges": 0 3193 | }, 3194 | "type": "date_histogram" 3195 | } 3196 | ], 3197 | "metrics": [ 3198 | { 3199 | "field": "node_stats.indices.request_cache.memory_size_in_bytes", 3200 | "id": "1", 3201 | "meta": {}, 3202 | "settings": {}, 3203 | "type": "max" 3204 | } 3205 | ], 3206 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 3207 | "refId": "A", 3208 | "timeField": "timestamp" 3209 | } 3210 | ], 3211 | "thresholds": [], 3212 | "timeFrom": null, 3213 | "timeRegions": [], 3214 | "timeShift": null, 3215 | "title": "Request cache memory size", 3216 | "tooltip": { 3217 | "shared": true, 3218 | "sort": 0, 3219 | "value_type": "individual" 3220 | }, 3221 | "type": "graph", 3222 | "xaxis": { 3223 | "buckets": null, 3224 | "mode": "time", 3225 | "name": null, 3226 | "show": true, 3227 | "values": [] 3228 | }, 3229 | "yaxes": [ 3230 | { 3231 | "format": "decbytes", 3232 | "label": null, 3233 | "logBase": 1, 3234 | "max": null, 3235 | "min": "0", 3236 | "show": true 3237 | }, 3238 | { 3239 | "format": "short", 3240 | "label": null, 3241 | "logBase": 1, 3242 | "max": null, 3243 | "min": null, 3244 | "show": true 3245 | } 3246 | ], 3247 | "yaxis": { 3248 | "align": false, 3249 | "alignLevel": null 3250 | } 3251 | }, 3252 | { 3253 | "aliasColors": {}, 3254 | "bars": false, 3255 | "dashLength": 10, 3256 | "dashes": false, 3257 | "datasource": "Elasticsearch Monitor", 3258 | "fill": 1, 3259 | "fillGradient": 0, 3260 | "gridPos": { 3261 | "h": 9, 3262 | "w": 12, 3263 | "x": 0, 3264 | "y": 98 3265 | }, 3266 | "hiddenSeries": false, 3267 | "id": 10, 3268 | "legend": { 3269 | "avg": false, 3270 | "current": false, 3271 | "max": false, 3272 | "min": false, 3273 | "show": true, 3274 | "total": false, 3275 | "values": false 3276 | }, 3277 | "lines": true, 3278 | "linewidth": 1, 3279 | "links": [], 3280 | "nullPointMode": "null", 3281 | "options": { 3282 | "dataLinks": [] 3283 | }, 3284 | "percentage": false, 3285 | "pointradius": 5, 3286 | "points": false, 3287 | "renderer": "flot", 3288 | "seriesOverrides": [], 3289 | "spaceLength": 10, 3290 | "stack": false, 3291 | "steppedLine": false, 3292 | "targets": [ 3293 | { 3294 | "alias": "", 3295 | "bucketAggs": [ 3296 | { 3297 | "fake": true, 3298 | "field": "source_node.name", 3299 | "id": "4", 3300 | "settings": { 3301 | "min_doc_count": 1, 3302 | "order": "desc", 3303 | "orderBy": "_term", 3304 | "size": "0" 3305 | }, 3306 | "type": "terms" 3307 | }, 3308 | { 3309 | "field": "timestamp", 3310 | "id": "2", 3311 | "settings": { 3312 | "interval": "auto", 3313 | "min_doc_count": 0, 3314 | "trimEdges": 0 3315 | }, 3316 | "type": "date_histogram" 3317 | } 3318 | ], 3319 | "metrics": [ 3320 | { 3321 | "field": "node_stats.indices.query_cache.evictions", 3322 | "hide": true, 3323 | "id": "1", 3324 | "meta": {}, 3325 | "settings": {}, 3326 | "type": "max" 3327 | }, 3328 | { 3329 | "field": "1", 3330 | "id": "3", 3331 | "meta": {}, 3332 | "pipelineAgg": "1", 3333 | "settings": {}, 3334 | "type": "derivative" 3335 | } 3336 | ], 3337 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 3338 | "refId": "A", 3339 | "timeField": "timestamp" 3340 | } 3341 | ], 3342 | "thresholds": [], 3343 | "timeFrom": null, 3344 | "timeRegions": [], 3345 | "timeShift": null, 3346 | "title": "Query cache evictions", 3347 | "tooltip": { 3348 | "shared": true, 3349 | "sort": 0, 3350 | "value_type": "individual" 3351 | }, 3352 | "type": "graph", 3353 | "xaxis": { 3354 | "buckets": null, 3355 | "mode": "time", 3356 | "name": null, 3357 | "show": true, 3358 | "values": [] 3359 | }, 3360 | "yaxes": [ 3361 | { 3362 | "format": "short", 3363 | "label": null, 3364 | "logBase": 1, 3365 | "max": null, 3366 | "min": "0", 3367 | "show": true 3368 | }, 3369 | { 3370 | "format": "short", 3371 | "label": null, 3372 | "logBase": 1, 3373 | "max": null, 3374 | "min": null, 3375 | "show": true 3376 | } 3377 | ], 3378 | "yaxis": { 3379 | "align": false, 3380 | "alignLevel": null 3381 | } 3382 | }, 3383 | { 3384 | "aliasColors": {}, 3385 | "bars": false, 3386 | "dashLength": 10, 3387 | "dashes": false, 3388 | "datasource": "Elasticsearch Monitor", 3389 | "fill": 1, 3390 | "fillGradient": 0, 3391 | "gridPos": { 3392 | "h": 9, 3393 | "w": 12, 3394 | "x": 12, 3395 | "y": 98 3396 | }, 3397 | "hiddenSeries": false, 3398 | "id": 13, 3399 | "legend": { 3400 | "avg": false, 3401 | "current": false, 3402 | "max": false, 3403 | "min": false, 3404 | "show": true, 3405 | "total": false, 3406 | "values": false 3407 | }, 3408 | "lines": true, 3409 | "linewidth": 1, 3410 | "links": [], 3411 | "nullPointMode": "connected", 3412 | "options": { 3413 | "dataLinks": [] 3414 | }, 3415 | "percentage": false, 3416 | "pointradius": 5, 3417 | "points": false, 3418 | "renderer": "flot", 3419 | "seriesOverrides": [], 3420 | "spaceLength": 10, 3421 | "stack": false, 3422 | "steppedLine": false, 3423 | "targets": [ 3424 | { 3425 | "alias": "", 3426 | "bucketAggs": [ 3427 | { 3428 | "fake": true, 3429 | "field": "source_node.name", 3430 | "id": "4", 3431 | "settings": { 3432 | "min_doc_count": 1, 3433 | "order": "desc", 3434 | "orderBy": "_term", 3435 | "size": "0" 3436 | }, 3437 | "type": "terms" 3438 | }, 3439 | { 3440 | "field": "timestamp", 3441 | "id": "2", 3442 | "settings": { 3443 | "interval": "auto", 3444 | "min_doc_count": 0, 3445 | "trimEdges": 0 3446 | }, 3447 | "type": "date_histogram" 3448 | } 3449 | ], 3450 | "metrics": [ 3451 | { 3452 | "field": "node_stats.indices.query_cache.memory_size_in_bytes", 3453 | "id": "1", 3454 | "meta": {}, 3455 | "settings": {}, 3456 | "type": "max" 3457 | } 3458 | ], 3459 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 3460 | "refId": "A", 3461 | "timeField": "timestamp" 3462 | } 3463 | ], 3464 | "thresholds": [], 3465 | "timeFrom": null, 3466 | "timeRegions": [], 3467 | "timeShift": null, 3468 | "title": "Query cache memory size", 3469 | "tooltip": { 3470 | "shared": true, 3471 | "sort": 0, 3472 | "value_type": "individual" 3473 | }, 3474 | "type": "graph", 3475 | "xaxis": { 3476 | "buckets": null, 3477 | "mode": "time", 3478 | "name": null, 3479 | "show": true, 3480 | "values": [] 3481 | }, 3482 | "yaxes": [ 3483 | { 3484 | "format": "decbytes", 3485 | "label": null, 3486 | "logBase": 1, 3487 | "max": null, 3488 | "min": "0", 3489 | "show": true 3490 | }, 3491 | { 3492 | "format": "short", 3493 | "label": null, 3494 | "logBase": 1, 3495 | "max": null, 3496 | "min": null, 3497 | "show": true 3498 | } 3499 | ], 3500 | "yaxis": { 3501 | "align": false, 3502 | "alignLevel": null 3503 | } 3504 | }, 3505 | { 3506 | "collapsed": false, 3507 | "datasource": null, 3508 | "gridPos": { 3509 | "h": 1, 3510 | "w": 24, 3511 | "x": 0, 3512 | "y": 107 3513 | }, 3514 | "id": 36, 3515 | "panels": [], 3516 | "title": "Storage", 3517 | "type": "row" 3518 | }, 3519 | { 3520 | "aliasColors": {}, 3521 | "bars": false, 3522 | "dashLength": 10, 3523 | "dashes": false, 3524 | "datasource": "Elasticsearch Monitor", 3525 | "fill": 1, 3526 | "fillGradient": 0, 3527 | "gridPos": { 3528 | "h": 9, 3529 | "w": 12, 3530 | "x": 0, 3531 | "y": 108 3532 | }, 3533 | "hiddenSeries": false, 3534 | "id": 38, 3535 | "legend": { 3536 | "avg": false, 3537 | "current": false, 3538 | "max": true, 3539 | "min": true, 3540 | "show": true, 3541 | "total": false, 3542 | "values": true 3543 | }, 3544 | "lines": true, 3545 | "linewidth": 1, 3546 | "links": [], 3547 | "nullPointMode": "connected", 3548 | "options": { 3549 | "dataLinks": [] 3550 | }, 3551 | "percentage": false, 3552 | "pointradius": 5, 3553 | "points": false, 3554 | "renderer": "flot", 3555 | "seriesOverrides": [], 3556 | "spaceLength": 10, 3557 | "stack": false, 3558 | "steppedLine": false, 3559 | "targets": [ 3560 | { 3561 | "alias": "{{source_node.name}} writes / sec", 3562 | "bucketAggs": [ 3563 | { 3564 | "fake": true, 3565 | "field": "source_node.name", 3566 | "id": "4", 3567 | "settings": { 3568 | "min_doc_count": 1, 3569 | "order": "desc", 3570 | "orderBy": "_term", 3571 | "size": "0" 3572 | }, 3573 | "type": "terms" 3574 | }, 3575 | { 3576 | "field": "timestamp", 3577 | "id": "2", 3578 | "settings": { 3579 | "interval": "auto", 3580 | "min_doc_count": 0, 3581 | "trimEdges": 0 3582 | }, 3583 | "type": "date_histogram" 3584 | } 3585 | ], 3586 | "metrics": [ 3587 | { 3588 | "field": "node_stats.indices.segments.count", 3589 | "id": "1", 3590 | "meta": {}, 3591 | "settings": {}, 3592 | "type": "max" 3593 | } 3594 | ], 3595 | "query": "type:node_stats AND node_stats.node_master:false AND node_stats.indices.docs.count:[1 TO *] AND source_node.name:($host) AND cluster_uuid:$cluster", 3596 | "refId": "A", 3597 | "timeField": "timestamp" 3598 | } 3599 | ], 3600 | "thresholds": [], 3601 | "timeFrom": null, 3602 | "timeRegions": [], 3603 | "timeShift": null, 3604 | "title": "Segments Count", 3605 | "tooltip": { 3606 | "shared": true, 3607 | "sort": 0, 3608 | "value_type": "individual" 3609 | }, 3610 | "type": "graph", 3611 | "xaxis": { 3612 | "buckets": null, 3613 | "mode": "time", 3614 | "name": null, 3615 | "show": true, 3616 | "values": [] 3617 | }, 3618 | "yaxes": [ 3619 | { 3620 | "format": "short", 3621 | "label": null, 3622 | "logBase": 1, 3623 | "max": null, 3624 | "min": "0", 3625 | "show": true 3626 | }, 3627 | { 3628 | "format": "short", 3629 | "label": null, 3630 | "logBase": 1, 3631 | "max": null, 3632 | "min": null, 3633 | "show": false 3634 | } 3635 | ], 3636 | "yaxis": { 3637 | "align": false, 3638 | "alignLevel": null 3639 | } 3640 | } 3641 | ], 3642 | "refresh": "1m", 3643 | "schemaVersion": 22, 3644 | "style": "dark", 3645 | "tags": [], 3646 | "templating": { 3647 | "list": [ 3648 | { 3649 | "allValue": "*", 3650 | "current": { 3651 | "text": "All", 3652 | "value": [ 3653 | "$__all" 3654 | ] 3655 | }, 3656 | "datasource": "Elasticsearch Monitor", 3657 | "definition": "", 3658 | "hide": 0, 3659 | "includeAll": true, 3660 | "label": "host", 3661 | "multi": true, 3662 | "name": "host", 3663 | "options": [], 3664 | "query": "{\"find\": \"terms\", \"field\": \"source_node.name\"}", 3665 | "refresh": 1, 3666 | "regex": "", 3667 | "skipUrlSync": false, 3668 | "sort": 1, 3669 | "tagValuesQuery": "", 3670 | "tags": [], 3671 | "tagsQuery": "", 3672 | "type": "query", 3673 | "useTags": false 3674 | }, 3675 | { 3676 | "allValue": null, 3677 | "current": { 3678 | "text": "All", 3679 | "value": [ 3680 | "$__all" 3681 | ] 3682 | }, 3683 | "datasource": "Elasticsearch Monitor", 3684 | "definition": "", 3685 | "hide": 0, 3686 | "includeAll": true, 3687 | "label": "index", 3688 | "multi": true, 3689 | "name": "index", 3690 | "options": [], 3691 | "query": "{\"find\": \"terms\", \"field\": \"index_stats.index\"}", 3692 | "refresh": 1, 3693 | "regex": "", 3694 | "skipUrlSync": false, 3695 | "sort": 1, 3696 | "tagValuesQuery": "", 3697 | "tags": [], 3698 | "tagsQuery": "", 3699 | "type": "query", 3700 | "useTags": false 3701 | }, 3702 | { 3703 | "allValue": "*", 3704 | "current": { 3705 | "selected": true, 3706 | "text": "Cn2TEmrYSqyRkY9EYMKsww", 3707 | "value": "Cn2TEmrYSqyRkY9EYMKsww" 3708 | }, 3709 | "datasource": "Elasticsearch Monitor", 3710 | "definition": "{\"find\": \"terms\", \"field\": \"cluster_uuid\"}", 3711 | "hide": 0, 3712 | "includeAll": false, 3713 | "label": "cluster", 3714 | "multi": false, 3715 | "name": "cluster", 3716 | "options": [], 3717 | "query": "{\"find\": \"terms\", \"field\": \"cluster_uuid\"}", 3718 | "refresh": 1, 3719 | "regex": "", 3720 | "skipUrlSync": false, 3721 | "sort": 1, 3722 | "tagValuesQuery": "", 3723 | "tags": [], 3724 | "tagsQuery": "", 3725 | "type": "query", 3726 | "useTags": false 3727 | } 3728 | ] 3729 | }, 3730 | "time": { 3731 | "from": "now-30m", 3732 | "to": "now" 3733 | }, 3734 | "timepicker": { 3735 | "refresh_intervals": [ 3736 | "5s", 3737 | "10s", 3738 | "30s", 3739 | "1m", 3740 | "5m", 3741 | "15m", 3742 | "30m", 3743 | "1h", 3744 | "2h", 3745 | "1d" 3746 | ], 3747 | "time_options": [ 3748 | "5m", 3749 | "15m", 3750 | "1h", 3751 | "6h", 3752 | "12h", 3753 | "24h", 3754 | "2d", 3755 | "7d", 3756 | "30d" 3757 | ] 3758 | }, 3759 | "timezone": "", 3760 | "title": "Elasticsearch Cluster Monitoring", 3761 | "uid": "9BHh3jfmz", 3762 | "version": 7 3763 | } 3764 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elasticsearch monitoring with Pulse 2 | 3 | This repository contains everything required for end-to-end thorough monitoring of an Elasticsearch cluster. 4 | 5 | In general, you should use metricbeat with Pulse in order to get the optimal monitoring experience for your Elasticsearch cluster. 6 | 7 | Pulse was crafted and is continually being updated and improved based on experience with debugging and stabilizing many Elasticsearch clusters world-wide. 8 | 9 | For more information on Pulse: 10 | 11 | https://bigdataboutique.com/contact 12 | 13 | As metricbeat is not an option for AWS Elasticsearch clusters, you can use this script to fetch data from them. 14 | 15 | This repository still allows relaying monitoring data to any Elasticsearch target, and contains a Grafana dashboard that works with the data created by it. 16 | 17 | However further improvements as well as advanced capabilities such as alerts are available only for Pulse. 18 | 19 | 20 | Elasticsearch monitoring with Grafana 21 | 22 | ## Gathering metrics 23 | 24 | ### Using Metricbeat 25 | 26 | Elastic's Metricbeat is provided with an agent that can ship metrics to the cluster used for monitoring. This is a push-based approach, and requires installing and configuring metricbeat. To go that route, please follow the installation instructions here: https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-metricbeat.html 27 | 28 | ### Using provided script 29 | 30 | Another approach is to use the elasticsearch.monitoring script provided with this repository, which you can find at `elasticsearch.monitoring/fetch_stats.py`. 31 | You can either do this directly with python, or use the Dockerfile in this repository. See instructions for docker use below. 32 | Use environment variables or program arguments to set the URLs for the monitored cluster and the cluster that is being used for monitoring. 33 | By default they are both configured to be http://localhost:9200/ , make sure to use the format http://host:port/ . 34 | 35 | * ES_METRICS_CLUSTER_URL - A comma seperated list of the monitored nodes (client/LB or data nodes if no other option). 36 | * ES_METRICS_MONITORING_CLUSTER_URL - host for the monitoring cluster. 37 | * ES_METRICS_INDEX_NAME - The index into which the monitoring data will go, can be left as is. default is '.monitoring-es-7-' 38 | * NUMBER_OF_REPLICAS - number of replicas for indices created on the monitoring server. Default is 1 39 | * ES_METRICS_MONITORING_AUTH_TOKEN - X-Auth-Token header value if required for the target cluster 40 | * ES_METRICS_CLOUD_PROVIDER - either 'Amazon Elasticsearch' or 'Elastic Cloud' or not provided 41 | * ES_USERNAME and ES_PASSWORD - if applicable, for authenticating against the source cluster 42 | 43 | You can also set polling interval (10 seconds by default) and a prefix for the index name. See the script for more details. 44 | 45 | Don't forget to install all dependencies by running: 46 | 47 | `pip install -r requirements.txt` 48 | 49 | The benefit of this approach is that it doesn't require installing a plugin, and is shipping the necessary information, with some important extras, as opposed to the Metricbeat approach. 50 | 51 | Once installed and configured, have the Python script run as a service to continuously collect metrics (with systemd for instance: https://linuxconfig.org/how-to-create-systemd-service-unit-in-linux). 52 | At launch, you can see a printed message verifying the script is drawing data from and into the correct hosts. 53 | To further validate that, you can also check the values of field source_node.host in the index with the monitoring data. 54 | 55 | *NOTE:* If the cluster you are using for monitoring is 2.x, you will need to edit the template files (`elasticsearch.monitoring/templates/*`) and change all occurences of `"type": "keyword"` with `"type": "string", "index": "not_analyzed"`. 56 | 57 | ### Docker setup 58 | 59 | `sudo apt update` 60 | 61 | `sudo apt install docker.io` 62 | 63 | `sudo docker build . -t fetch_stats` 64 | 65 | `sudo docker run --net=host --env ES_METRICS_CLUSTER_URL=http://localhost:9200/ fetch_stats /app/elasticsearch.monitoring/fetch_stats.py` 66 | 67 | where ES_METRICS_CLUSTER_URL is setup to the monitored ES, and obviously adding additional variables if required. 68 | Run once in the foreground to validate that the script works correctly, then use -d in the docker run to run in background. 69 | 70 | ### AWS pulse setup 71 | 72 | Same as above - you'll get specific values for ES_METRICS_MONITORING_CLUSTER_URL and ES_METRICS_MONITORING_AUTH_TOKEN, 73 | 74 | In addition you should set ES_METRICS_CLOUD_PROVIDER to 'Amazon Elasticsearch'. 75 | 76 | when running, make sure to set --health-flag=False . 77 | 78 | You can opt not to verify your cluster's TLS certificate with --verify-flag=False . 79 | 80 | ## Visualizing with Grafana 81 | 82 | You will need to create an Elasticsearch data source that is pointing to the cluster you use for monitoring. 83 | The following are the settings that conform with the data collected by the provided script: 84 | * Assuming Grafana runs from the same host as the monitor, leave URL as `http://localhost:9200`, otherwise change it 85 | * Set the name to `Elasticsearch Monitor` 86 | * Set the index pattern to be `[.monitoring-es-*-]YYYY.MM.DD` 87 | * Set the version to the version of your monitoring cluster 88 | * Set Pattern to daily. 89 | * Set the timestamp field to `timestamp`. 90 | * Save and test the connection. 91 | You can now import the dashboard found at `Elasticsearch Cluster Monitoring.json` and start monitoring your cluster! 92 | 93 | For more details see the official documentation here: http://docs.grafana.org/features/datasources/elasticsearch/. 94 | 95 | For guidance on importing the dashboard see the official Grafana documentation: http://docs.grafana.org/reference/export_import/#importing-a-dashboard. 96 | 97 | An outdated, but useful, information on collected metrics: 98 | https://www.elastic.co/guide/en/elasticsearch/guide/current/_monitoring_individual_nodes.html 99 | -------------------------------------------------------------------------------- /elasticsearch.monitoring/__init__.py: -------------------------------------------------------------------------------- 1 | VERSION = (5, 0, 0) 2 | __version__ = VERSION 3 | __versionstr__ = '.'.join(map(str, VERSION)) 4 | -------------------------------------------------------------------------------- /elasticsearch.monitoring/fetch_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import datetime 3 | import json 4 | import logging 5 | import os 6 | import random 7 | import socket 8 | import sys 9 | import time 10 | 11 | import click 12 | import requests 13 | import urllib3 14 | from requests.auth import HTTPBasicAuth 15 | from dictor import dictor 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | working_dir = os.path.dirname(os.path.realpath(__file__)) 20 | def merge(one, two): 21 | cp = one.copy() 22 | cp.update(two) 23 | return cp 24 | 25 | def color_to_level(color): 26 | return { 27 | 'green': 0, 28 | 'yellow': 1, 29 | 'red': 2 30 | }.get(color, 3) 31 | 32 | def lookup(data, selector): 33 | keys = selector.split('.') 34 | value = data 35 | while keys: 36 | value = value[keys.pop(0)] 37 | return value 38 | 39 | def delete_path(data, selector): 40 | keys = selector.split('.') 41 | value = data 42 | while keys: 43 | k = keys.pop(0) 44 | if k not in value: 45 | return 46 | value = value[k] 47 | 48 | def flatten_json(y): 49 | out = {} 50 | def flatten(x, name=''): 51 | if type(x) is dict: 52 | for a in x: 53 | flatten(x[a], name + a + '.') 54 | elif type(x) is list: 55 | i = 0 56 | for a in x: 57 | flatten(a, name + str(i) + '.') 58 | i += 1 59 | else: 60 | out[name[:-1]] = x 61 | flatten(y) 62 | return out 63 | 64 | def assert_http_status(response, expected_status_code=200): 65 | if response.status_code != expected_status_code: 66 | print(response.text) 67 | raise Exception('Expected HTTP status code %d but got %d' % (expected_status_code, response.status_code)) 68 | 69 | cluster_uuid = None 70 | auth = None 71 | 72 | # Elasticsearch Cluster to send metrics to 73 | monitoringCluster = os.environ.get('ES_METRICS_MONITORING_CLUSTER_URL', 'http://localhost:9200/') 74 | if not monitoringCluster.endswith("/"): 75 | monitoringCluster = monitoringCluster + '/' 76 | indexPrefix = os.environ.get('ES_METRICS_INDEX_NAME', '.monitoring-es-7-') 77 | numberOfReplicas = os.environ.get('NUMBER_OF_REPLICAS', '1') 78 | 79 | #shaig 19.7 - turning cluster health into cluster stats 80 | def fetch_cluster_stats(base_url='http://localhost:9200/',health = True,verify=True,cloud_provider = None): 81 | metric_docs = [] 82 | try: 83 | response = requests.get(base_url + '_cluster/health', timeout=(5, 5), auth=auth, verify=verify) 84 | if response.status_code != 200: 85 | try: 86 | error_msg = response.json() 87 | except Exception: 88 | error_msg = response.text 89 | logger.error("bad response while getting cluster health. response is:\n" + error_msg) 90 | return metric_docs, None 91 | cluster_health = response.json() 92 | utc_datetime = datetime.datetime.utcnow() 93 | ts = str(utc_datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z') 94 | cluster_health['timestamp'] = ts 95 | cluster_health['status_code'] = color_to_level(cluster_health['status']) 96 | if health: 97 | metric_docs.append(cluster_health) 98 | response = requests.get(base_url + '_cluster/stats', timeout=(5, 5), auth=auth, verify=verify) 99 | if response.status_code != 200: 100 | logger.error("bad response while getting cluster stats. response is:\n" + response.json()) 101 | return metric_docs,None 102 | cluster_stats = response.json() 103 | # creating cluster stats json 104 | cluster_stats_and_state = {'type':'cluster_stats','cluster_stats': cluster_stats, 'cluster_name': cluster_stats['cluster_name'], 'timestamp': ts, '@timestamp': cluster_stats['timestamp'],'cluster_uuid':cluster_stats['cluster_uuid'],'cloud_provider':cloud_provider} 105 | response = requests.get(base_url + '_cluster/state', timeout=(5, 5), auth=auth, verify=verify) 106 | if response.status_code != 200: 107 | logger.error("bad response while getting cluster state. response is:\n" + response.json()) 108 | return metric_docs,None 109 | cluster_state = response.json() 110 | cluster_state_json = {'nodes': cluster_state['nodes'],'cluster_uuid':cluster_state['cluster_uuid'],'state_uuid':cluster_state['state_uuid'],'master_node':cluster_state['master_node'],'version':cluster_state['version'],'status':cluster_health['status']} 111 | routing_table = cluster_state['routing_table']['indices'] 112 | if type(routing_table) is not dict: 113 | logger.error("bad routing table from cluster state. response is:\n"+ response.json()) 114 | cluster_stats_and_state['cluster_state'] = cluster_state_json 115 | metric_docs.append(cluster_stats_and_state) 116 | return metric_docs,routing_table 117 | except (requests.exceptions.Timeout, socket.timeout) as e: 118 | logger.error("[%s] Timeout received on trying to get cluster stats" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 119 | return [],[] 120 | 121 | node_stats_to_collect = ["indices", "os", "process", "jvm", "thread_pool", "fs", "transport", "http", "breakers", "script"] 122 | 123 | def fetch_nodes_stats(base_url='http://localhost:9200/',verify=True): 124 | metric_docs = [] 125 | r_json = None 126 | try: 127 | response = requests.get(base_url + '_nodes/stats', timeout=(5, 5), auth=auth, verify=verify) 128 | if response.status_code != 200: 129 | try: 130 | error_msg = response.json() 131 | except Exception: 132 | error_msg = response.text 133 | logger.error("bad response while getting node stats. response is:\n" + error_msg) 134 | return metric_docs 135 | r_json = response.json() 136 | cluster_name = r_json['cluster_name'] 137 | 138 | # we are opting to not use the timestamp as reported by the actual node 139 | # to be able to better sync the various metrics collected 140 | utc_datetime = datetime.datetime.utcnow() 141 | ts = str(utc_datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z') 142 | nodes_dict = r_json['nodes'] 143 | if type(nodes_dict) is not dict: 144 | logger.error("bad node stats. response is:\n" + response.json()) 145 | return metric_docs 146 | for node_id, node in nodes_dict.items(): 147 | doc_timestamp = datetime.datetime.fromtimestamp(node['timestamp']/1000.0) 148 | doc_ts = str(doc_timestamp.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z') 149 | node_data = { 150 | "timestamp": ts, 151 | "@timestamp": doc_ts, 152 | "cluster_name": cluster_name, 153 | "cluster_uuid": cluster_uuid, 154 | "source_node": { 155 | "uuid": node_id, 156 | "host": node.get('host'), 157 | "transport_address": node.get('transport_address'), 158 | "ip": node.get('ip'), 159 | "name": node.get('name'), 160 | "roles": node.get('roles'), 161 | "attributes": {} # TODO do we want to bring anything here? 162 | }, 163 | } 164 | 165 | is_master = ('roles' in node and 'master' in node['roles']) or ('attributes' in node and 'master' in node['attributes'] and ( 166 | node['attributes']['master'] == 'true' or node['attributes']['master'] == True)) 167 | node_data["node_stats"] = { 168 | "node_id": node_id, 169 | "node_master": is_master, 170 | "mlockall": True, # TODO here for compat reasons only 171 | } 172 | 173 | for k in node_stats_to_collect: 174 | node_data["node_stats"][k] = node.get(k) 175 | 176 | # clean up some stuff 177 | delete_path(node_data["node_stats"], "os.timestamp") 178 | del node_data["node_stats"]["process"]["timestamp"] 179 | del node_data["node_stats"]["os"]["timestamp"] 180 | del node_data["node_stats"]["jvm"]["timestamp"] 181 | del node_data["node_stats"]["jvm"]["mem"]["pools"] 182 | del node_data["node_stats"]["jvm"]["buffer_pools"] 183 | del node_data["node_stats"]["jvm"]["uptime_in_millis"] 184 | # TODO remove some thread pools stats 185 | del node_data["node_stats"]["fs"]["timestamp"] 186 | del node_data["node_stats"]["fs"]["data"] 187 | metric_docs.append(node_data) 188 | except (requests.exceptions.Timeout, socket.timeout) as e: 189 | logger.error("[%s] Timeout received on trying to get nodes stats" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 190 | return metric_docs 191 | 192 | 193 | def get_shard_data(routing_table): 194 | primaries = 0 195 | replicas = 0 196 | 197 | active_primaries = 0 198 | active_replicas = 0 199 | 200 | unassigned_primaries = 0 201 | unassigned_replicas = 0 202 | 203 | initializing = 0 204 | relocating = 0 205 | for shard in routing_table.items(): 206 | key,unique_shard = shard 207 | for replica in unique_shard: 208 | isPrimary = bool(replica['primary']) 209 | state = replica['state'] 210 | if isPrimary: 211 | primaries+= 1 212 | if state == 'STARTED': 213 | active_primaries += 1 214 | elif state == 'UNASSIGNED': 215 | unassigned_primaries += 1 216 | else: 217 | replicas += 1 218 | if state == 'STARTED': 219 | active_replicas += 1 220 | elif state == 'UNASSIGNED': 221 | unassigned_replicas += 1 222 | if state == 'INITIALIZING': 223 | initializing += 1 224 | if state == 'RELOCATING': 225 | relocating += 1 226 | if primaries == 0 or primaries != active_primaries: 227 | index_status = 'red' 228 | elif replicas != active_replicas: 229 | index_status = 'yellow' 230 | else: 231 | index_status = 'green' 232 | return { 233 | 'total': primaries + replicas, 234 | 'primaries': primaries, 235 | 'replicas': replicas, 236 | 'active_total' : active_primaries + active_replicas, 237 | 'active_primaries': active_primaries, 238 | 'active_replicas': active_replicas, 239 | 240 | 'unassigned_total' : unassigned_primaries + unassigned_replicas, 241 | 'unassigned_primaries': unassigned_primaries, 242 | 'unassigned_replicas': unassigned_replicas, 243 | 'initializing': initializing, 244 | 'relocating': relocating 245 | },index_status 246 | 247 | 248 | # shaig 18.3 - adding data for index stats 249 | def fetch_index_stats(routing_table,base_url='http://localhost:9200/',verify=True): 250 | metric_docs = [] 251 | try: 252 | # getting timestamp 253 | utc_datetime = datetime.datetime.utcnow() 254 | ts = str(utc_datetime.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z') 255 | # getting index stats for all indices 256 | response = requests.get(base_url + '_stats', timeout=(5, 5), auth=auth, verify=verify) 257 | if response.status_code != 200: 258 | logger.error("bad response while getting index stats. response is:\n"+response.json()) 259 | return metric_docs 260 | index_stats = response.json() 261 | # creating index stats json 262 | indices = index_stats['indices'] 263 | #AWS doesn't accept /_settings 264 | response = requests.get(base_url + '*/_settings', timeout=(5, 5), auth=auth, verify=verify) 265 | if response.status_code != 200: 266 | logger.error("bad response while getting index settings. response is:\n" + response.json()) 267 | return metric_docs 268 | index_settings = response.json() 269 | if type(index_settings) is not dict: 270 | logger.error("bad index settings. response is:\n" + response.json()) 271 | return metric_docs 272 | index_settings_ordered = dict(sorted(index_settings.items())) 273 | routing_table_ordered = dict(sorted(routing_table.items())) 274 | 275 | for index_name in indices: 276 | routing_table = routing_table_ordered[index_name]['shards'] 277 | shards,index_status = get_shard_data(routing_table) 278 | #logger.info("Building log for index " + index_name) 279 | # unlike other stats types, @timestamp is based on current time because there is no document timestamp 280 | index_data = { 281 | "timestamp": ts , 282 | "@timestamp": ts , 283 | "cluster_uuid": cluster_uuid, 284 | "type": "index_stats" 285 | } 286 | index_data['index_stats'] = indices[index_name] 287 | index_data['index_stats']['index'] = index_name 288 | index_data['index_stats']['shards'] = shards 289 | index_name_escaped = index_name 290 | if index_name.startswith("."): 291 | index_name_escaped = index_name.replace(".","\.") 292 | index_data['index_stats']['created'] = \ 293 | dictor(index_settings_ordered, index_name_escaped + '.settings.index.creation_date') 294 | # hidden is a new proprety from 7.7. Notice it is not identical to system indices 295 | hidden = dictor(index_settings_ordered, index_name_escaped + '.settings.index.hidden') 296 | if hidden is not None: 297 | index_data['index_stats']['hidden'] = hidden 298 | index_data['index_stats']['status'] = index_status 299 | metric_docs.append(index_data) 300 | # creating indices stats json 301 | summary = index_stats["_all"] 302 | summary_data = { 303 | "timestamp": ts, 304 | "cluster_uuid": cluster_uuid, 305 | "type": "indices_stats" 306 | } 307 | summary_data["indices_stats"] = {} 308 | summary_data["indices_stats"]["_all"] = summary 309 | metric_docs.append(summary_data) 310 | except (requests.exceptions.Timeout, socket.timeout) as e: 311 | logger.error("[%s] Timeout received on getting index stats" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 312 | return metric_docs 313 | 314 | def create_templates(): 315 | for filename in os.listdir(os.path.join(working_dir, 'templates')): 316 | if filename.endswith(".json"): 317 | with open(os.path.join(working_dir, 'templates', filename)) as query_base: 318 | template = query_base.read() 319 | template = template.replace('{{INDEX_PREFIX}}', indexPrefix + '*').strip() 320 | template = template.replace('{{NUMBER_OF_REPLICAS}}', numberOfReplicas).strip() 321 | templates_response = requests.put(monitoringCluster + '_template/' + indexPrefix.replace('.', '') + filename[:-5], 322 | data = template, 323 | headers={'Content-Type': 'application/json;charset=UTF-8'}, 324 | timeout=(30, 30)) 325 | assert_http_status(templates_response) 326 | 327 | 328 | def poll_metrics(cluster_host, monitor, monitor_host,health,verify,auth_token,cloud_provider): 329 | cluster_stats, node_stats,index_stats = get_all_data(cluster_host,health,verify,cloud_provider) 330 | if monitor == 'elasticsearch': 331 | into_elasticsearch(monitor_host, cluster_stats, node_stats,index_stats,auth_token) 332 | 333 | 334 | def get_all_data(cluster_host,health,verify,cloud_provider): 335 | cluster_stats,routing_table = fetch_cluster_stats(cluster_host,health,verify,cloud_provider) 336 | node_stats = fetch_nodes_stats(cluster_host,verify) 337 | if type(routing_table) is not dict: 338 | index_stats = [] 339 | else: 340 | index_stats = fetch_index_stats(routing_table,cluster_host,verify) 341 | return cluster_stats, node_stats,index_stats 342 | 343 | def into_elasticsearch(monitor_host, cluster_stats, node_stats,index_stats,auth_token): 344 | utc_datetime = datetime.datetime.utcnow() 345 | index_name = indexPrefix + str(utc_datetime.strftime('%Y.%m.%d')) 346 | 347 | cluster_stats_data = ['{"index":{"_index":"'+index_name+'","_type":"_doc"}}\n' + json.dumps(o) for o in cluster_stats] 348 | node_stats_data = ['{"index":{"_index":"'+index_name+'","_type":"_doc"}}\n' + json.dumps(with_type(o, 'node_stats')) for o in node_stats] 349 | data = node_stats_data + cluster_stats_data 350 | if index_stats is not None: 351 | index_stats_data = ['{"index":{"_index":"' + index_name + '","_type":"_doc"}}\n' + json.dumps( 352 | o) for o in index_stats] 353 | data += index_stats_data 354 | 355 | if len(data) > 0: 356 | try: 357 | headers = {'Content-Type': 'application/x-ndjson'} 358 | if auth_token is not None: 359 | headers['X-Auth-Token'] = auth_token 360 | bulk_response = requests.post(monitor_host + '_bulk', 361 | data='\n'.join(data), 362 | headers=headers, 363 | timeout=(30, 30)) 364 | assert_http_status(bulk_response) 365 | for item in bulk_response.json()["items"]: 366 | if item.get("index") and item.get("index").get("status") != 201: 367 | click.echo(json.dumps(item.get("index").get("error"))) 368 | click.echo(cluster_stats_data) 369 | click.echo("[%s] Pushed data successfully" % (time.strftime("%Y-%m-%d %H:%M:%S"),)) 370 | except (requests.exceptions.Timeout, socket.timeout): 371 | logger.error("[%s] Timeout received while pushing collected metrics to Elasticsearch" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 372 | 373 | 374 | def with_type(o, _type): 375 | o["type"] = _type 376 | return o 377 | 378 | 379 | @click.command() 380 | @click.argument('monitor-host', default=monitoringCluster) 381 | @click.argument('monitor', default='elasticsearch') 382 | @click.argument('cluster-host', default='http://localhost:9200/') 383 | @click.option('--username', default=None) 384 | @click.option('--pwd', default=None) 385 | @click.option('--interval', default=10, help='Interval (in seconds) to run this') 386 | @click.option('--index-prefix', default='', help='Index prefix for Elastic monitor') 387 | @click.option('--generate-templates', default=False) 388 | @click.option('--health-flag', default=True) 389 | @click.option('--verify-flag', default=True) 390 | def main(monitor_host, monitor, cluster_host, username, pwd, interval, index_prefix,generate_templates,health_flag,verify_flag ): 391 | global cluster_uuid, indexPrefix, auth 392 | verify = verify_flag == 'True' 393 | if not verify: 394 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 395 | health = health_flag == 'True' 396 | monitored_cluster = os.environ.get('ES_METRICS_CLUSTER_URL', cluster_host) 397 | auth_token = os.environ.get('ES_METRICS_MONITORING_AUTH_TOKEN') 398 | cloud_provider = os.environ.get('ES_METRICS_CLOUD_PROVIDER') 399 | username = os.environ.get('ES_USERNAME', username) 400 | pwd = os.environ.get('ES_PASSWORD', pwd) 401 | if cloud_provider is not None and cloud_provider not in ['Amazon Elasticsearch','Elastic Cloud']: 402 | click.echo('supported cloud providers are "Amazon Elasticsearch" and "Elastic Cloud"') 403 | sys.exit(1) 404 | if ',' in monitored_cluster: 405 | cluster_hosts_source = monitored_cluster.split(',') 406 | else: 407 | cluster_hosts_source = [monitored_cluster] 408 | cluster_hosts = [] 409 | for cluster in cluster_hosts_source: 410 | if not cluster.endswith("/"): 411 | cluster_hosts.append(cluster + "/") 412 | else: 413 | cluster_hosts.append(cluster) 414 | 415 | indexPrefix = index_prefix or indexPrefix 416 | 417 | click.echo('[%s] Monitoring %s into %s at %s' % (time.strftime("%Y-%m-%d %H:%M:%S"), monitored_cluster, monitor, monitor_host)) 418 | if username and pwd: 419 | auth = HTTPBasicAuth(username, pwd) 420 | 421 | init = False 422 | while not init: 423 | try: 424 | response = requests.get(random.choice(cluster_hosts), auth=auth, timeout=(5, 5), verify=verify) 425 | assert_http_status(response) 426 | cluster_uuid = response.json().get('cluster_uuid') 427 | init = True 428 | if monitor == 'elasticsearch' and generate_templates: 429 | try: 430 | create_templates() 431 | except (requests.exceptions.Timeout, socket.timeout, requests.exceptions.ConnectionError): 432 | click.echo("[%s] Timeout received when trying to put template" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 433 | except (requests.exceptions.Timeout, socket.timeout, requests.exceptions.ConnectionError) as e: 434 | click.echo("[%s] Timeout received on trying to get cluster uuid" % (time.strftime("%Y-%m-%d %H:%M:%S"))) 435 | sys.exit(1) 436 | 437 | click.echo('[%s] Started' % (time.strftime("%Y-%m-%d %H:%M:%S"),)) 438 | 439 | recurring = interval > 0 440 | if not recurring: 441 | poll_metrics(cluster_host, monitor, monitor_host,health,verify,auth_token,cloud_provider) 442 | else: 443 | try: 444 | nextRun = 0 445 | while True: 446 | if time.time() >= nextRun: 447 | nextRun = time.time() + interval 448 | now = time.time() 449 | 450 | poll_metrics(random.choice(cluster_hosts), monitor, monitor_host,health,verify,auth_token,cloud_provider) 451 | 452 | elapsed = time.time() - now 453 | # click.echo("[%s] Total Elapsed Time: %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), elapsed)) 454 | timeDiff = nextRun - time.time() 455 | 456 | # Check timediff , if timediff >=0 sleep, if < 0 send metrics to es 457 | if timeDiff >= 0: 458 | time.sleep(timeDiff) 459 | except requests.exceptions.ConnectionError as e: 460 | click.echo("[%s] FATAL Connection error %s, quitting" % (time.strftime("%Y-%m-%d %H:%M:%S"), e)) 461 | time.sleep(10) 462 | except KeyboardInterrupt: 463 | click.echo('Interrupted') 464 | try: 465 | sys.exit(0) 466 | except SystemExit as e: 467 | logger.error(e) 468 | os._exit(1) 469 | 470 | 471 | if __name__ == '__main__': 472 | main() 473 | -------------------------------------------------------------------------------- /elasticsearch.monitoring/templates/monitoring-es.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": 0, 3 | "template": "{{INDEX_PREFIX}}", 4 | "settings": { 5 | "index": { 6 | "codec": "best_compression", 7 | "number_of_shards": "1", 8 | "number_of_replicas": "{{NUMBER_OF_REPLICAS}}" 9 | } 10 | }, 11 | "mappings":{ 12 | "dynamic": "false", 13 | "date_detection": false, 14 | "properties": { 15 | "ccr_auto_follow_stats": { 16 | "properties": { 17 | "auto_followed_clusters": { 18 | "type": "nested", 19 | "properties": { 20 | "cluster_name": { 21 | "type": "keyword" 22 | }, 23 | "last_seen_metadata_version": { 24 | "type": "long" 25 | }, 26 | "time_since_last_check_millis": { 27 | "type": "long" 28 | } 29 | } 30 | }, 31 | "number_of_failed_follow_indices": { 32 | "type": "long" 33 | }, 34 | "number_of_failed_remote_cluster_state_requests": { 35 | "type": "long" 36 | }, 37 | "number_of_successful_follow_indices": { 38 | "type": "long" 39 | }, 40 | "recent_auto_follow_errors": { 41 | "type": "nested", 42 | "properties": { 43 | "auto_follow_exception": { 44 | "properties": { 45 | "reason": { 46 | "type": "text" 47 | }, 48 | "type": { 49 | "type": "keyword" 50 | } 51 | } 52 | }, 53 | "leader_index": { 54 | "type": "keyword" 55 | }, 56 | "timestamp": { 57 | "type": "long" 58 | } 59 | } 60 | } 61 | } 62 | }, 63 | "ccr_stats": { 64 | "properties": { 65 | "bytes_read": { 66 | "type": "long" 67 | }, 68 | "failed_read_requests": { 69 | "type": "long" 70 | }, 71 | "failed_write_requests": { 72 | "type": "long" 73 | }, 74 | "fatal_exception": { 75 | "properties": { 76 | "reason": { 77 | "type": "text" 78 | }, 79 | "type": { 80 | "type": "keyword" 81 | } 82 | } 83 | }, 84 | "follower_aliases_version": { 85 | "type": "long" 86 | }, 87 | "follower_global_checkpoint": { 88 | "type": "long" 89 | }, 90 | "follower_index": { 91 | "type": "keyword" 92 | }, 93 | "follower_mapping_version": { 94 | "type": "long" 95 | }, 96 | "follower_max_seq_no": { 97 | "type": "long" 98 | }, 99 | "follower_settings_version": { 100 | "type": "long" 101 | }, 102 | "last_requested_seq_no": { 103 | "type": "long" 104 | }, 105 | "leader_global_checkpoint": { 106 | "type": "long" 107 | }, 108 | "leader_index": { 109 | "type": "keyword" 110 | }, 111 | "leader_max_seq_no": { 112 | "type": "long" 113 | }, 114 | "operations_read": { 115 | "type": "long" 116 | }, 117 | "operations_written": { 118 | "type": "long" 119 | }, 120 | "outstanding_read_requests": { 121 | "type": "long" 122 | }, 123 | "outstanding_write_requests": { 124 | "type": "long" 125 | }, 126 | "read_exceptions": { 127 | "type": "nested", 128 | "properties": { 129 | "exception": { 130 | "properties": { 131 | "reason": { 132 | "type": "text" 133 | }, 134 | "type": { 135 | "type": "keyword" 136 | } 137 | } 138 | }, 139 | "from_seq_no": { 140 | "type": "long" 141 | }, 142 | "retries": { 143 | "type": "integer" 144 | } 145 | } 146 | }, 147 | "remote_cluster": { 148 | "type": "keyword" 149 | }, 150 | "shard_id": { 151 | "type": "integer" 152 | }, 153 | "successful_read_requests": { 154 | "type": "long" 155 | }, 156 | "successful_write_requests": { 157 | "type": "long" 158 | }, 159 | "time_since_last_read_millis": { 160 | "type": "long" 161 | }, 162 | "total_read_remote_exec_time_millis": { 163 | "type": "long" 164 | }, 165 | "total_read_time_millis": { 166 | "type": "long" 167 | }, 168 | "total_write_time_millis": { 169 | "type": "long" 170 | }, 171 | "write_buffer_operation_count": { 172 | "type": "long" 173 | }, 174 | "write_buffer_size_in_bytes": { 175 | "type": "long" 176 | } 177 | } 178 | }, 179 | "cluster_state": { 180 | "properties": { 181 | "master_node": { 182 | "type": "keyword" 183 | }, 184 | "nodes": { 185 | "type": "object" 186 | }, 187 | "nodes_hash": { 188 | "type": "integer" 189 | }, 190 | "shards": { 191 | "type": "object" 192 | }, 193 | "state_uuid": { 194 | "type": "keyword" 195 | }, 196 | "status": { 197 | "type": "keyword" 198 | }, 199 | "version": { 200 | "type": "long" 201 | } 202 | } 203 | }, 204 | "cluster_stats": { 205 | "properties": { 206 | "indices": { 207 | "type": "object" 208 | }, 209 | "nodes": { 210 | "type": "object" 211 | } 212 | } 213 | }, 214 | "cluster_uuid": { 215 | "type": "keyword" 216 | }, 217 | "index_recovery": { 218 | "type": "object" 219 | }, 220 | "index_stats": { 221 | "properties": { 222 | "index": { 223 | "type": "keyword" 224 | }, 225 | "primaries": { 226 | "properties": { 227 | "docs": { 228 | "properties": { 229 | "count": { 230 | "type": "long" 231 | } 232 | } 233 | }, 234 | "fielddata": { 235 | "properties": { 236 | "evictions": { 237 | "type": "long" 238 | }, 239 | "memory_size_in_bytes": { 240 | "type": "long" 241 | } 242 | } 243 | }, 244 | "indexing": { 245 | "properties": { 246 | "index_time_in_millis": { 247 | "type": "long" 248 | }, 249 | "index_total": { 250 | "type": "long" 251 | }, 252 | "throttle_time_in_millis": { 253 | "type": "long" 254 | } 255 | } 256 | }, 257 | "merges": { 258 | "properties": { 259 | "total_size_in_bytes": { 260 | "type": "long" 261 | } 262 | } 263 | }, 264 | "query_cache": { 265 | "properties": { 266 | "evictions": { 267 | "type": "long" 268 | }, 269 | "hit_count": { 270 | "type": "long" 271 | }, 272 | "memory_size_in_bytes": { 273 | "type": "long" 274 | }, 275 | "miss_count": { 276 | "type": "long" 277 | } 278 | } 279 | }, 280 | "refresh": { 281 | "properties": { 282 | "total_time_in_millis": { 283 | "type": "long" 284 | } 285 | } 286 | }, 287 | "request_cache": { 288 | "properties": { 289 | "evictions": { 290 | "type": "long" 291 | }, 292 | "hit_count": { 293 | "type": "long" 294 | }, 295 | "memory_size_in_bytes": { 296 | "type": "long" 297 | }, 298 | "miss_count": { 299 | "type": "long" 300 | } 301 | } 302 | }, 303 | "search": { 304 | "properties": { 305 | "query_time_in_millis": { 306 | "type": "long" 307 | }, 308 | "query_total": { 309 | "type": "long" 310 | } 311 | } 312 | }, 313 | "segments": { 314 | "properties": { 315 | "count": { 316 | "type": "integer" 317 | }, 318 | "doc_values_memory_in_bytes": { 319 | "type": "long" 320 | }, 321 | "fixed_bit_set_memory_in_bytes": { 322 | "type": "long" 323 | }, 324 | "index_writer_memory_in_bytes": { 325 | "type": "long" 326 | }, 327 | "memory_in_bytes": { 328 | "type": "long" 329 | }, 330 | "norms_memory_in_bytes": { 331 | "type": "long" 332 | }, 333 | "points_memory_in_bytes": { 334 | "type": "long" 335 | }, 336 | "stored_fields_memory_in_bytes": { 337 | "type": "long" 338 | }, 339 | "term_vectors_memory_in_bytes": { 340 | "type": "long" 341 | }, 342 | "terms_memory_in_bytes": { 343 | "type": "long" 344 | }, 345 | "version_map_memory_in_bytes": { 346 | "type": "long" 347 | } 348 | } 349 | }, 350 | "store": { 351 | "properties": { 352 | "size_in_bytes": { 353 | "type": "long" 354 | } 355 | } 356 | } 357 | } 358 | }, 359 | "total": { 360 | "properties": { 361 | "docs": { 362 | "properties": { 363 | "count": { 364 | "type": "long" 365 | } 366 | } 367 | }, 368 | "fielddata": { 369 | "properties": { 370 | "evictions": { 371 | "type": "long" 372 | }, 373 | "memory_size_in_bytes": { 374 | "type": "long" 375 | } 376 | } 377 | }, 378 | "indexing": { 379 | "properties": { 380 | "index_time_in_millis": { 381 | "type": "long" 382 | }, 383 | "index_total": { 384 | "type": "long" 385 | }, 386 | "throttle_time_in_millis": { 387 | "type": "long" 388 | } 389 | } 390 | }, 391 | "merges": { 392 | "properties": { 393 | "total_size_in_bytes": { 394 | "type": "long" 395 | } 396 | } 397 | }, 398 | "query_cache": { 399 | "properties": { 400 | "evictions": { 401 | "type": "long" 402 | }, 403 | "hit_count": { 404 | "type": "long" 405 | }, 406 | "memory_size_in_bytes": { 407 | "type": "long" 408 | }, 409 | "miss_count": { 410 | "type": "long" 411 | } 412 | } 413 | }, 414 | "refresh": { 415 | "properties": { 416 | "total_time_in_millis": { 417 | "type": "long" 418 | } 419 | } 420 | }, 421 | "request_cache": { 422 | "properties": { 423 | "evictions": { 424 | "type": "long" 425 | }, 426 | "hit_count": { 427 | "type": "long" 428 | }, 429 | "memory_size_in_bytes": { 430 | "type": "long" 431 | }, 432 | "miss_count": { 433 | "type": "long" 434 | } 435 | } 436 | }, 437 | "search": { 438 | "properties": { 439 | "query_time_in_millis": { 440 | "type": "long" 441 | }, 442 | "query_total": { 443 | "type": "long" 444 | } 445 | } 446 | }, 447 | "segments": { 448 | "properties": { 449 | "count": { 450 | "type": "integer" 451 | }, 452 | "doc_values_memory_in_bytes": { 453 | "type": "long" 454 | }, 455 | "fixed_bit_set_memory_in_bytes": { 456 | "type": "long" 457 | }, 458 | "index_writer_memory_in_bytes": { 459 | "type": "long" 460 | }, 461 | "memory_in_bytes": { 462 | "type": "long" 463 | }, 464 | "norms_memory_in_bytes": { 465 | "type": "long" 466 | }, 467 | "points_memory_in_bytes": { 468 | "type": "long" 469 | }, 470 | "stored_fields_memory_in_bytes": { 471 | "type": "long" 472 | }, 473 | "term_vectors_memory_in_bytes": { 474 | "type": "long" 475 | }, 476 | "terms_memory_in_bytes": { 477 | "type": "long" 478 | }, 479 | "version_map_memory_in_bytes": { 480 | "type": "long" 481 | } 482 | } 483 | }, 484 | "store": { 485 | "properties": { 486 | "size_in_bytes": { 487 | "type": "long" 488 | } 489 | } 490 | } 491 | } 492 | } 493 | } 494 | }, 495 | "indices_stats": { 496 | "properties": { 497 | "_all": { 498 | "properties": { 499 | "primaries": { 500 | "properties": { 501 | "docs": { 502 | "properties": { 503 | "count": { 504 | "type": "long" 505 | } 506 | } 507 | }, 508 | "indexing": { 509 | "properties": { 510 | "index_time_in_millis": { 511 | "type": "long" 512 | }, 513 | "index_total": { 514 | "type": "long" 515 | } 516 | } 517 | }, 518 | "search": { 519 | "properties": { 520 | "query_time_in_millis": { 521 | "type": "long" 522 | }, 523 | "query_total": { 524 | "type": "long" 525 | } 526 | } 527 | } 528 | } 529 | }, 530 | "total": { 531 | "properties": { 532 | "docs": { 533 | "properties": { 534 | "count": { 535 | "type": "long" 536 | } 537 | } 538 | }, 539 | "indexing": { 540 | "properties": { 541 | "index_time_in_millis": { 542 | "type": "long" 543 | }, 544 | "index_total": { 545 | "type": "long" 546 | } 547 | } 548 | }, 549 | "search": { 550 | "properties": { 551 | "query_time_in_millis": { 552 | "type": "long" 553 | }, 554 | "query_total": { 555 | "type": "long" 556 | } 557 | } 558 | } 559 | } 560 | } 561 | } 562 | } 563 | } 564 | }, 565 | "interval_ms": { 566 | "type": "long" 567 | }, 568 | "job_stats": { 569 | "properties": { 570 | "data_counts": { 571 | "properties": { 572 | "bucket_count": { 573 | "type": "long" 574 | }, 575 | "earliest_record_timestamp": { 576 | "type": "date" 577 | }, 578 | "empty_bucket_count": { 579 | "type": "long" 580 | }, 581 | "input_bytes": { 582 | "type": "long" 583 | }, 584 | "latest_record_timestamp": { 585 | "type": "date" 586 | }, 587 | "processed_record_count": { 588 | "type": "long" 589 | }, 590 | "sparse_bucket_count": { 591 | "type": "long" 592 | } 593 | } 594 | }, 595 | "job_id": { 596 | "type": "keyword" 597 | }, 598 | "model_size_stats": { 599 | "properties": { 600 | "bucket_allocation_failures_count": { 601 | "type": "long" 602 | }, 603 | "model_bytes": { 604 | "type": "long" 605 | } 606 | } 607 | }, 608 | "node": { 609 | "properties": { 610 | "id": { 611 | "type": "keyword" 612 | } 613 | } 614 | }, 615 | "state": { 616 | "type": "keyword" 617 | } 618 | } 619 | }, 620 | "node_stats": { 621 | "properties": { 622 | "fs": { 623 | "properties": { 624 | "data": { 625 | "properties": { 626 | "spins": { 627 | "type": "boolean" 628 | } 629 | } 630 | }, 631 | "io_stats": { 632 | "properties": { 633 | "devices": { 634 | "properties": { 635 | "operations": { 636 | "type": "long" 637 | }, 638 | "read_kilobytes": { 639 | "type": "long" 640 | }, 641 | "read_operations": { 642 | "type": "long" 643 | }, 644 | "write_kilobytes": { 645 | "type": "long" 646 | }, 647 | "write_operations": { 648 | "type": "long" 649 | } 650 | } 651 | }, 652 | "total": { 653 | "properties": { 654 | "operations": { 655 | "type": "long" 656 | }, 657 | "read_kilobytes": { 658 | "type": "long" 659 | }, 660 | "read_operations": { 661 | "type": "long" 662 | }, 663 | "write_kilobytes": { 664 | "type": "long" 665 | }, 666 | "write_operations": { 667 | "type": "long" 668 | }, 669 | "read_ops_current": { 670 | "type": "long" 671 | }, 672 | "write_ops_current": { 673 | "type": "long" 674 | } 675 | } 676 | } 677 | 678 | } 679 | }, 680 | "total": { 681 | "properties": { 682 | "available_in_bytes": { 683 | "type": "long" 684 | }, 685 | "free_in_bytes": { 686 | "type": "long" 687 | }, 688 | "spins":{ 689 | "type": "boolean" 690 | }, 691 | "total_in_bytes": { 692 | "type": "long" 693 | } 694 | 695 | } 696 | } 697 | } 698 | }, 699 | "indices": { 700 | "properties": { 701 | "completion" : { 702 | "properties":{ 703 | "size_in_bytes" : { 704 | "type": "long" 705 | } 706 | } 707 | }, 708 | "docs": { 709 | "properties": { 710 | "deleted": { 711 | "type": "long" 712 | }, 713 | "count": { 714 | "type": "long" 715 | } 716 | } 717 | }, 718 | "fielddata": { 719 | "properties": { 720 | "evictions": { 721 | "type": "long" 722 | }, 723 | "memory_size_in_bytes": { 724 | "type": "long" 725 | } 726 | } 727 | }, 728 | "flush" : { 729 | "properties":{ 730 | "total" : { 731 | "type": "long" 732 | }, 733 | "total_time_in_millis" : { 734 | "type": "long" 735 | } 736 | } 737 | }, 738 | "get" : { 739 | "properties":{ 740 | "current" : { 741 | "type": "long" 742 | }, 743 | "exists_time_in_millis" : { 744 | "type": "long" 745 | }, 746 | "exists_total" : { 747 | "type": "long" 748 | }, 749 | "time_in_millis" : { 750 | "type": "long" 751 | }, 752 | "missing_time_in_millis" : { 753 | "type": "long" 754 | }, 755 | "missing_total" : { 756 | "type": "long" 757 | }, 758 | "total" : { 759 | "type": "long" 760 | } 761 | } 762 | }, 763 | "indexing": { 764 | "properties": { 765 | "delete_current" : { 766 | "type": "long" 767 | }, 768 | "delete_time_in_millis" : { 769 | "type": "long" 770 | }, 771 | "delete_total" : { 772 | "type": "long" 773 | }, 774 | "index_current" : { 775 | "type": "long" 776 | }, 777 | "index_failed" : { 778 | "type": "long" 779 | }, 780 | "index_time_in_millis": { 781 | "type": "long" 782 | }, 783 | "index_total": { 784 | "type": "long" 785 | }, 786 | "is_throttled" : { 787 | "type": "boolean" 788 | }, 789 | "noop_update_total" : { 790 | "type": "long" 791 | }, 792 | "throttle_time_in_millis": { 793 | "type": "long" 794 | } 795 | } 796 | }, 797 | "merges" : { 798 | "properties":{ 799 | "current" : { 800 | "type": "long" 801 | }, 802 | "current_docs" : { 803 | "type": "long" 804 | }, 805 | "current_size_in_bytes" : { 806 | "type": "long" 807 | }, 808 | "total" : { 809 | "type": "long" 810 | }, 811 | 812 | "total_auto_throttle_in_bytes" : { 813 | "type": "long" 814 | }, 815 | "total_docs" : { 816 | "type": "long" 817 | }, 818 | "total_size_in_bytes" : { 819 | "type": "long" 820 | }, 821 | "total_time_in_millis" : { 822 | "type": "long" 823 | }, 824 | 825 | "total_stopped_time_in_millis" : { 826 | "type": "long" 827 | }, 828 | "total_throttled_time_in_millis" : { 829 | "type": "long" 830 | } 831 | 832 | 833 | } 834 | }, 835 | "query_cache": { 836 | "properties": { 837 | "cache_count" : { 838 | "type": "long" 839 | }, 840 | "cache_size" : { 841 | "type": "long" 842 | }, 843 | "evictions": { 844 | "type": "long" 845 | }, 846 | "hit_count": { 847 | "type": "long" 848 | }, 849 | "memory_size_in_bytes": { 850 | "type": "long" 851 | }, 852 | "miss_count": { 853 | "type": "long" 854 | }, 855 | "total_count" : { 856 | "type": "long" 857 | } 858 | } 859 | }, 860 | "recovery" : { 861 | "properties":{ 862 | "current_as_source" : { 863 | "type": "long" 864 | }, 865 | "current_as_target" : { 866 | "type": "long" 867 | }, 868 | "throttle_time_in_millis" : { 869 | "type": "long" 870 | } 871 | } 872 | }, 873 | "refresh" : { 874 | "properties":{ 875 | "listeners" : { 876 | "type": "long" 877 | }, 878 | "total" : { 879 | "type": "long" 880 | }, 881 | "total_time_in_millis" : { 882 | "type": "long" 883 | } 884 | } 885 | }, 886 | "request_cache": { 887 | "properties": { 888 | "evictions": { 889 | "type": "long" 890 | }, 891 | "hit_count": { 892 | "type": "long" 893 | }, 894 | "memory_size_in_bytes": { 895 | "type": "long" 896 | }, 897 | "miss_count": { 898 | "type": "long" 899 | } 900 | } 901 | }, 902 | "search": { 903 | "properties": { 904 | "fetch_current" : { 905 | "type": "long" 906 | }, 907 | "fetch_time_in_millis" : { 908 | "type": "long" 909 | }, 910 | "fetch_total" : { 911 | "type": "long" 912 | }, 913 | "open_contexts" : { 914 | "type": "long" 915 | }, 916 | "query_current" : { 917 | "type": "long" 918 | }, 919 | "query_time_in_millis": { 920 | "type": "long" 921 | }, 922 | "query_total": { 923 | "type": "long" 924 | }, 925 | "scroll_current" : { 926 | "type": "long" 927 | }, 928 | "scroll_time_in_millis" : { 929 | "type": "long" 930 | }, 931 | "scroll_total" : { 932 | "type": "long" 933 | }, 934 | "suggest_current" : { 935 | "type": "long" 936 | }, 937 | "suggest_time_in_millis" : { 938 | "type": "long" 939 | }, 940 | "suggest_total" : { 941 | "type": "long" 942 | }, 943 | "query_avg_time": { 944 | "type": "float" 945 | }, 946 | "query_time_current": { 947 | "type": "long" 948 | }, 949 | "query_count_delta": { 950 | "type": "long" 951 | } 952 | } 953 | }, 954 | "segments": { 955 | "properties": { 956 | "count": { 957 | "type": "integer" 958 | }, 959 | "doc_values_memory_in_bytes": { 960 | "type": "long" 961 | }, 962 | "fixed_bit_set_memory_in_bytes": { 963 | "type": "long" 964 | }, 965 | "index_writer_memory_in_bytes": { 966 | "type": "long" 967 | }, 968 | "max_unsafe_auto_id_timestamp" : { 969 | "type": "long" 970 | }, 971 | "memory_in_bytes": { 972 | "type": "long" 973 | }, 974 | "norms_memory_in_bytes": { 975 | "type": "long" 976 | }, 977 | "points_memory_in_bytes": { 978 | "type": "long" 979 | }, 980 | "stored_fields_memory_in_bytes": { 981 | "type": "long" 982 | }, 983 | "term_vectors_memory_in_bytes": { 984 | "type": "long" 985 | }, 986 | "terms_memory_in_bytes": { 987 | "type": "long" 988 | }, 989 | "version_map_memory_in_bytes": { 990 | "type": "long" 991 | } 992 | 993 | } 994 | }, 995 | "store" : { 996 | "properties":{ 997 | "size_in_bytes" : { 998 | "type": "long" 999 | }, 1000 | "throttle_time_in_millis" : { 1001 | "type": "long" 1002 | } 1003 | } 1004 | }, 1005 | "translog" : { 1006 | "properties":{ 1007 | "operations" : { 1008 | "type": "long" 1009 | }, 1010 | "size_in_bytes" : { 1011 | "type": "long" 1012 | } 1013 | } 1014 | }, 1015 | "warmer" : { 1016 | "properties":{ 1017 | "current" : { 1018 | "type": "long" 1019 | }, 1020 | "total" : { 1021 | "type": "long" 1022 | }, 1023 | "total_time_in_millis" : { 1024 | "type": "long" 1025 | } 1026 | } 1027 | } 1028 | } 1029 | }, 1030 | "http":{ 1031 | "properties":{ 1032 | "current_open":{ 1033 | "type": "long" 1034 | }, 1035 | "total_opened":{ 1036 | "type":"long" 1037 | } 1038 | } 1039 | }, 1040 | "jvm": { 1041 | "properties": { 1042 | "gc": { 1043 | "properties": { 1044 | "collectors": { 1045 | "properties": { 1046 | "old": { 1047 | "properties": { 1048 | "collection_count": { 1049 | "type": "long" 1050 | }, 1051 | "collection_time_in_millis": { 1052 | "type": "long" 1053 | } 1054 | } 1055 | }, 1056 | "young": { 1057 | "properties": { 1058 | "collection_count": { 1059 | "type": "long" 1060 | }, 1061 | "collection_time_in_millis": { 1062 | "type": "long" 1063 | } 1064 | } 1065 | } 1066 | } 1067 | } 1068 | } 1069 | }, 1070 | "mem": { 1071 | "properties": { 1072 | "heap_max_in_bytes": { 1073 | "type": "long" 1074 | }, 1075 | "heap_used_in_bytes": { 1076 | "type": "long" 1077 | }, 1078 | "heap_used_percent": { 1079 | "type": "half_float" 1080 | }, 1081 | "non_heap_committed_in_bytes" : { 1082 | "type": "long" 1083 | }, 1084 | "heap_committed_in_bytes" : { 1085 | "type": "long" 1086 | }, 1087 | "non_heap_used_in_bytes" : { 1088 | "type": "long" 1089 | } 1090 | } 1091 | }, 1092 | "classes" : { 1093 | "properties":{ 1094 | "current_loaded_count" : { 1095 | "type": "long" 1096 | }, 1097 | "total_unloaded_count" : { 1098 | "type": "long" 1099 | }, 1100 | "total_loaded_count" : { 1101 | "type": "long" 1102 | } 1103 | } 1104 | }, 1105 | "threads" : { 1106 | "properties":{ 1107 | "count" : { 1108 | "type": "long" 1109 | }, 1110 | "peak_count" : { 1111 | "type": "long" 1112 | } 1113 | 1114 | } 1115 | } 1116 | } 1117 | }, 1118 | "mlockall": { 1119 | "type": "boolean" 1120 | }, 1121 | "node_id": { 1122 | "type": "keyword" 1123 | }, 1124 | "node_master": { 1125 | "type": "boolean" 1126 | }, 1127 | "os": { 1128 | "properties": { 1129 | "load_average" : { 1130 | "type": "half_float" 1131 | }, 1132 | "cgroup": { 1133 | "properties": { 1134 | "cpu": { 1135 | "properties": { 1136 | "cfs_quota_micros": { 1137 | "type": "long" 1138 | }, 1139 | "control_group": { 1140 | "type": "keyword" 1141 | }, 1142 | "stat": { 1143 | "properties": { 1144 | "number_of_elapsed_periods": { 1145 | "type": "long" 1146 | }, 1147 | "number_of_times_throttled": { 1148 | "type": "long" 1149 | }, 1150 | "time_throttled_nanos": { 1151 | "type": "long" 1152 | } 1153 | } 1154 | } 1155 | } 1156 | }, 1157 | "cpuacct": { 1158 | "properties": { 1159 | "control_group": { 1160 | "type": "keyword" 1161 | }, 1162 | "usage_nanos": { 1163 | "type": "long" 1164 | } 1165 | } 1166 | }, 1167 | "memory": { 1168 | "properties": { 1169 | "control_group": { 1170 | "type": "keyword" 1171 | }, 1172 | "limit_in_bytes": { 1173 | "type": "keyword" 1174 | }, 1175 | "usage_in_bytes": { 1176 | "type": "keyword" 1177 | } 1178 | } 1179 | } 1180 | } 1181 | }, 1182 | "cpu": { 1183 | "properties": { 1184 | "load_average": { 1185 | "properties": { 1186 | "15m": { 1187 | "type": "half_float" 1188 | }, 1189 | "1m": { 1190 | "type": "half_float" 1191 | }, 1192 | "5m": { 1193 | "type": "half_float" 1194 | } 1195 | } 1196 | }, 1197 | "percent":{ 1198 | "type": "half_float" 1199 | } 1200 | } 1201 | }, 1202 | "mem" : { 1203 | "properties":{ 1204 | "free_in_bytes" : { 1205 | "type": "long" 1206 | }, 1207 | "free_percent" : { 1208 | "type": "half_float" 1209 | }, 1210 | "total_in_bytes" : { 1211 | "type": "long" 1212 | }, 1213 | "used_percent" : { 1214 | "type": "half_float" 1215 | }, 1216 | "used_in_bytes" : { 1217 | "type": "long" 1218 | } 1219 | } 1220 | }, 1221 | "swap" : { 1222 | "properties":{ 1223 | "free_in_bytes" : { 1224 | "type": "long" 1225 | }, 1226 | "total_in_bytes" : { 1227 | "type": "long" 1228 | }, 1229 | "used_in_bytes" : { 1230 | "type": "long" 1231 | } 1232 | } 1233 | } 1234 | } 1235 | }, 1236 | "process": { 1237 | "properties": { 1238 | "cpu": { 1239 | "properties": { 1240 | "percent": { 1241 | "type": "half_float" 1242 | } 1243 | } 1244 | }, 1245 | "max_file_descriptors": { 1246 | "type": "long" 1247 | }, 1248 | "open_file_descriptors": { 1249 | "type": "long" 1250 | } 1251 | } 1252 | }, 1253 | "script":{ 1254 | "properties":{ 1255 | "cache_evictions":{ 1256 | "type": "long" 1257 | }, 1258 | "compilations":{ 1259 | "type":"long" 1260 | } 1261 | } 1262 | }, 1263 | "transport":{ 1264 | "properties":{ 1265 | "rx_count":{ 1266 | "type":"long" 1267 | }, 1268 | "rx_size_in_bytes":{ 1269 | "type": "long" 1270 | }, 1271 | "server_open":{ 1272 | "type":"long" 1273 | }, 1274 | "tx_size_in_bytes":{ 1275 | "type":"long" 1276 | }, 1277 | "tx_count":{ 1278 | "type":"long" 1279 | } 1280 | } 1281 | }, 1282 | "thread_pool": { 1283 | "properties": { 1284 | "bulk": { 1285 | "properties": { 1286 | "queue": { 1287 | "type": "integer" 1288 | }, 1289 | "rejected": { 1290 | "type": "long" 1291 | }, 1292 | "threads": { 1293 | "type": "integer" 1294 | } 1295 | } 1296 | }, 1297 | "generic": { 1298 | "properties": { 1299 | "queue": { 1300 | "type": "integer" 1301 | }, 1302 | "rejected": { 1303 | "type": "long" 1304 | }, 1305 | "threads": { 1306 | "type": "integer" 1307 | } 1308 | } 1309 | }, 1310 | "get": { 1311 | "properties": { 1312 | "queue": { 1313 | "type": "integer" 1314 | }, 1315 | "rejected": { 1316 | "type": "long" 1317 | }, 1318 | "threads": { 1319 | "type": "integer" 1320 | } 1321 | } 1322 | }, 1323 | "index": { 1324 | "properties": { 1325 | "queue": { 1326 | "type": "integer" 1327 | }, 1328 | "rejected": { 1329 | "type": "long" 1330 | }, 1331 | "threads": { 1332 | "type": "integer" 1333 | } 1334 | } 1335 | }, 1336 | "management": { 1337 | "properties": { 1338 | "queue": { 1339 | "type": "integer" 1340 | }, 1341 | "rejected": { 1342 | "type": "long" 1343 | }, 1344 | "threads": { 1345 | "type": "integer" 1346 | } 1347 | } 1348 | }, 1349 | "search": { 1350 | "properties": { 1351 | "queue": { 1352 | "type": "integer" 1353 | }, 1354 | "rejected": { 1355 | "type": "long" 1356 | }, 1357 | "threads": { 1358 | "type": "integer" 1359 | } 1360 | } 1361 | }, 1362 | "watcher": { 1363 | "properties": { 1364 | "queue": { 1365 | "type": "integer" 1366 | }, 1367 | "rejected": { 1368 | "type": "long" 1369 | }, 1370 | "threads": { 1371 | "type": "integer" 1372 | } 1373 | } 1374 | }, 1375 | "write": { 1376 | "properties": { 1377 | "queue": { 1378 | "type": "integer" 1379 | }, 1380 | "rejected": { 1381 | "type": "long" 1382 | } 1383 | } 1384 | } 1385 | } 1386 | } 1387 | } 1388 | }, 1389 | "shard": { 1390 | "properties": { 1391 | "index": { 1392 | "type": "keyword" 1393 | }, 1394 | "node": { 1395 | "type": "keyword" 1396 | }, 1397 | "primary": { 1398 | "type": "boolean" 1399 | }, 1400 | "relocating_node": { 1401 | "type": "keyword" 1402 | }, 1403 | "shard": { 1404 | "type": "long" 1405 | }, 1406 | "state": { 1407 | "type": "keyword" 1408 | } 1409 | } 1410 | }, 1411 | "source_node": { 1412 | "properties": { 1413 | "host": { 1414 | "type": "keyword" 1415 | }, 1416 | "ip": { 1417 | "type": "keyword" 1418 | }, 1419 | "name": { 1420 | "type": "keyword" 1421 | }, 1422 | "timestamp": { 1423 | "type": "date", 1424 | "format": "date_time" 1425 | }, 1426 | "transport_address": { 1427 | "type": "keyword" 1428 | }, 1429 | "roles": { 1430 | "type": "keyword" 1431 | }, 1432 | "uuid": { 1433 | "type": "keyword" 1434 | } 1435 | } 1436 | }, 1437 | "state_uuid": { 1438 | "type": "keyword" 1439 | }, 1440 | "timestamp": { 1441 | "type": "date", 1442 | "format": "date_time" 1443 | }, 1444 | "type": { 1445 | "type": "keyword" 1446 | }, 1447 | "cluster_name": { 1448 | "type": "keyword" 1449 | } 1450 | } 1451 | } 1452 | } 1453 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | requests 3 | dictor -------------------------------------------------------------------------------- /test-cluster/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ES_VERSION 2 | FROM docker.elastic.co/elasticsearch/elasticsearch:$ES_VERSION 3 | EXPOSE 9200 4 | ARG ES_VERSION 5 | ADD elasticsearch.yml /usr/share/elasticsearch/config/ 6 | USER root 7 | RUN chown elasticsearch:elasticsearch config/elasticsearch.yml 8 | USER elasticsearch 9 | #WORKDIR /usr/share/elasticsearch 10 | -------------------------------------------------------------------------------- /test-cluster/README.md: -------------------------------------------------------------------------------- 1 | Running the test cluster is done via: 2 | 3 | ```bash 4 | export ES_VERSION=5.5.0 5 | docker build --build-arg ES_VERSION=$ES_VERSION -t code972/elasticsearch-test-cluster:v$ES_VERSION test-cluster 6 | ``` 7 | 8 | Then run the cluster using: 9 | 10 | ```bash 11 | docker run -p 0.0.0.0:9200:9200 -t code972/elasticsearch-test-cluster:v$ES_VERSION 12 | ``` 13 | -------------------------------------------------------------------------------- /test-cluster/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | discovery.zen.minimum_master_nodes: 1 2 | xpack.security.enabled: false 3 | network.host: 0.0.0.0 4 | --------------------------------------------------------------------------------