├── .dockerignore ├── .gitattributes ├── .github ├── CODEOWNERS ├── pull_request_template.md └── workflows │ ├── ci.yml │ ├── publish-release.yml │ └── python-tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── HEADER.txt ├── LICENSE.txt ├── NOTICE.template ├── NOTICE.txt ├── README.md ├── docker-build.sh ├── docker-run.sh ├── pom.xml ├── push-docker.sh ├── renovate.json ├── scripts ├── diagnostics.bat ├── diagnostics.sh ├── export-monitoring.bat ├── export-monitoring.sh ├── import-monitoring.bat ├── import-monitoring.sh ├── scrub.bat ├── scrub.sh └── share_ad_job_state │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── export_model_snapshot.py │ ├── import_model_snapshot.py │ ├── noxfile.py │ ├── poetry.lock │ ├── pyproject.toml │ └── tests │ ├── __init__.py │ ├── test_export_model_snapshot.py │ └── test_import_model_snapshot.py └── src ├── main ├── assembly │ └── assembly.xml ├── java │ └── co │ │ └── elastic │ │ └── support │ │ ├── BaseConfig.java │ │ ├── BaseInputs.java │ │ ├── BaseService.java │ │ ├── Constants.java │ │ ├── diagnostics │ │ ├── DiagConfig.java │ │ ├── DiagnosticApp.java │ │ ├── DiagnosticException.java │ │ ├── DiagnosticInputs.java │ │ ├── DiagnosticService.java │ │ ├── JavaPlatform.java │ │ ├── ProcessProfile.java │ │ ├── ShowHelpException.java │ │ ├── chain │ │ │ ├── Command.java │ │ │ ├── DiagnosticChainExec.java │ │ │ └── DiagnosticContext.java │ │ └── commands │ │ │ ├── BaseQuery.java │ │ │ ├── CheckDiagnosticVersion.java │ │ │ ├── CheckElasticsearchVersion.java │ │ │ ├── CheckKibanaVersion.java │ │ │ ├── CheckPlatformDetails.java │ │ │ ├── CheckUserAuthLevel.java │ │ │ ├── CollectDockerInfo.java │ │ │ ├── CollectKibanaLogs.java │ │ │ ├── CollectLogs.java │ │ │ ├── CollectSystemCalls.java │ │ │ ├── GenerateDiagnosticManifest.java │ │ │ ├── GenerateLogstashDiagnostics.java │ │ │ ├── GenerateManifest.java │ │ │ ├── KibanaGetDetails.java │ │ │ ├── RetrieveSystemDigest.java │ │ │ ├── RunClusterQueries.java │ │ │ ├── RunKibanaQueries.java │ │ │ └── RunLogstashQueries.java │ │ ├── monitoring │ │ ├── MonitoringExportApp.java │ │ ├── MonitoringExportConfig.java │ │ ├── MonitoringExportInputs.java │ │ ├── MonitoringExportService.java │ │ ├── MonitoringImportApp.java │ │ ├── MonitoringImportConfig.java │ │ ├── MonitoringImportInputs.java │ │ ├── MonitoringImportProcessor.java │ │ └── MonitoringImportService.java │ │ ├── rest │ │ ├── ElasticRestClientInputs.java │ │ ├── ElasticRestClientService.java │ │ ├── RestClient.java │ │ ├── RestEntry.java │ │ ├── RestEntryConfig.java │ │ └── RestResult.java │ │ ├── scrub │ │ ├── ScrubApp.java │ │ ├── ScrubConfig.java │ │ ├── ScrubInputs.java │ │ ├── ScrubProcessor.java │ │ ├── ScrubService.java │ │ ├── ScrubTask.java │ │ ├── ScrubTokenEntry.java │ │ └── TokenGenerator.java │ │ └── util │ │ ├── ArchiveEntryProcessor.java │ │ ├── ArchiveUtils.java │ │ ├── FileTaskEntry.java │ │ ├── JsonYamlUtils.java │ │ ├── LocalSystem.java │ │ ├── RemoteSystem.java │ │ ├── RemoteUserInfo.java │ │ ├── ResourceCache.java │ │ ├── SystemCommand.java │ │ ├── SystemProperties.java │ │ ├── SystemUtils.java │ │ ├── TaskEntry.java │ │ ├── TextIOManager.java │ │ ├── UrlUtils.java │ │ └── ZipFileTaskEntry.java └── resources │ ├── diags.yml │ ├── elastic-rest.yml │ ├── kibana-rest.yml │ ├── log4j2.xml │ ├── logstash-diagnostic-templates │ └── flow_metrics.html.ftlh │ ├── logstash-rest.yml │ ├── monitoring-extract │ ├── cluster_id_check.json │ ├── cluster_ids.json │ ├── general.json │ ├── index_all.json │ ├── index_stats.json │ ├── metricbeat.json │ └── templates │ │ ├── metricbeat-system-diag.json │ │ ├── monitoring-es-diag.json │ │ └── monitoring-logstash-diag.json │ ├── monitoring-rest.yml │ └── scrub.yml └── test ├── java └── co │ └── elastic │ └── support │ ├── diagnostics │ ├── TestDiagnosticService.java │ └── commands │ │ ├── TestCheckKibanaVersion.java │ │ └── TestKibanaGetDetails.java │ └── rest │ ├── TestRestConfigFileValidity.java │ └── TestRestExecCalls.java └── resources ├── darwin-process-list.txt ├── diags-test.yml ├── linux-process-list.txt ├── log4j2-test.xml ├── proc └── 1 │ ├── cgroup-no │ └── cgroup-yes ├── rest-diags.yml └── win-process-list.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | # Build files 2 | build 3 | diagnostic-output 4 | target 5 | 6 | # Support files 7 | .github 8 | .idea 9 | .vscode 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Files default to Unix line endings (\n) 2 | * text eol=lf 3 | 4 | # Windows line endings (\r\n) 5 | *.bat text eol=crlf 6 | *.cmd text eol=crlf 7 | *.ps1 text eol=crlf 8 | 9 | *.zip binary 10 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # used for branch protection 2 | * @pickypg 3 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ### Checklist 2 | 3 | - [ ] I have verified that the APIs in this pull request do not return sensitive data 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Java CI with Maven 10 | 11 | on: 12 | push: 13 | branches: [ "main" ] 14 | pull_request: 15 | branches: [ "main" ] 16 | 17 | jobs: 18 | build: 19 | 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 24 | 25 | - name: Set up JDK 23 26 | uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 27 | with: 28 | java-version: '23' 29 | distribution: 'zulu' 30 | cache: maven 31 | 32 | - name: Maven test 33 | run: mvn test --batch-mode 34 | -------------------------------------------------------------------------------- /.github/workflows/publish-release.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a package using Maven and then publish it to GitHub packages when a release is created 2 | # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path 3 | 4 | name: Maven Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: ubuntu-latest 14 | permissions: 15 | contents: write 16 | packages: write 17 | 18 | steps: 19 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 20 | 21 | - name: Set up JDK 23 22 | uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 23 | with: 24 | java-version: '23' 25 | distribution: 'zulu' 26 | # server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 27 | # settings-path: ${{ github.workspace }} # location for the settings.xml file 28 | 29 | - name: Build with Maven 30 | run: mvn -B deploy 31 | 32 | - name: Add output files to the Github release 33 | shell: bash 34 | run: | 35 | gh release upload $TAG target/diagnostics-${TAG:1}-dist.zip target/diagnostics-${TAG:1}-dist.zip.sha256 36 | env: 37 | GITHUB_TOKEN: ${{ github.TOKEN }} 38 | TAG: ${{ github.event.release.tag_name }} 39 | 40 | # - name: Publish to GitHub Packages Apache Maven 41 | # run: mvn deploy -s $GITHUB_WORKSPACE/settings.xml 42 | # env: 43 | # GITHUB_TOKEN: ${{ github.token }} 44 | -------------------------------------------------------------------------------- /.github/workflows/python-tests.yml: -------------------------------------------------------------------------------- 1 | name: Share AD Job State Tests 2 | on: pull_request 3 | jobs: 4 | tests: 5 | runs-on: ubuntu-latest 6 | defaults: 7 | run: 8 | shell: bash 9 | working-directory: scripts/share_ad_job_state 10 | # timeout after 5 minutes to limit costs in case of process leaking 11 | timeout-minutes: 5 12 | steps: 13 | - name: Checkout Repository 14 | uses: actions/checkout@v4 15 | - name: Install poetry 16 | run: pip install poetry==2.1.3 17 | - name: Set up Python 3 18 | uses: actions/setup-python@v5 19 | with: 20 | python-version: '3.12' 21 | cache: 'poetry' 22 | - name: Install nox 23 | run: pip install nox==2025.5.1 24 | - name: Run linting 25 | run: nox -s lint 26 | - name: Run tests 27 | run: | 28 | poetry install 29 | poetry run pytest --cov tests/ -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io 2 | 3 | ### Intellij ### 4 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 5 | 6 | *.iml 7 | 8 | ### Generated log files ### 9 | *.log 10 | 11 | ## Directory-based project format: 12 | .idea/ 13 | # if you remove the above rule, at least ignore the following: 14 | 15 | # User-specific stuff: 16 | # .idea/workspace.xml 17 | # .idea/tasks.xml 18 | # .idea/dictionaries 19 | 20 | # Sensitive or high-churn files: 21 | # .idea/dataSources.ids 22 | # .idea/dataSources.xml 23 | # .idea/sqlDataSources.xml 24 | # .idea/dynamic.xml 25 | # .idea/uiDesigner.xml 26 | 27 | # Gradle: 28 | # .idea/gradle.xml 29 | # .idea/libraries 30 | 31 | # Mongo Explorer plugin: 32 | # .idea/mongoSettings.xml 33 | 34 | ## File-based project format: 35 | *.ipr 36 | *.iws 37 | 38 | ## Plugin-specific files: 39 | 40 | # IntelliJ 41 | /out/ 42 | 43 | # mpeltonen/sbt-idea plugin 44 | .idea_modules/ 45 | 46 | # JIRA plugin 47 | atlassian-ide-plugin.xml 48 | 49 | # Crashlytics plugin (for Android Studio and IntelliJ) 50 | com_crashlytics_export_strings.xml 51 | crashlytics.properties 52 | crashlytics-build.properties 53 | 54 | 55 | ### Eclipse ### 56 | *.pydevproject 57 | .metadata 58 | .gradle 59 | bin/ 60 | tmp/ 61 | *.tmp 62 | *.bak 63 | *.swp 64 | *~.nib 65 | local.properties 66 | .settings/ 67 | .loadpath 68 | 69 | # Eclipse Core 70 | .project 71 | 72 | # External tool builders 73 | .externalToolBuilders/ 74 | 75 | # Locally stored "Eclipse launch configurations" 76 | *.launch 77 | 78 | # CDT-specific 79 | .cproject 80 | 81 | # JDT-specific (Eclipse Java Development Tools) 82 | .classpath 83 | 84 | # PDT-specific 85 | .buildpath 86 | 87 | # sbteclipse plugin 88 | .target 89 | 90 | # TeXlipse plugin 91 | .texlipse 92 | 93 | 94 | ### Maven ### 95 | target/ 96 | pom.xml.tag 97 | pom.xml.releaseBackup 98 | pom.xml.versionsBackup 99 | pom.xml.next 100 | release.properties 101 | dependency-reduced-pom.xml 102 | buildNumber.properties 103 | 104 | .DS_Store 105 | 106 | 107 | 108 | ./diagnostic-output 109 | ### VSCode 110 | .vscode/launch.json 111 | 112 | ### Temporary diags 113 | *.zip -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | This is a tool used by the Elastic Support team to collect the necessary data to troubleshoot problems. For new features or enhancements, open a github issue to discuss before submitting a pull request. 4 | 5 | ## Getting Started 6 | 7 | ### Using IntelliJ IDEA 8 | 9 | - Clone the project 10 | - In IntelliJ, use `New -> Project from Existing Sources...` 11 | - Select `Import project from external model` and use the `Maven` option 12 | - Use the default options for the Project 13 | - You will need to add the following to the POM file 14 | ```xml 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | ``` 25 | - Using the `Run` menu and select `Edit Configurations...`. Use the `+` to add an `Application` configuration. 26 | - Main class: `co.elastic.support.diagnostics.DiagnosticApp` 27 | - Program arguments (example): `-o ~/tmp/diag-output -h localhost -u elastic --passwordText changeme`. Put whatever arguments you would like to run the application with as default. 28 | 29 | ### Releasing to Maven Central 30 | 31 | In order to release the code to Maven Central, you must have a Sonatype account 32 | with the permissions to deploy to the `co.elastic` `groupId`. 33 | 34 | Once created, you will need to create or modify your Maven `settings.xml` 35 | (`~/.m2/settings.xml` is for global usage). Example: 36 | 37 | ```xml 38 | 39 | 40 | 41 | ossrh 42 | 43 | true 44 | 45 | 46 | 47 | 48 | 49 | ossrh 50 | your_sonatype_username 51 | your_xml_encoded_sonatype_password 52 | 53 | 54 | 55 | ``` 56 | 57 | Note: `ossrh` matches the `id` used in the `pom.xml`. You can use any version of 58 | `gpg` that you want. 59 | 60 | Once the `settings.xml` is setup, you can run 61 | 62 | ``` 63 | mvn clean deploy 64 | ``` 65 | 66 | This will deploy based on the version in the `pom.xml` file (`-SNAPSHOT` creates 67 | it in their snapshot repository, which you should always do before a real 68 | release). 69 | 70 | More detailed instructions can be found on 71 | [Sonatype's website](https://central.sonatype.org/publish/publish-maven/). 72 | 73 | Once deployed, you must release the library through 74 | [Sonatype's staging repository](https://oss.sonatype.org/#stagingRepositories), 75 | using your Sonatype credentials. You first "Close" the staged deployment, then 76 | "Release" it after it passes the validations from "Close". 77 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/wolfi/jdk:openjdk-23.0.2-r4-dev@sha256:ce56f09eaca0ab98d4a9aa5625f048d91e69a5955a42a7f21cc8af019ebe1423 AS builder 2 | 3 | ##################### 4 | # Install dev tools 5 | ##################### 6 | USER root 7 | 8 | # need to be root to be able to install maven 9 | RUN apk add --no-cache maven 10 | 11 | ##################### 12 | # Build code 13 | ##################### 14 | WORKDIR /build 15 | 16 | COPY ./ ./ 17 | 18 | RUN mvn package 19 | 20 | FROM docker.elastic.co/wolfi/jdk:openjdk-23.0.2-r4@sha256:cdea8b6002469f1f5275813cdfcc7cf8a05323dd77c4d8219acb7d43c12c51c0 AS runner 21 | 22 | ######################## 23 | # Prepare the code to run 24 | ######################## 25 | WORKDIR /support-diagnostics 26 | 27 | COPY --from=builder /build/scripts /support-diagnostics 28 | COPY --from=builder /build/target/lib /support-diagnostics/lib 29 | COPY --from=builder /build/target/diagnostics-*.jar /support-diagnostics/lib 30 | COPY --from=builder /build/src/main/resources /support-diagnostics/config 31 | -------------------------------------------------------------------------------- /HEADER.txt: -------------------------------------------------------------------------------- 1 | Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | or more contributor license agreements. Licensed under the Elastic License 3 | 2.0; you may not use this file except in compliance with the Elastic License 4 | 2.0. -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Elastic License 2.0 2 | 3 | URL: https://www.elastic.co/licensing/elastic-license 4 | 5 | ## Acceptance 6 | 7 | By using the software, you agree to all of the terms and conditions below. 8 | 9 | ## Copyright License 10 | 11 | The licensor grants you a non-exclusive, royalty-free, worldwide, 12 | non-sublicensable, non-transferable license to use, copy, distribute, make 13 | available, and prepare derivative works of the software, in each case subject to 14 | the limitations and conditions below. 15 | 16 | ## Limitations 17 | 18 | You may not provide the software to third parties as a hosted or managed 19 | service, where the service provides users with access to any substantial set of 20 | the features or functionality of the software. 21 | 22 | You may not move, change, disable, or circumvent the license key functionality 23 | in the software, and you may not remove or obscure any functionality in the 24 | software that is protected by the license key. 25 | 26 | You may not alter, remove, or obscure any licensing, copyright, or other notices 27 | of the licensor in the software. Any use of the licensor’s trademarks is subject 28 | to applicable law. 29 | 30 | ## Patents 31 | 32 | The licensor grants you a license, under any patent claims the licensor can 33 | license, or becomes able to license, to make, have made, use, sell, offer for 34 | sale, import and have imported the software, in each case subject to the 35 | limitations and conditions in this license. This license does not cover any 36 | patent claims that you cause to be infringed by modifications or additions to 37 | the software. If you or your company make any written claim that the software 38 | infringes or contributes to infringement of any patent, your patent license for 39 | the software granted under these terms ends immediately. If your company makes 40 | such a claim, your patent license ends immediately for work on behalf of your 41 | company. 42 | 43 | ## Notices 44 | 45 | You must ensure that anyone who gets a copy of any part of the software from you 46 | also gets a copy of these terms. 47 | 48 | If you modify the software, you must include in any modified copies of the 49 | software prominent notices stating that you have modified the software. 50 | 51 | ## No Other Rights 52 | 53 | These terms do not imply any licenses other than those expressly granted in 54 | these terms. 55 | 56 | ## Termination 57 | 58 | If you use the software in violation of these terms, such use is not licensed, 59 | and your licenses will automatically terminate. If the licensor provides you 60 | with a notice of your violation, and you cease all violation of this license no 61 | later than 30 days after you receive that notice, your licenses will be 62 | reinstated retroactively. However, if you violate these terms after such 63 | reinstatement, any additional violation of these terms will cause your licenses 64 | to terminate automatically and permanently. 65 | 66 | ## No Liability 67 | 68 | *As far as the law allows, the software comes as is, without any warranty or 69 | condition, and the licensor will not be liable to you for any damages arising 70 | out of these terms or the use or nature of the software, under any kind of 71 | legal claim.* 72 | 73 | ## Definitions 74 | 75 | The **licensor** is the entity offering these terms, and the **software** is the 76 | software the licensor makes available under these terms, including any portion 77 | of it. 78 | 79 | **you** refers to the individual or entity agreeing to these terms. 80 | 81 | **your company** is any legal entity, sole proprietorship, or other kind of 82 | organization that you work for, plus all organizations that have control over, 83 | are under the control of, or are under common control with that 84 | organization. **control** means ownership of substantially all the assets of an 85 | entity, or the power to direct its management and policies by vote, contract, or 86 | otherwise. Control can be direct or indirect. 87 | 88 | **your licenses** are all the licenses granted to you for the software under 89 | these terms. 90 | 91 | **use** means anything you do with the software requiring one of your licenses. 92 | 93 | **trademark** means trademarks, service marks, and similar rights. -------------------------------------------------------------------------------- /NOTICE.template: -------------------------------------------------------------------------------- 1 | Support Diagnostics Utilities 2 | Copyright 2014 Elasticsearch B.V. 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). 6 | 7 | ================================================================ 8 | Third party libraries used by the Support Diagnostics Utilities: 9 | ================================================================ 10 | 11 | #GENERATED_NOTICES# -------------------------------------------------------------------------------- /docker-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker build -f Dockerfile -t docker.elastic.co/support/diagnostics:latest . 4 | -------------------------------------------------------------------------------- /docker-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Sample script to set up an interactive session in the Docker diagnostic image created via the 5 | # docker-build.sh script. The -v volume setting sends the output from the diagnostic to 6 | # a directory named diagnostic-output under the same directory where the user is running 7 | # the script. Simply change the directory location to the left of the colon if you wish to 8 | # write to a different location. Be sure that this target folder has sufficient permissions 9 | # to create the output files. 10 | # 11 | 12 | docker run --network host -it -v ${PWD}/diagnostic-output:/diagnostic-output docker.elastic.co/support/diagnostics:latest sh 13 | -------------------------------------------------------------------------------- /push-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Pushes Support Diagnostic images to hosted Docker. 5 | # 6 | # 1) Pushes the image tagged `latest` (which should have been built via 7 | # `$ docker compose build`). 8 | # 2) If a version was passed in: 9 | # 2a) Add that version as a tag to latest. 10 | # 2b) Push that newly tagged version to hosted Docker. 11 | # 12 | # Example usages: 13 | # 1) Push latest 14 | # $ ./push-docker.sh 15 | # 2) Push latest and also tag as 9.1.1 16 | # $ ./push-docker.sh 9.1.1 17 | # 18 | 19 | IMAGE="docker.elastic.co/support/diagnostics" 20 | 21 | echo "$IMAGE" 22 | 23 | echo "Pushing latest" 24 | 25 | # NOTE: The pattern with ( set -x ; command ) makes it so that the command runs 26 | # in a sub-shell where the command is echo'd to the screen when run. We don't 27 | # want that mode permanently because the echo lines will duplicate in the console. 28 | ( set -x ; docker push "$IMAGE:latest" ) 29 | 30 | # If there is a version parameter passed in 31 | if [[ $# -eq 1 ]]; then 32 | VERSION=${1} 33 | 34 | echo "Tagging version $VERSION" 35 | ( set -x ; docker tag "$IMAGE:latest" "$IMAGE:$VERSION" ) 36 | 37 | echo "Pushing version $VERSION" 38 | ( set -x ; docker push "$IMAGE:$VERSION" ) 39 | fi 40 | 41 | echo "All done!" 42 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>elastic/renovate-config" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /scripts/diagnostics.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | SET scriptpath=%~dp0 5 | SET diagpath=%scriptpath:~0,-1% 6 | SET libpath=%diagpath%\lib\NUL 7 | 8 | IF NOT EXIST %libpath% ( 9 | ECHO Diagnostic executable not found: 10 | ECHO. 11 | ECHO Please make sure that you are running with the archive ending with 12 | ECHO '-dist.zip' in the name and not the one labeled 'Source code'. 13 | ECHO. 14 | ECHO Download at https://github.com/elastic/support-diagnostics/releases/latest 15 | EXIT /b 400 16 | ) 17 | 18 | set JAVA_EXEC=java 19 | if not defined JAVA_HOME ( 20 | set JAVA_EXEC=java 21 | echo No Java Home was found. Using current path. If execution fails please install Java and make sure it is in the search path or exposed via the JAVA_HOME environment variable. 22 | ) else ( 23 | echo JAVA_HOME found, using !JAVA_HOME! 24 | set JAVA_EXEC=!JAVA_HOME!\bin\java 25 | ) 26 | 27 | if defined DIAG_DEBUG ( 28 | set DIAG_DEBUG_OPTS=-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y 29 | ) 30 | 31 | if not defined DIAG_JAVA_OPTS ( 32 | set DIAG_JAVA_OPTS=-Xms2g -Xmx2g 33 | ) 34 | 35 | echo Using %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% for options. 36 | "%JAVA_EXEC%" %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% -cp %diagpath%\config;%diagpath%\lib\* co.elastic.support.diagnostics.DiagnosticApp %* 37 | 38 | endlocal 39 | -------------------------------------------------------------------------------- /scripts/diagnostics.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | scriptDir="$(cd -- "$(dirname -- "$0")" && pwd)" 4 | libDir="$scriptDir"'/lib' 5 | 6 | if [ ! -d "$libDir" ]; then 7 | echo "Diagnostic executable not found:" 8 | echo "" 9 | echo "Please make sure that you are running with the archive ending with" 10 | echo "'-dist.zip' in the name and not the one labeled 'Source code'." 11 | echo "" 12 | echo "Download at https://github.com/elastic/support-diagnostics/releases/latest" 13 | exit 4 14 | fi 15 | 16 | if [ -x "${JAVA_HOME}/bin/java" ]; then 17 | JAVA="${JAVA_HOME}/bin/java" 18 | else 19 | JAVA=`which java` 20 | fi 21 | 22 | echo "Using ${JAVA} as Java Runtime" 23 | 24 | if [ ! -x "$JAVA" ]; then 25 | echo "Could not find any executable java binary. Please install java in your PATH and/or set JAVA_HOME" 26 | exit 1 27 | fi 28 | 29 | [ "${DIAG_DEBUG}" != "" ] && export DIAG_DEBUG_OPTS="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y" 30 | 31 | [ "${DIAG_JAVA_OPTS}" = "" ] && export DIAG_JAVA_OPTS="-Xms2g -Xmx2g" 32 | 33 | echo "Using ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} for options." 34 | "$JAVA" ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} -cp "${scriptDir}/config:${scriptDir}/lib/*" co.elastic.support.diagnostics.DiagnosticApp "$@" 35 | -------------------------------------------------------------------------------- /scripts/export-monitoring.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | SET scriptpath=%~dp0 5 | SET diagpath=%scriptpath:~0,-1% 6 | SET libpath=%diagpath%\lib\NUL 7 | 8 | IF NOT EXIST %libpath% ( 9 | ECHO Diagnostic executable not found: 10 | ECHO. 11 | ECHO Please make sure that you are running with the archive ending with 12 | ECHO '-dist.zip' in the name and not the one labeled 'Source code'. 13 | ECHO. 14 | ECHO Download at https://github.com/elastic/support-diagnostics/releases/latest 15 | EXIT /b 400 16 | ) 17 | 18 | set JAVA_EXEC=java 19 | if not defined JAVA_HOME ( 20 | set JAVA_EXEC=java 21 | echo No Java Home was found. Using current path. If execution fails please install Java and make sure it is in the search path or exposed via the JAVA_HOME environment variable. 22 | ) else ( 23 | echo JAVA_HOME found, using !JAVA_HOME! 24 | set JAVA_EXEC=!JAVA_HOME!\bin\java 25 | ) 26 | 27 | if defined DIAG_DEBUG ( 28 | set DIAG_DEBUG_OPTS=-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y 29 | ) 30 | 31 | if not defined DIAG_JAVA_OPTS ( 32 | set DIAG_JAVA_OPTS=-Xms2g -Xmx2g 33 | ) 34 | 35 | echo Using %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% for options. 36 | "%JAVA_EXEC%" %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% -cp %diagpath%\config;%diagpath%\lib\* co.elastic.support.monitoring.MonitoringExportApp %* 37 | 38 | endlocal 39 | -------------------------------------------------------------------------------- /scripts/export-monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | scriptDir="$(cd -- "$(dirname -- "$0")" && pwd)" 4 | libDir="$scriptDir"'/lib' 5 | 6 | if [ ! -d "$libDir" ]; then 7 | echo "Diagnostic executable not found:" 8 | echo "" 9 | echo "Please make sure that you are running with the archive ending with" 10 | echo "'-dist.zip' in the name and not the one labeled 'Source code'." 11 | echo "" 12 | echo "Download at https://github.com/elastic/support-diagnostics/releases/latest" 13 | exit 4 14 | fi 15 | 16 | if [ -x "${JAVA_HOME}/bin/java" ]; then 17 | JAVA="${JAVA_HOME}/bin/java" 18 | else 19 | JAVA=`which java` 20 | fi 21 | 22 | echo "Using ${JAVA} as Java Runtime" 23 | 24 | if [ ! -x "$JAVA" ]; then 25 | echo "Could not find any executable java binary. Please install java in your PATH and/or set JAVA_HOME" 26 | exit 1 27 | fi 28 | 29 | [ "${DIAG_DEBUG}" != "" ] && export DIAG_DEBUG_OPTS="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y" 30 | 31 | [ "${DIAG_JAVA_OPTS}" = "" ] && export DIAG_JAVA_OPTS="-Xms2g -Xmx2g" 32 | 33 | echo "Using ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} for options." 34 | "$JAVA" ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} -cp "${scriptDir}/config:${scriptDir}/lib/*" co.elastic.support.monitoring.MonitoringExportApp "$@" 35 | -------------------------------------------------------------------------------- /scripts/import-monitoring.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | SET scriptpath=%~dp0 5 | SET diagpath=%scriptpath:~0,-1% 6 | SET libpath=%diagpath%\lib\NUL 7 | 8 | IF NOT EXIST %libpath% ( 9 | ECHO Diagnostic executable not found: 10 | ECHO. 11 | ECHO Please make sure that you are running with the archive ending with 12 | ECHO '-dist.zip' in the name and not the one labeled 'Source code'. 13 | ECHO. 14 | ECHO Download at https://github.com/elastic/support-diagnostics/releases/latest 15 | EXIT /b 400 16 | ) 17 | 18 | set JAVA_EXEC=java 19 | if not defined JAVA_HOME ( 20 | set JAVA_EXEC=java 21 | echo No Java Home was found. Using current path. If execution fails please install Java and make sure it is in the search path or exposed via the JAVA_HOME environment variable. 22 | ) else ( 23 | echo JAVA_HOME found, using !JAVA_HOME! 24 | set JAVA_EXEC=!JAVA_HOME!\bin\java 25 | ) 26 | 27 | if defined DIAG_DEBUG ( 28 | set DIAG_DEBUG_OPTS=-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y 29 | ) 30 | 31 | if not defined DIAG_JAVA_OPTS ( 32 | set DIAG_JAVA_OPTS=-Xms2g -Xmx2g 33 | ) 34 | 35 | echo Using %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% for options. 36 | "%JAVA_EXEC%" %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% -cp %diagpath%\config;%diagpath%\lib\* co.elastic.support.monitoring.MonitoringImportApp %* 37 | 38 | endlocal 39 | -------------------------------------------------------------------------------- /scripts/import-monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | scriptDir="$(cd -- "$(dirname -- "$0")" && pwd)" 4 | libDir="$scriptDir"'/lib' 5 | 6 | if [ ! -d "$libDir" ]; then 7 | echo "Diagnostic executable not found:" 8 | echo "" 9 | echo "Please make sure that you are running with the archive ending with" 10 | echo "'-dist.zip' in the name and not the one labeled 'Source code'." 11 | echo "" 12 | echo "Download at https://github.com/elastic/support-diagnostics/releases/latest" 13 | exit 4 14 | fi 15 | 16 | if [ -x "${JAVA_HOME}/bin/java" ]; then 17 | JAVA="${JAVA_HOME}/bin/java" 18 | else 19 | JAVA=`which java` 20 | fi 21 | 22 | echo "Using ${JAVA} as Java Runtime" 23 | 24 | if [ ! -x "$JAVA" ]; then 25 | echo "Could not find any executable java binary. Please install java in your PATH and/or set JAVA_HOME" 26 | exit 1 27 | fi 28 | 29 | [ "${DIAG_DEBUG}" != "" ] && export DIAG_DEBUG_OPTS="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y" 30 | 31 | [ "${DIAG_JAVA_OPTS}" = "" ] && export DIAG_JAVA_OPTS="-Xms2g -Xmx2g" 32 | 33 | echo "Using ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} for options." 34 | "$JAVA" ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} -cp "${scriptDir}/config:${scriptDir}/lib/*" co.elastic.support.monitoring.MonitoringImportApp "$@" 35 | -------------------------------------------------------------------------------- /scripts/scrub.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | setlocal enabledelayedexpansion 3 | 4 | SET scriptpath=%~dp0 5 | SET diagpath=%scriptpath:~0,-1% 6 | SET libpath=%diagpath%\lib\NUL 7 | 8 | IF NOT EXIST %libpath% ( 9 | ECHO Diagnostic executable not found: 10 | ECHO. 11 | ECHO Please make sure that you are running with the archive ending with 12 | ECHO '-dist.zip' in the name and not the one labeled 'Source code'. 13 | ECHO. 14 | ECHO Download at https://github.com/elastic/support-diagnostics/releases/latest 15 | EXIT /b 400 16 | ) 17 | 18 | set JAVA_EXEC=java 19 | if not defined JAVA_HOME ( 20 | set JAVA_EXEC=java 21 | echo No Java Home was found. Using current path. If execution fails please install Java and make sure it is in the search path or exposed via the JAVA_HOME environment variable. 22 | ) else ( 23 | echo JAVA_HOME found, using !JAVA_HOME! 24 | set JAVA_EXEC=!JAVA_HOME!\bin\java 25 | ) 26 | 27 | if defined DIAG_DEBUG ( 28 | set DIAG_DEBUG_OPTS=-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y 29 | ) 30 | 31 | if not defined DIAG_JAVA_OPTS ( 32 | set DIAG_JAVA_OPTS=-Xms8g -Xmx8g 33 | ) 34 | 35 | echo Using %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% for options. 36 | "%JAVA_EXEC%" %DIAG_JAVA_OPTS% %DIAG_DEBUG_OPTS% -cp %diagpath%\config;%diagpath%\lib\* co.elastic.support.scrub.ScrubApp %* 37 | 38 | endlocal 39 | -------------------------------------------------------------------------------- /scripts/scrub.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | scriptDir="$(cd -- "$(dirname -- "$0")" && pwd)" 4 | libDir="$scriptDir"'/lib' 5 | 6 | if [ ! -d "$libDir" ]; then 7 | echo "Diagnostic executable not found:" 8 | echo "" 9 | echo "Please make sure that you are running with the archive ending with" 10 | echo "'-dist.zip' in the name and not the one labeled 'Source code'." 11 | echo "" 12 | echo "Download at https://github.com/elastic/support-diagnostics/releases/latest" 13 | exit 4 14 | fi 15 | 16 | if [ -x "${JAVA_HOME}/bin/java" ]; then 17 | JAVA="${JAVA_HOME}/bin/java" 18 | else 19 | JAVA=`which java` 20 | fi 21 | 22 | echo "Using ${JAVA} as Java Runtime" 23 | 24 | if [ ! -x "$JAVA" ]; then 25 | echo "Could not find any executable java binary. Please install java in your PATH and/or set JAVA_HOME" 26 | exit 1 27 | fi 28 | 29 | [ "${DIAG_DEBUG}" != "" ] && export DIAG_DEBUG_OPTS="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=y" 30 | 31 | [ "${DIAG_JAVA_OPTS}" = "" ] && export DIAG_JAVA_OPTS="-Xms8g -Xmx8g" 32 | 33 | echo "Using ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} for options." 34 | "$JAVA" ${DIAG_JAVA_OPTS} ${DIAG_DEBUG_OPTS} -cp "${scriptDir}/config:${scriptDir}/lib/*" co.elastic.support.scrub.ScrubApp "$@" 35 | -------------------------------------------------------------------------------- /scripts/share_ad_job_state/.gitignore: -------------------------------------------------------------------------------- 1 | **/__pycache__ 2 | .coverage -------------------------------------------------------------------------------- /scripts/share_ad_job_state/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/support-diagnostics/9033eef1233c7fbe533b8f5215e1beada4766da4/scripts/share_ad_job_state/__init__.py -------------------------------------------------------------------------------- /scripts/share_ad_job_state/noxfile.py: -------------------------------------------------------------------------------- 1 | # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 2 | # or more contributor license agreements. Licensed under the Elastic License 3 | # 2.0; you may not use this file except in compliance with the Elastic License 4 | # 2.0. 5 | 6 | from pathlib import Path 7 | 8 | import nox 9 | 10 | BASE_DIR = Path(__file__).parent 11 | SOURCE_FILES = ( 12 | "noxfile.py", 13 | "export_model_snapshot.py", 14 | "import_model_snapshot.py", 15 | "tests/", 16 | ) 17 | 18 | 19 | @nox.session(python=["3.12"], reuse_venv=True) 20 | def format(session): 21 | session.install("black==25.1.0") 22 | session.install("isort==6.0.1") 23 | session.run("black", "--target-version=py312", *SOURCE_FILES) 24 | session.run("isort", *SOURCE_FILES) 25 | lint(session) 26 | 27 | 28 | @nox.session(python=["3.12"], reuse_venv=True) 29 | def lint(session): 30 | session.install("black==25.1.0") 31 | session.install("isort==6.0.1") 32 | session.run("black", "--check", "--diff", "--target-version=py312", *SOURCE_FILES) 33 | session.run("isort", "--check", "--diff", *SOURCE_FILES) 34 | 35 | 36 | @nox.session(python=["3.12"], reuse_venv=True) 37 | def test(session): 38 | session.run("poetry", "install", external=True) 39 | 40 | pytest_args = ("poetry", "run", "pytest", "--cov-report", "lcov") 41 | session.run(*pytest_args, *(session.posargs or ("tests/",)), external=True) 42 | -------------------------------------------------------------------------------- /scripts/share_ad_job_state/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "ad-state-sharing" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>"] 6 | readme = "README.md" 7 | package-mode = false 8 | 9 | [tool.poetry.dependencies] 10 | python = "^3.12" 11 | elasticsearch = "^8.15.1" 12 | loguru = "^0.7.2" 13 | tqdm = "^4.66.5" 14 | 15 | [tool.poetry.group.dev.dependencies] 16 | coverage = {extras = ["toml"], version = "^7.2.0"} 17 | pytest-cov = "^6.0.0" 18 | black = "25.1.0" 19 | nox = "2025.5.1" 20 | isort = "^6.0.1" 21 | 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /scripts/share_ad_job_state/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/support-diagnostics/9033eef1233c7fbe533b8f5215e1beada4766da4/scripts/share_ad_job_state/tests/__init__.py -------------------------------------------------------------------------------- /src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | dist 3 | 4 | 5 | zip 6 | 7 | 8 | 9 | 10 | 11 | ${project.basedir}/docker 12 | ./docker 13 | 14 | * 15 | 16 | 17 | 18 | 19 | ${project.basedir}/scripts 20 | . 21 | 22 | **/* 23 | 24 | 25 | share_ad_job_state/tests/** 26 | 27 | 28 | 29 | 30 | ${project.basedir} 31 | . 32 | 33 | LICENSE.txt 34 | NOTICE.txt 35 | README.md 36 | 37 | 38 | 39 | 40 | ${project.basedir}/src/main/resources 41 | ./config 42 | 43 | **/* 44 | 45 | 46 | 47 | 48 | ${project.build.directory} 49 | ./lib 50 | 51 | diagnostics-*.jar 52 | 53 | 54 | 55 | 56 | ${project.build.directory}/lib 57 | ./lib 58 | 59 | * 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/BaseConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support; 8 | 9 | import org.apache.commons.lang3.StringUtils; 10 | 11 | import java.util.Map; 12 | 13 | public class BaseConfig { 14 | 15 | public String diagnosticVersion; 16 | public int connectionTimeout; 17 | public int connectionRequestTimeout; 18 | public int socketTimeout; 19 | public int maxTotalConn; 20 | public int maxConnPerRoute; 21 | public Map extraHeaders; 22 | 23 | public String diagReleaseHost = "api.github.com"; 24 | public String diagReleaseDest = "/repos/elastic/support-diagnostics/releases/latest"; 25 | public String diagReleaseScheme = "https"; 26 | public String diagLatestRelease = "https://api.github.com/repos/elastic/support-diagnostics/releases/latest"; 27 | 28 | public Map dockerGlobal; 29 | public Map dockerContainer; 30 | public Map kubernates; 31 | public String dockerContainerIds; 32 | public String dockerExecutablePath; 33 | 34 | protected Map configuration; 35 | 36 | public BaseConfig(Map configuration) { 37 | this.configuration = configuration; 38 | 39 | Map githubSettings = (Map) configuration.get("github-settings"); 40 | 41 | if ( githubSettings != null){ 42 | 43 | if (StringUtils.isNotEmpty(githubSettings.get("diagReleaseHost"))) { 44 | diagReleaseHost = githubSettings.get("diagReleaseHost"); 45 | } 46 | 47 | if (StringUtils.isNotEmpty(githubSettings.get("diagReleaseDest"))) { 48 | diagReleaseDest = githubSettings.get("diagReleaseDest"); 49 | } 50 | 51 | if (StringUtils.isNotEmpty(githubSettings.get("diagReleaseScheme"))) { 52 | diagReleaseScheme = githubSettings.get("diagReleaseScheme"); 53 | } 54 | 55 | if (StringUtils.isNotEmpty(githubSettings.get("diagLatestRelease"))) { 56 | diagLatestRelease = githubSettings.get("diagLatestRelease"); 57 | } 58 | } 59 | 60 | Map restConfig = (Map) configuration.get("rest-config"); 61 | 62 | connectionTimeout = restConfig.get("connectTimeout") * 1000; 63 | connectionRequestTimeout = restConfig.get("requestTimeout") * 1000; 64 | socketTimeout = restConfig.get("socketTimeout") * 1000; 65 | maxTotalConn = restConfig.get("maxTotalConn"); 66 | maxConnPerRoute = restConfig.get("maxConnPerRoute"); 67 | 68 | extraHeaders = (Map) configuration.get("extra-headers"); 69 | 70 | dockerGlobal = (Map) configuration.get("docker-global"); 71 | 72 | dockerContainer = (Map) configuration.get("docker-container"); 73 | dockerContainerIds = (String) configuration.get("docker-container-ids"); 74 | dockerExecutablePath = (String) configuration.get("docker-executable-location"); 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/BaseService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support; 8 | 9 | import co.elastic.support.diagnostics.DiagnosticException; 10 | import co.elastic.support.util.SystemProperties; 11 | import co.elastic.support.util.ArchiveUtils; 12 | import org.apache.logging.log4j.LogManager; 13 | import org.apache.logging.log4j.Logger; 14 | import org.apache.logging.log4j.core.Appender; 15 | import org.apache.logging.log4j.core.LoggerContext; 16 | import org.apache.logging.log4j.core.appender.FileAppender; 17 | import org.apache.logging.log4j.core.config.AppenderRef; 18 | import org.apache.logging.log4j.core.config.Configuration; 19 | import org.apache.logging.log4j.core.layout.PatternLayout; 20 | 21 | import java.io.File; 22 | 23 | public abstract class BaseService { 24 | 25 | private Logger logger = LogManager.getLogger(BaseService.class); 26 | protected LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); 27 | protected Configuration logConfig = loggerContext.getConfiguration(); 28 | 29 | protected void closeLogs() { 30 | logger.info(Constants.CONSOLE, "Closing loggers."); 31 | 32 | Appender appender = logConfig.getAppender("diag"); 33 | if (appender != null && appender.isStarted()) { 34 | appender.stop(); 35 | } 36 | 37 | logConfig.getRootLogger().removeAppender("File"); 38 | } 39 | 40 | protected void createFileAppender(String logDir, String logFile) { 41 | Appender diagAppender = FileAppender.newBuilder() 42 | .setConfiguration(logConfig) 43 | .withFileName(logDir + SystemProperties.fileSeparator + logFile) 44 | .withAppend(false) 45 | .withLocking(false) 46 | .setName("packaged") 47 | .setIgnoreExceptions(false) 48 | .setLayout( 49 | PatternLayout.newBuilder() 50 | .withConfiguration(logConfig) 51 | .withPattern("%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n") 52 | .build()) 53 | .build(); 54 | 55 | Appender oldAppender = logConfig.getAppender("packaged"); 56 | if (oldAppender != null && oldAppender.isStarted()) { 57 | oldAppender.stop(); 58 | logConfig.getRootLogger().removeAppender("packaged"); 59 | } 60 | 61 | diagAppender.start(); 62 | logConfig.addAppender(diagAppender); 63 | AppenderRef.createAppenderRef("packaged", null, null); 64 | logConfig.getRootLogger().addAppender(diagAppender, null, null); 65 | loggerContext.updateLoggers(); 66 | logger.info(Constants.CONSOLE, "Diagnostic logger reconfigured for inclusion into archive"); 67 | } 68 | 69 | public File createArchive(String tempDir) throws DiagnosticException { 70 | logger.info(Constants.CONSOLE, "Archiving diagnostic results."); 71 | return ArchiveUtils.createZipArchive(tempDir, SystemProperties.getFileDateString()); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support; 8 | 9 | import org.apache.logging.log4j.Marker; 10 | import org.apache.logging.log4j.MarkerManager; 11 | 12 | import java.util.Arrays; 13 | import java.util.List; 14 | 15 | public class Constants { 16 | 17 | public static final Marker CONSOLE = MarkerManager.getMarker("CONSOLE"); 18 | 19 | public static final String ES_DIAG = "diagnostics"; 20 | public static final String NOT_FOUND = "not found"; 21 | public static final String CHECK_LOG = "Check diagnostics.log in the archive file for more detail."; 22 | 23 | public static final String DIAG_CONFIG = "diags.yml"; 24 | public static final String ES_REST = "elastic-rest.yml"; 25 | public static final String KIBANA_REST = "kibana-rest.yml"; 26 | public static final String LS_REST = "logstash-rest.yml"; 27 | public static final String MONITORING_REST = "monitoring-rest.yml"; 28 | 29 | public static final String QUERY_CONFIG_PACKAGE = "monitoring-extract/"; 30 | public static final String TEMPLATE_CONFIG_PACKAGE = "monitoring-extract/templates/"; 31 | public static final String MONITORING_DIR = "monitoring-export"; 32 | 33 | public static final String DIAG_VERSION = "diagVersion"; 34 | 35 | public static final int DEEFAULT_HTTPS_PORT = 443; 36 | public static final int DEEFAULT_PROXY_PORT = 8080; 37 | 38 | public static final int KIBANA_PORT = 5601; 39 | public static final int LOGSTASH_PORT = 9600; 40 | public static final String[] LOCAL_ADDRESSES = {"127.0.0.1","localhost", "[::1]"}; 41 | public static final List localAddressList = Arrays.asList(LOCAL_ADDRESSES); 42 | public static final String UTF_8 = "UTF-8"; 43 | public static final String UTF_16 = "UTF-16"; 44 | public static final String[] exePaths = {"/bin/","/usr/bin/","/usr/sbin/","/usr/local/bin/","/usr/local/sbin/"}; 45 | 46 | public static final String IPv6Regex = 47 | "(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))"; 48 | public static final String IPv4Regex = "((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"; 49 | public static final String MacAddrRegex = "([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})?"; 50 | public static final String timeZoneRegex = "^([+-](?:2[0-3]|[01][0-9]):[0-5][0-9])$"; 51 | 52 | public static final String winPlatform = "winOS"; 53 | public static final String linuxPlatform = "linuxOS"; 54 | public static final String macPlatform = "macOS"; 55 | 56 | public final static String local = "local"; 57 | public final static String api = "api"; 58 | public final static String remote = "remote"; 59 | 60 | public final static String logstashLocal = "logstash-local"; 61 | public final static String logstashRemote = "logstash-remote"; 62 | public final static String logstashApi = "logstash-api"; 63 | 64 | public static final String restInputHost = "rest:InputHost"; 65 | public static final String systemCommands = "ssh:SystemCommand"; 66 | 67 | public final static String kibanaLocal = "kibana-local"; 68 | public final static String kibanaRemote = "kibana-remote"; 69 | public final static String kibanaApi = "kibana-api"; 70 | 71 | public static final String runningInIde = "runningInIde"; 72 | public static final String interactiveMsg = "Command line options can be displayed with the --help argument. " + 73 | "Entering interactive mode."; 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/DiagConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | import co.elastic.support.BaseConfig; 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | 13 | import java.util.Map; 14 | 15 | public class DiagConfig extends BaseConfig { 16 | private static Logger logger = LogManager.getLogger(DiagConfig.class); 17 | 18 | public int callRetries, pauseRetries, maxLogs, maxGcLogs; 19 | 20 | public DiagConfig(Map configuration) { 21 | super(configuration); 22 | 23 | // When we retry a failed call how many times, and how long to wait before reattempting. 24 | callRetries = (Integer) configuration.get("call-retries"); 25 | pauseRetries = (Integer) configuration.get("pause-retries"); 26 | 27 | // How many rolled over logs do we get? 28 | Map logSettings = (Map) configuration.get("log-settings"); 29 | maxGcLogs = logSettings.get("maxGcLogs"); 30 | maxLogs = logSettings.get("maxLogs"); 31 | } 32 | 33 | public Map> getSysCalls(String key){ 34 | return (Map>)configuration.get(key); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/DiagnosticApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 10 | import co.elastic.support.util.JsonYamlUtils; 11 | import co.elastic.support.util.ResourceCache; 12 | import co.elastic.support.util.SystemUtils; 13 | import co.elastic.support.Constants; 14 | import co.elastic.support.util.TextIOManager; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | public class DiagnosticApp { 22 | 23 | private static final Logger logger = LogManager.getLogger(DiagnosticApp.class); 24 | 25 | public static void main(String[] args) { 26 | try( 27 | ResourceCache resourceCache = new ResourceCache(); 28 | TextIOManager textIOManager = new TextIOManager(); 29 | ) { 30 | DiagnosticInputs diagnosticInputs = new DiagnosticInputs("cli"); 31 | if (args.length == 0) { 32 | logger.info(Constants.CONSOLE, Constants.interactiveMsg); 33 | diagnosticInputs.interactive = true; 34 | diagnosticInputs.runInteractive(textIOManager); 35 | } else { 36 | List errors = diagnosticInputs.parseInputs(textIOManager, args); 37 | if (errors.size() > 0) { 38 | for (String err : errors) { 39 | logger.error(Constants.CONSOLE, err); 40 | } 41 | diagnosticInputs.usage(); 42 | SystemUtils.quitApp(); 43 | } 44 | } 45 | 46 | Map diagMap = JsonYamlUtils.readYamlFromClasspath(Constants.DIAG_CONFIG, true); 47 | DiagConfig diagConfig = new DiagConfig(diagMap); 48 | DiagnosticService diag = new DiagnosticService(); 49 | DiagnosticContext context = new DiagnosticContext(diagConfig, diagnosticInputs, resourceCache, true); 50 | 51 | diag.exec(context); 52 | } catch (ShowHelpException she){ 53 | SystemUtils.quitApp(); 54 | } catch (Exception e) { 55 | logger.error(Constants.CONSOLE,"FATAL ERROR occurred: {}. {}", e.getMessage(), Constants.CHECK_LOG, e); 56 | } 57 | } 58 | 59 | 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/DiagnosticException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | public class DiagnosticException extends Exception { 10 | public DiagnosticException() { 11 | super(); 12 | } 13 | 14 | public DiagnosticException(String message) { 15 | super(message); 16 | } 17 | 18 | public DiagnosticException(String message, Throwable cause) { 19 | super(message, cause); 20 | } 21 | } -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/DiagnosticService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | import co.elastic.support.rest.ElasticRestClientService; 10 | import co.elastic.support.util.SystemProperties; 11 | import co.elastic.support.util.SystemUtils; 12 | import co.elastic.support.Constants; 13 | import co.elastic.support.diagnostics.chain.DiagnosticChainExec; 14 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 15 | import co.elastic.support.rest.RestClient; 16 | import org.apache.commons.io.FileUtils; 17 | import org.apache.logging.log4j.LogManager; 18 | import org.apache.logging.log4j.Logger; 19 | 20 | import java.io.File; 21 | import java.io.IOException; 22 | import java.nio.file.Files; 23 | import java.nio.file.Paths; 24 | 25 | public class DiagnosticService extends ElasticRestClientService { 26 | 27 | private Logger logger = LogManager.getLogger(DiagnosticService.class); 28 | 29 | public File exec(DiagnosticContext context) throws DiagnosticException { 30 | DiagConfig config = context.diagsConfig; 31 | DiagnosticInputs inputs = context.diagnosticInputs; 32 | File file; 33 | 34 | try ( 35 | RestClient esRestClient = RestClient.getClient( 36 | inputs.host, 37 | inputs.port, 38 | inputs.scheme, 39 | inputs.user, 40 | inputs.password, 41 | inputs.proxyHost, 42 | inputs.proxyPort, 43 | inputs.proxyUser, 44 | inputs.proxyPassword, 45 | inputs.pkiKeystore, 46 | inputs.pkiKeystorePass, 47 | inputs.skipVerification, 48 | config.extraHeaders, 49 | config.connectionTimeout, 50 | config.connectionRequestTimeout, 51 | config.socketTimeout)) { 52 | 53 | context.resourceCache.addRestClient(Constants.restInputHost, esRestClient); 54 | 55 | // Create the temp directory - delete if first if it exists from a previous run 56 | String outputDir = inputs.outputDir; 57 | context.tempDir = outputDir + SystemProperties.fileSeparator + inputs.diagType + "-" + Constants.ES_DIAG; 58 | logger.info(Constants.CONSOLE, "{}Creating temp directory: {}", SystemProperties.lineSeparator, 59 | context.tempDir); 60 | 61 | try { 62 | FileUtils.deleteDirectory(new File(context.tempDir)); 63 | Files.createDirectories(Paths.get(context.tempDir)); 64 | } catch (IOException ioe) { 65 | logger.error("Temp directory error", ioe); 66 | logger.info(Constants.CONSOLE, 67 | String.format("Issue with creating temp directory. %s", Constants.CHECK_LOG)); 68 | throw new DiagnosticException("Could not create temporary directory", ioe); 69 | } 70 | 71 | // Modify the log file setup since we're going to package it with the 72 | // diagnostic. 73 | // The log4 configuration file sets up 2 loggers, one strictly for the console 74 | // and a file log in the working directory to handle 75 | // any errors we get at the library level that occur before we can get it 76 | // initiailized. When we have a target directory to 77 | // redirect output to we can reconfigure the appender to go to the diagnostic 78 | // output temp directory for packaging with the archive. 79 | // This lets you configure and create loggers via the file if you want to up the 80 | // level on one of the library dependencies as well 81 | // as internal classes. 82 | // If you want the output to also be shown on the console use: 83 | // logger.info/error/warn/debug(Constants.CONSOLE, "Some log message"); 84 | // This will also log that same output to the diagnostic log file. 85 | // To just log to the file log as normal: logger.info/error/warn/debug("Log 86 | // mewssage"); 87 | 88 | if (context.includeLogs) { 89 | logger.info(Constants.CONSOLE, "Configuring log file."); 90 | createFileAppender(context.tempDir, "diagnostics.log"); 91 | } 92 | DiagnosticChainExec.runDiagnostic(context, inputs.diagType); 93 | 94 | if (context.dockerPresent) { 95 | logger.info(Constants.CONSOLE, 96 | "Identified Docker installations - bypassed log collection and some system calls."); 97 | } 98 | 99 | checkAuthLevel(context.diagnosticInputs.user, context.isAuthorized); 100 | } finally { 101 | if (context.includeLogs) { 102 | closeLogs(); 103 | } 104 | file = createArchive(context.tempDir); 105 | SystemUtils.nukeDirectory(context.tempDir); 106 | } 107 | 108 | return file; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/JavaPlatform.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | import co.elastic.support.Constants; 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | 13 | public class JavaPlatform { 14 | 15 | private static final Logger logger = LogManager.getLogger(JavaPlatform.class); 16 | 17 | public String platform; 18 | private String javaHome = "not set"; 19 | public String javaExecutable ="/bin/java"; 20 | public String javac = "javac"; 21 | 22 | public JavaPlatform(String osName){ 23 | switch (osName){ 24 | case Constants.linuxPlatform: 25 | this.platform = Constants.linuxPlatform; 26 | break; 27 | 28 | case Constants.winPlatform: 29 | this.platform = Constants.winPlatform; 30 | javaExecutable = "\\bin\\java.exe"; 31 | break; 32 | 33 | case Constants.macPlatform: 34 | this.platform = Constants.macPlatform; 35 | break; 36 | 37 | default: 38 | // default it to Linux 39 | logger.info(Constants.CONSOLE, "Failed to detect operating system for: {}", osName); 40 | this.platform = Constants.linuxPlatform; 41 | } 42 | } 43 | 44 | public String extractJavaHome(String jdkProcessString){ 45 | // After the preceding cols are stripped, truncate the output behind the path to the executable. 46 | int jpathIndex = jdkProcessString.indexOf(javaExecutable); 47 | javaHome = jdkProcessString.substring(0, jpathIndex); 48 | 49 | return javaHome; 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/ProcessProfile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | import java.util.HashSet; 10 | import java.util.Set; 11 | 12 | public class ProcessProfile { 13 | public boolean isHttp = false; 14 | public boolean isDocker = false; 15 | public boolean currentMaster = false; 16 | public String name = ""; 17 | public String id = ""; 18 | public String pid = ""; 19 | public String jvmPid = ""; 20 | public String logDir = ""; 21 | public String networkHost; 22 | public String host; 23 | public String ip; 24 | public String httpPublishAddr = ""; 25 | public int httpPort; 26 | public String os; 27 | public Set boundAddresses = new HashSet<>(); 28 | public JavaPlatform javaPlatform; 29 | 30 | 31 | public boolean equals(Object obj) { 32 | if (!(obj instanceof ProcessProfile)) { 33 | return false; 34 | } 35 | ProcessProfile input = (ProcessProfile) obj; 36 | if (input.id.equals(id)) { 37 | return true; 38 | } 39 | return false; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "ProcessProfile{" + 45 | "isHttp=" + isHttp + 46 | ", isDocker=" + isDocker + 47 | ", currentMaster=" + currentMaster + 48 | ", name='" + name + '\'' + 49 | ", id='" + id + '\'' + 50 | ", pid='" + pid + '\'' + 51 | ", jvmPid='" + jvmPid + '\'' + 52 | ", logDir='" + logDir + '\'' + 53 | ", networkHost='" + networkHost + '\'' + 54 | ", host='" + host + '\'' + 55 | ", ip='" + ip + '\'' + 56 | ", httpPublishAddr='" + httpPublishAddr + '\'' + 57 | ", httpPort=" + httpPort + 58 | ", os='" + os + '\'' + 59 | ", boundAddresses=" + boundAddresses + 60 | ", javaPlatform=" + javaPlatform + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/ShowHelpException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics; 8 | 9 | public class ShowHelpException extends RuntimeException { 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/chain/Command.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.chain; 8 | 9 | import co.elastic.support.diagnostics.DiagnosticException; 10 | 11 | public interface Command { 12 | void execute(DiagnosticContext context) throws DiagnosticException; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/chain/DiagnosticContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.chain; 8 | 9 | import co.elastic.support.diagnostics.DiagnosticInputs; 10 | import co.elastic.support.diagnostics.ProcessProfile; 11 | import co.elastic.support.rest.RestEntry; 12 | import co.elastic.support.diagnostics.DiagConfig; 13 | import co.elastic.support.util.ResourceCache; 14 | import org.semver4j.Semver; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | public class DiagnosticContext { 21 | 22 | public boolean runSystemCalls = true; 23 | public boolean isAuthorized = true; 24 | public boolean dockerPresent = false; 25 | public int perPage = 0; 26 | /** whether to include log file in generated diagnostic bundle */ 27 | public boolean includeLogs; 28 | 29 | public String clusterName = ""; 30 | public String tempDir = ""; 31 | public String diagVersion; 32 | 33 | // public RestClient esRestClient; 34 | public DiagConfig diagsConfig; 35 | public DiagnosticInputs diagnosticInputs; 36 | public ProcessProfile targetNode; 37 | public Semver version; 38 | 39 | public List dockerContainers = new ArrayList(); 40 | public Map elasticRestCalls; 41 | public Map fullElasticRestCalls; 42 | 43 | public ResourceCache resourceCache; 44 | 45 | public DiagnosticContext(DiagConfig diagConfig, DiagnosticInputs diagnosticInputs, ResourceCache resourceCache, 46 | boolean includeLogs) { 47 | this.diagsConfig = diagConfig; 48 | this.diagnosticInputs = diagnosticInputs; 49 | this.resourceCache = resourceCache; 50 | this.includeLogs = includeLogs; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/BaseQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.chain.Command; 11 | import co.elastic.support.rest.RestClient; 12 | import co.elastic.support.rest.RestEntry; 13 | import co.elastic.support.rest.RestResult; 14 | import co.elastic.support.util.SystemProperties; 15 | import org.apache.commons.lang3.StringUtils; 16 | import org.apache.logging.log4j.LogManager; 17 | import org.apache.logging.log4j.Logger; 18 | 19 | import java.io.File; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | public abstract class BaseQuery implements Command { 24 | 25 | private final Logger logger = LogManager.getLogger(BaseQuery.class); 26 | 27 | /* 28 | * This class has shared functionality for both the Elasticsearch and 29 | * Logstash based REST calls. It interates through set of endpoints from the 30 | * configuration and executes each. In all cases the results are written 31 | * directly to disk to a successful access. For some specialized configured 32 | * cases such as the node and shard calls, a failure will result in a reattempt 33 | * after the configured number of seconds. 34 | */ 35 | public int runQueries(RestClient restClient, List entries, String tempDir, int retries, int pause) { 36 | 37 | // Run through the query list, first pass. If anything that's retryable failed the 38 | // RestEntry will be in the returned retry list. 39 | List retryList = execQueryList(restClient, entries, tempDir); 40 | int totalRetries = retryList.size(); 41 | 42 | for (int i = 0; i < retries; i++) { 43 | // If no failed entries are in the list, get out 44 | if (retryList.size() == 0) { 45 | break; 46 | } 47 | 48 | // Wait the configured pause time before trying again 49 | try { 50 | logger.warn(Constants.CONSOLE, "Some calls failed but were flagged as recoverable: retrying in {} seconds.", pause / 1000); 51 | Thread.sleep(pause); 52 | } catch (Exception e) { 53 | logger.info(Constants.CONSOLE, "Failed pause on error.", e); 54 | } 55 | 56 | retryList = execQueryList(restClient, retryList, tempDir); 57 | totalRetries += retryList.size(); 58 | 59 | } 60 | return totalRetries; 61 | } 62 | 63 | List execQueryList(RestClient restClient, List calls, String tempdir){ 64 | 65 | List retryFailed = new ArrayList<>(); 66 | 67 | for (RestEntry entry : calls) { 68 | try { 69 | String subdir = entry.getSubdir(); 70 | if(StringUtils.isEmpty(subdir)){ 71 | subdir = tempdir; 72 | } 73 | else { 74 | subdir = tempdir + SystemProperties.fileSeparator + subdir; 75 | File nestedFolder = new File(subdir); 76 | if( ! nestedFolder.isDirectory() ){ 77 | nestedFolder.mkdir(); 78 | } 79 | } 80 | String fileName = subdir + SystemProperties.fileSeparator + entry.getName() + entry.getExtension(); 81 | RestResult restResult = restClient.execQuery(entry.getUrl(), fileName); 82 | if (restResult.isValid()) { 83 | logger.info(Constants.CONSOLE, "Results written to: {}", fileName); 84 | } 85 | else{ 86 | if(entry.isRetry() && restResult.isRetryable()){ 87 | retryFailed.add(entry); 88 | logger.info("{} {} failed.", entry.getName(), entry.getUrl()); 89 | logger.info(restResult.formatStatusMessage("Flagged for retry.")); 90 | } 91 | else{ 92 | logger.info(Constants.CONSOLE, "{} {} failed. Bypassing", entry.getName(), entry.getUrl()); 93 | logger.info(Constants.CONSOLE, restResult.formatStatusMessage("See archived diagnostics.log for more detail.")); 94 | } 95 | } 96 | } catch (Exception e) { 97 | // Something happens just log it and go to the next query. 98 | logger.error( "Error occurred executing query {}", entry.getName() + " - " + entry.getUrl(), e); 99 | continue; 100 | } 101 | 102 | } 103 | return retryFailed; 104 | 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CheckDiagnosticVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.rest.RestResult; 10 | import co.elastic.support.util.JsonYamlUtils; 11 | import co.elastic.support.Constants; 12 | import co.elastic.support.diagnostics.chain.Command; 13 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 14 | import co.elastic.support.rest.RestClient; 15 | import com.fasterxml.jackson.databind.JsonNode; 16 | import com.fasterxml.jackson.databind.node.ArrayNode; 17 | import org.apache.commons.lang3.StringUtils; 18 | import org.apache.logging.log4j.LogManager; 19 | import org.apache.logging.log4j.Logger; 20 | import org.semver4j.Semver; 21 | 22 | import java.util.List; 23 | import java.util.Scanner; 24 | 25 | 26 | public class CheckDiagnosticVersion implements Command { 27 | 28 | private final Logger logger = LogManager.getLogger(CheckDiagnosticVersion.class); 29 | 30 | /** 31 | * Checks the embedded version of the current diagnostic created when the jar was built 32 | * with the release version of the current deployment on Github. 33 | * If it is not that version, prompt the user on whether to continue. 34 | * It will also provide a download URL for the current version. 35 | */ 36 | 37 | public void execute(DiagnosticContext context) { 38 | 39 | // For airgapped environments allow them to bypass this check 40 | if (context.diagnosticInputs.bypassDiagVerify) { 41 | return; 42 | } 43 | 44 | logger.info(Constants.CONSOLE, "Checking for diagnostic version updates."); 45 | // Only need this once so let it auto-close at the end of the try catch block. 46 | try(RestClient restClient = RestClient.getClient( 47 | context.diagsConfig.diagReleaseHost, 48 | Constants.DEEFAULT_HTTPS_PORT, 49 | context.diagsConfig.diagReleaseScheme, 50 | "", 51 | "", 52 | "", 53 | 0, 54 | "", 55 | "", 56 | "", 57 | "", 58 | true, 59 | context.diagsConfig.extraHeaders, 60 | context.diagsConfig.connectionTimeout, 61 | context.diagsConfig.connectionRequestTimeout, 62 | context.diagsConfig.socketTimeout)){ 63 | 64 | // Get the current diagnostic version that was populated in the 65 | // manifest generation step - if we're running in 66 | // the IDE via a run configuration and/or debugger it will 67 | // have a value of "debug" instead of an actual version. 68 | context.diagVersion = getToolVersion(); 69 | if (context.diagVersion.equals(Constants.runningInIde) || StringUtils.isEmpty(context.diagVersion)) { 70 | logger.info(Constants.CONSOLE, "Running in IDE"); 71 | // Default it to something that won't blow up the Semver but shows it's not a normal run. 72 | context.diagVersion = "0.0.0"; 73 | return; 74 | } 75 | 76 | RestResult restResult = new RestResult(restClient.execGet( 77 | context.diagsConfig.diagLatestRelease), context.diagsConfig.diagLatestRelease); 78 | JsonNode rootNode = JsonYamlUtils.createJsonNodeFromString(restResult.toString()); 79 | // newer tags are prefixed with `v`, so remove it 80 | String ver = rootNode.path("tag_name").asText().replaceAll("^v", ""); 81 | 82 | Semver diagVer = new Semver(context.diagVersion); 83 | String rule = ">= " + ver; 84 | 85 | if (!diagVer.satisfies(rule)) { 86 | 87 | logger.info(Constants.CONSOLE, "Warning: DiagnosticService version:{} is not the current recommended release", context.diagVersion); 88 | logger.info(Constants.CONSOLE, "The current release is {}", ver); 89 | 90 | // Try to get the link for the download url of the current release. 91 | List assets = rootNode.findValues("assets"); 92 | JsonNode asset = assets.get(0); 93 | ArrayNode attachments = null; 94 | if(asset.isArray()){ 95 | attachments = (ArrayNode)asset; 96 | asset = attachments.get(0); 97 | } 98 | String downloadUrl = asset.path("browser_download_url").asText(); 99 | if(StringUtils.isEmpty(downloadUrl)){ 100 | downloadUrl = context.diagsConfig.diagLatestRelease; 101 | } 102 | 103 | logger.info(Constants.CONSOLE, "The latest version can be downloaded at {}", downloadUrl); 104 | logger.info(Constants.CONSOLE, "Press the Enter key to continue."); 105 | 106 | Scanner sc = new Scanner(System.in); 107 | sc.nextLine(); 108 | } 109 | 110 | } catch (Exception e) { 111 | logger.error( e); 112 | logger.info(Constants.CONSOLE, "Issue encountered while checking diagnostic version for updates."); 113 | logger.info(Constants.CONSOLE, "Failed to get current diagnostic version from Github."); 114 | logger.info(Constants.CONSOLE, "If Github is not accessible from this environment current supported version cannot be confirmed."); 115 | } 116 | } 117 | 118 | public String getToolVersion() { 119 | String ver = GenerateManifest.class.getPackage().getImplementationVersion(); 120 | return (ver != null) ? ver : Constants.runningInIde; 121 | } 122 | 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CheckElasticsearchVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.DiagConfig; 11 | import co.elastic.support.diagnostics.DiagnosticException; 12 | import co.elastic.support.diagnostics.DiagnosticInputs; 13 | import co.elastic.support.diagnostics.chain.Command; 14 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 15 | import co.elastic.support.rest.RestClient; 16 | import co.elastic.support.rest.RestEntryConfig; 17 | import co.elastic.support.rest.RestResult; 18 | import co.elastic.support.util.JsonYamlUtils; 19 | import co.elastic.support.util.SystemProperties; 20 | import com.fasterxml.jackson.databind.JsonNode; 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.semver4j.Semver; 24 | 25 | import java.util.Map; 26 | 27 | public class CheckElasticsearchVersion implements Command { 28 | 29 | /** 30 | * Gets the version of Elasticsearch that is running. This also 31 | * acts as a sanity check. If there are connection issues and it fails 32 | * this will bet the first indication since this is lightweight enough 33 | * that is should usually succeed. If we don't have a version we 34 | * won't be able to generate the correct call selection later on. 35 | */ 36 | private static final Logger logger = LogManager.getLogger(CheckElasticsearchVersion.class); 37 | 38 | public void execute(DiagnosticContext context) throws DiagnosticException { 39 | 40 | // Get the version number from the JSON returned 41 | // by just submitting the host/port combo 42 | logger.info(Constants.CONSOLE, "Getting Elasticsearch Version."); 43 | DiagnosticInputs inputs = context.diagnosticInputs; 44 | DiagConfig config = context.diagsConfig; 45 | 46 | try { 47 | RestClient restClient = RestClient.getClient( 48 | inputs.host, 49 | inputs.port, 50 | inputs.scheme, 51 | inputs.user, 52 | inputs.password, 53 | inputs.proxyHost, 54 | inputs.proxyPort, 55 | inputs.proxyUser, 56 | inputs.proxyPassword, 57 | inputs.pkiKeystore, 58 | inputs.pkiKeystorePass, 59 | inputs.skipVerification, 60 | config.extraHeaders, 61 | config.connectionTimeout, 62 | config.connectionRequestTimeout, 63 | config.socketTimeout); 64 | 65 | // Add it to the global cache - automatically closed on exit. 66 | context.resourceCache.addRestClient(Constants.restInputHost, restClient); 67 | context.version = getElasticsearchVersion(restClient); 68 | String version = context.version.getVersion(); 69 | RestEntryConfig builder = new RestEntryConfig(version, inputs.mode); 70 | Map restCalls = JsonYamlUtils.readYamlFromClasspath(Constants.ES_REST, true); 71 | 72 | context.elasticRestCalls = builder.buildEntryMap(restCalls); 73 | // For Elasticsearch, we use some API calls based 74 | context.fullElasticRestCalls = new RestEntryConfig(version).buildEntryMap(restCalls); 75 | } catch (Exception e) { 76 | logger.error("Unanticipated error:", e); 77 | throw new DiagnosticException(String.format( 78 | "Could not retrieve the Elasticsearch version due to a system or network error - unable to continue. %s%s%s", 79 | e.getMessage(), SystemProperties.lineSeparator, Constants.CHECK_LOG)); 80 | } 81 | } 82 | 83 | public static Semver getElasticsearchVersion(RestClient client) throws DiagnosticException { 84 | RestResult res = client.execQuery("/"); 85 | if (!res.isValid()) { 86 | throw new DiagnosticException( 87 | res.formatStatusMessage("Could not retrieve the Elasticsearch version - unable to continue.")); 88 | } 89 | String result = res.toString(); 90 | JsonNode root = JsonYamlUtils.createJsonNodeFromString(result); 91 | String version = root.path("version").path("number").asText(); 92 | // Workaround for the semver issue with pre-release versions 93 | // https://github.com/semver4j/semver4j/issues/307 94 | return new Semver(version).withClearedPreRelease(); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CheckKibanaVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.DiagnosticException; 11 | import co.elastic.support.diagnostics.chain.Command; 12 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 13 | import co.elastic.support.rest.RestClient; 14 | import co.elastic.support.rest.RestEntryConfig; 15 | import co.elastic.support.rest.RestResult; 16 | import co.elastic.support.util.JsonYamlUtils; 17 | import co.elastic.support.util.SystemProperties; 18 | import com.fasterxml.jackson.databind.JsonNode; 19 | import org.apache.logging.log4j.LogManager; 20 | import org.apache.logging.log4j.Logger; 21 | import org.semver4j.Semver; 22 | import org.semver4j.SemverException; 23 | 24 | import java.util.Map; 25 | 26 | /** 27 | * {@code CheckKibanaVersion} uses the REST configuration to fetch the version 28 | * of 29 | * Kibana from the server. 30 | * 31 | * If this request fails, then the rest of the diagnostic cannot process because 32 | * REST 33 | * calls are setup against specific versions and, without having a version, they 34 | * cannot 35 | * be setup. 36 | */ 37 | public class CheckKibanaVersion implements Command { 38 | 39 | private static final Logger logger = LogManager.getLogger(CheckKibanaVersion.class); 40 | 41 | public void execute(DiagnosticContext context) throws DiagnosticException { 42 | 43 | // Get the version number from the JSON returned 44 | // by just submitting the host/port combo 45 | logger.info(Constants.CONSOLE, "Getting Kibana Version."); 46 | 47 | try { 48 | RestClient restClient = RestClient.getClient( 49 | context.diagnosticInputs.host, 50 | context.diagnosticInputs.port, 51 | context.diagnosticInputs.scheme, 52 | context.diagnosticInputs.user, 53 | context.diagnosticInputs.password, 54 | context.diagnosticInputs.proxyHost, 55 | context.diagnosticInputs.proxyPort, 56 | context.diagnosticInputs.proxyUser, 57 | context.diagnosticInputs.proxyPassword, 58 | context.diagnosticInputs.pkiKeystore, 59 | context.diagnosticInputs.pkiKeystorePass, 60 | context.diagnosticInputs.skipVerification, 61 | context.diagsConfig.extraHeaders, 62 | context.diagsConfig.connectionTimeout, 63 | context.diagsConfig.connectionRequestTimeout, 64 | context.diagsConfig.socketTimeout); 65 | 66 | // Add it to the global cache - automatically closed on exit. 67 | context.resourceCache.addRestClient(Constants.restInputHost, restClient); 68 | context.version = getKibanaVersion(restClient); 69 | String version = context.version.getVersion(); 70 | RestEntryConfig builder = new RestEntryConfig(version); 71 | Map restCalls = JsonYamlUtils.readYamlFromClasspath(Constants.KIBANA_REST, true); 72 | 73 | // logger.info(Constants.CONSOLE, restCalls); 74 | logger.info(Constants.CONSOLE, "Run basic queries for Kibana: {}", restCalls); 75 | 76 | context.elasticRestCalls = builder.buildEntryMap(restCalls); 77 | context.fullElasticRestCalls = context.elasticRestCalls; 78 | } catch (Exception e) { 79 | logger.error("Unanticipated error:", e); 80 | String errorLog = "Could't retrieve Kibana version due to a system or network error. %s%s%s"; 81 | errorLog = String.format(errorLog, e.getMessage(), 82 | SystemProperties.lineSeparator, 83 | Constants.CHECK_LOG); 84 | throw new DiagnosticException(errorLog); 85 | } 86 | } 87 | 88 | /** 89 | * Fetch the Kibana version using the {@code client}, which is used to then to 90 | * determine which 91 | * REST endpoints can be used from the diagnostic. 92 | * 93 | * @param client The configured client to connect to Kibana. 94 | * @return The Kibana version (semver). 95 | * @throws DiagnosticException if the request fails or the version is invalid 96 | */ 97 | public static Semver getKibanaVersion(RestClient client) throws DiagnosticException { 98 | RestResult res = client.execQuery("/api/stats"); 99 | if (!res.isValid()) { 100 | throw new DiagnosticException( 101 | res.formatStatusMessage("Could not retrieve the Kibana version - unable to continue.")); 102 | } 103 | String result = res.toString(); 104 | JsonNode root = JsonYamlUtils.createJsonNodeFromString(result); 105 | String version = root.path("kibana").path("version").asText(); 106 | 107 | logger.info(Constants.CONSOLE, String.format("Kibana Version is :%s", version)); 108 | 109 | try { 110 | // Workaround for the semver issue with pre-release versions 111 | // https://github.com/semver4j/semver4j/issues/307 112 | return new Semver(version).withClearedPreRelease(); 113 | } catch (SemverException ex) { 114 | throw new DiagnosticException( 115 | String.format("Kibana version format is wrong - unable to continue. (%s)", version)); 116 | } 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CheckUserAuthLevel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.util.UrlUtils; 10 | import co.elastic.support.Constants; 11 | import co.elastic.support.diagnostics.chain.Command; 12 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 13 | import co.elastic.support.rest.RestClient; 14 | import co.elastic.support.rest.RestEntry; 15 | import co.elastic.support.rest.RestResult; 16 | import co.elastic.support.util.JsonYamlUtils; 17 | import com.fasterxml.jackson.databind.JsonNode; 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.semver4j.Semver; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | public class CheckUserAuthLevel implements Command { 26 | @Override 27 | public void execute(DiagnosticContext context) { 28 | final String inputUsername = context.diagnosticInputs.user; 29 | 30 | // No user, it's not secured so no auth level or built-in admin role. 31 | if (StringUtils.isEmpty(inputUsername) || "elastic".equals(inputUsername)) { 32 | return; 33 | } 34 | 35 | // Unlike most APIs, the username is passed as a part of the URL and 36 | // thus it needs to be URL-encoded for the rare instance where special 37 | // characters are used 38 | String username = UrlUtils.encodeValue(inputUsername); 39 | 40 | // Should already be there. 41 | RestClient restClient = context.resourceCache.getRestClient(Constants.restInputHost); 42 | 43 | boolean hasAuthorization = false; 44 | Semver version = context.version; 45 | Map calls = context.fullElasticRestCalls; 46 | RestEntry entry = calls.get("security_users"); 47 | String url = entry.getUrl() + "/" + username; 48 | 49 | RestResult result = restClient.execQuery(url); 50 | 51 | if (result.getStatus() == 200) { 52 | String userJsonString = result.toString(); 53 | JsonNode userNode = JsonYamlUtils.createJsonNodeFromString(userJsonString); 54 | hasAuthorization = checkForAuth(version.getMajor(), context.diagnosticInputs.user, userNode); 55 | } 56 | 57 | context.isAuthorized = hasAuthorization; 58 | } 59 | 60 | public boolean checkForAuth(int major, String user, JsonNode userNode) { 61 | JsonNode rolesNode = userNode.path(user).path("roles"); 62 | boolean hasAuthorization = false; 63 | 64 | if (rolesNode.isArray()) { 65 | ObjectMapper mapper = new ObjectMapper(); 66 | List roles = mapper.convertValue(rolesNode, List.class); 67 | 68 | if (major <= 2) { 69 | hasAuthorization = roles.contains("admin"); 70 | } else { 71 | hasAuthorization = roles.contains("superuser"); 72 | } 73 | } 74 | 75 | return hasAuthorization; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CollectDockerInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.chain.Command; 11 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 12 | import co.elastic.support.util.LocalSystem; 13 | import co.elastic.support.util.SystemCommand; 14 | import co.elastic.support.util.SystemProperties; 15 | import co.elastic.support.util.SystemUtils; 16 | import org.apache.commons.io.IOUtils; 17 | import org.apache.commons.lang3.StringUtils; 18 | import org.apache.logging.log4j.LogManager; 19 | import org.apache.logging.log4j.Logger; 20 | 21 | import java.io.StringReader; 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | public class CollectDockerInfo implements Command { 27 | 28 | private static final Logger logger = LogManager.getLogger(CollectDockerInfo.class); 29 | 30 | public void execute(DiagnosticContext context) { 31 | 32 | SystemCommand systemCommand = context.resourceCache.getSystemCommand(Constants.systemCommands); 33 | 34 | // Run the system calls first to get the host's stats 35 | String targetDir = context.tempDir + SystemProperties.fileSeparator + "syscalls"; 36 | String pid = "1"; 37 | String platform = Constants.linuxPlatform; 38 | if (systemCommand instanceof LocalSystem) { 39 | SystemUtils.parseOperatingSystemName(SystemProperties.osName); 40 | } 41 | 42 | Map> osCmds = context.diagsConfig.getSysCalls(platform); 43 | Map sysCalls = osCmds.get("sys"); 44 | CollectSystemCalls.processCalls(targetDir, sysCalls, systemCommand, pid); 45 | 46 | targetDir = context.tempDir + SystemProperties.fileSeparator + "docker"; 47 | 48 | // Determine where Docker is located 49 | String dockerPath = getDockerPath(systemCommand, platform); 50 | // String kubePath = getKubectlPath(dockerPath); 51 | 52 | // Run the global calls. It's a single pass 53 | runDockerCalls(targetDir, context.diagsConfig.dockerGlobal, systemCommand, "", dockerPath); 54 | // runDockerCalls(targetDir, context.diagsConfig.kubernates, systemCommand, "", 55 | // kubePath); 56 | 57 | String idsCmd = context.diagsConfig.dockerContainerIds; 58 | 59 | List containerIds = getDockerContainerIds(systemCommand, idsCmd, dockerPath); 60 | for (String container : containerIds) { 61 | runDockerCalls(targetDir, context.diagsConfig.dockerContainer, systemCommand, container, dockerPath); 62 | } 63 | } 64 | 65 | public List getDockerContainerIds(SystemCommand systemCommand, String idsCmd, String dockerPath) { 66 | 67 | try { 68 | idsCmd = idsCmd.replace("{{dockerPath}}", dockerPath); 69 | String output = systemCommand.runCommand(idsCmd); 70 | 71 | // If there's content add it to the file list 72 | if (StringUtils.isNotEmpty(output.trim())) { 73 | try { 74 | List lines = IOUtils.readLines(new StringReader(output)); 75 | return lines; 76 | 77 | } catch (Exception e) { 78 | logger.error("Error getting directory listing.", e); 79 | } 80 | } 81 | 82 | } catch (Exception e) { 83 | logger.error("Error obtaining Docker Container Id's"); 84 | } 85 | 86 | // If anything happened just bypass processing and continue with the rest. 87 | return new ArrayList(); 88 | 89 | } 90 | 91 | private void runDockerCalls(String targetDir, Map commandMap, SystemCommand sysCmd, String token, 92 | String dockerPath) { 93 | String suffix = ""; 94 | if (StringUtils.isNotEmpty(token)) { 95 | suffix = "-" + token; 96 | } 97 | 98 | for (Map.Entry entry : commandMap.entrySet()) { 99 | try { 100 | String cmd = entry.getValue().replace("{{CONTAINER_ID}}", token); 101 | cmd = cmd.replace("{{dockerPath}}", dockerPath); 102 | 103 | String output = sysCmd.runCommand(cmd); 104 | SystemUtils.writeToFile(output, 105 | targetDir + SystemProperties.fileSeparator + entry.getKey() + suffix + ".txt"); 106 | } catch (Exception e) { 107 | logger.error(Constants.CONSOLE, e.getMessage()); 108 | } 109 | } 110 | 111 | } 112 | 113 | private String getDockerPath(SystemCommand systemCommand, String platform) { 114 | 115 | if (platform.equalsIgnoreCase(Constants.winPlatform)) { 116 | return "docker"; 117 | } 118 | 119 | for (String path : Constants.exePaths) { 120 | String expectedPath = path + "docker"; 121 | String dockerPath = systemCommand.runCommand("ls " + expectedPath); 122 | dockerPath = dockerPath.trim(); 123 | if (expectedPath.equalsIgnoreCase(dockerPath)) { 124 | return dockerPath; 125 | } 126 | } 127 | 128 | return "docker"; 129 | } 130 | 131 | private String getKubectlPath(String dockerPath) { 132 | 133 | return dockerPath.replace("docker", "kubectl"); 134 | 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CollectKibanaLogs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.ProcessProfile; 11 | import co.elastic.support.diagnostics.chain.Command; 12 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 13 | import co.elastic.support.util.ResourceCache; 14 | import co.elastic.support.util.SystemCommand; 15 | import co.elastic.support.util.SystemProperties; 16 | import org.apache.commons.io.IOUtils; 17 | import org.apache.commons.lang3.ObjectUtils; 18 | import org.apache.commons.lang3.StringUtils; 19 | import org.apache.logging.log4j.LogManager; 20 | import org.apache.logging.log4j.Logger; 21 | 22 | import java.io.File; 23 | import java.io.StringReader; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | /** 29 | * Kibana do not provide any API that expose the logs path 30 | * This class will check if the logs are in the default path and if they are collect them. 31 | */ 32 | public class CollectKibanaLogs implements Command { 33 | 34 | private static final Logger logger = LogManager.getLogger(CollectLogs.class); 35 | 36 | /** 37 | * We will get the information fron the DiagnosticContext to extract and copy the logs from Local or Remote system. 38 | */ 39 | public void execute(DiagnosticContext context) { 40 | // If we hit a snafu earlier in determining the details on where and how to run, then just get out. 41 | if(!context.runSystemCalls){ 42 | logger.info(Constants.CONSOLE, "There was an issue in setting up system logs collection - bypassing. {}", Constants.CHECK_LOG); 43 | return; 44 | } 45 | // the defaults are only set for RPM / Debian or Homebrew, in windows we can not collect any Kibana logs. 46 | if (context.targetNode.os.equals(Constants.winPlatform)) { 47 | logger.info(Constants.CONSOLE, "Kibana logs can not be collected for Windows, the log path is not shared in the Kibana APIs and there is no defaults."); 48 | return; 49 | } 50 | 51 | SystemCommand sysCmd = context.resourceCache.getSystemCommand(Constants.systemCommands); 52 | String targetDir = context.tempDir + SystemProperties.fileSeparator + "logs"; 53 | ProcessProfile targetNode = context.targetNode; 54 | 55 | Map> osCmds = context.diagsConfig.getSysCalls(context.targetNode.os); 56 | Map logCalls = osCmds.get("logs"); 57 | 58 | try{ 59 | // Create the log directory - should never exist but always pays to be cautious. 60 | File tempLogDir = new File(targetDir); 61 | if(!tempLogDir.exists()){ 62 | tempLogDir.mkdir(); 63 | } 64 | String logStatement = logCalls.get("kibana"); 65 | String logListing = sysCmd.runCommand(logStatement).trim(); 66 | if (StringUtils.isEmpty(logListing) || logListing.contains("No such file or directory")) { 67 | sysCmd.copyLogsFromJournalctl("kibana.service", targetDir); 68 | logger.info(Constants.CONSOLE, "No Kibana logs could be located at the default path. Searching for Journalctl logs with kibana.service name."); 69 | return; 70 | } 71 | List fileList = extractFilesFromList(logListing, 3); 72 | String kibanaPath = logCalls.get("kibana-default-path"); 73 | sysCmd.copyLogs(fileList, kibanaPath, targetDir); 74 | logger.info(Constants.CONSOLE, "Collecting logs from Kibana default path."); 75 | } catch (Exception e) { 76 | logger.info(Constants.CONSOLE, "An error occurred while copying the logs. It may not have completed normally {}.", Constants.CHECK_LOG); 77 | logger.error( e.getMessage(), e); 78 | } 79 | } 80 | 81 | /** 82 | * According with the OS response, read and extract the content for the logs files. 83 | * 84 | * @param output is the text returned by the ls/dir command 85 | * @param entries How many time we will iterate 86 | * @return Never {@code null}. Always the number specified by {@code entries}. 87 | */ 88 | protected List extractFilesFromList(String output, int entries) { 89 | List fileList = new ArrayList<>(entries); 90 | 91 | // Just in case, since NPE"s are a drag 92 | output = ObjectUtils.defaultIfNull(output, ""); 93 | // If there's content add it to the file list 94 | if (StringUtils.isNotEmpty(output.trim())) { 95 | try { 96 | List lines = IOUtils.readLines(new StringReader(output)); 97 | int sz = lines.size(); 98 | for (int i = 0; i < sz; i++) { 99 | fileList.add(lines.get(i)); 100 | if(i == entries){ 101 | break; 102 | } 103 | } 104 | } catch (Exception e) { 105 | logger.error( "Error getting directory listing.", e); 106 | } 107 | } 108 | 109 | return fileList; 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CollectLogs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.ProcessProfile; 11 | import co.elastic.support.diagnostics.JavaPlatform; 12 | import co.elastic.support.diagnostics.chain.Command; 13 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 14 | import co.elastic.support.util.ResourceCache; 15 | import co.elastic.support.util.SystemCommand; 16 | import co.elastic.support.util.SystemProperties; 17 | import org.apache.commons.io.IOUtils; 18 | import org.apache.commons.lang3.ObjectUtils; 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | 23 | import java.io.File; 24 | import java.io.StringReader; 25 | import java.util.ArrayList; 26 | import java.util.Arrays; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class CollectLogs implements Command { 31 | 32 | private static final Logger logger = LogManager.getLogger(CollectLogs.class); 33 | 34 | public void execute(DiagnosticContext context) { 35 | // If we hit a snafu earlier in determining the details on where and how to run, then just get out. 36 | if(!context.runSystemCalls){ 37 | logger.info(Constants.CONSOLE, "There was an issue in setting up system logs collection - bypassing. {}", Constants.CHECK_LOG); 38 | return; 39 | } 40 | 41 | // Should be cached from the PlatformDetails check. 42 | SystemCommand sysCmd = context.resourceCache.getSystemCommand(Constants.systemCommands); 43 | String targetDir = context.tempDir + SystemProperties.fileSeparator + "logs"; 44 | ProcessProfile targetNode = context.targetNode; 45 | JavaPlatform javaPlatform = targetNode.javaPlatform; 46 | String logDir = targetNode.logDir; 47 | String clusterName = context.clusterName; 48 | Map> osCmds = context.diagsConfig.getSysCalls(javaPlatform.platform); 49 | Map logCalls = osCmds.get("logs"); 50 | 51 | // Get the remote systemobject 52 | try{ 53 | 54 | // Create the log directory - should never exist but always pays to be cautious. 55 | File tempLogDir = new File(targetDir); 56 | if(!tempLogDir.exists()){ 57 | tempLogDir.mkdir(); 58 | } 59 | 60 | // Build up a list of files to copy 61 | List fileList = new ArrayList<>(); 62 | 63 | // Check for the base elasticsearch log - will always be .log 64 | // If it's not there, they probably have insufficient permissions so log it 65 | // to the console and exit gracefully. 66 | String logStatement = logCalls.get("elastic"); 67 | logStatement = logStatement.replace("{{CLUSTERNAME}}", clusterName); 68 | logStatement = logStatement.replace("{{LOGPATH}}", logDir); 69 | String logListing = sysCmd.runCommand(logStatement).trim(); 70 | if (StringUtils.isEmpty(logListing)) { 71 | logger.info(Constants.CONSOLE, "No Elasticsearch log could be located at the path configured for this node. The cause of this is usually insufficient read authority. Please be sure the account you are using to access the logs has the necessary permissions or use sudo. See the diagnostic README for more information. "); 72 | return; 73 | } 74 | 75 | fileList.addAll(Arrays.asList(logListing.split("\n"))); 76 | logStatement = logCalls.get("elastic-arc"); 77 | logStatement = logStatement.replace("{{CLUSTERNAME}}", clusterName); 78 | logStatement = logStatement.replace("{{LOGPATH}}", logDir); 79 | logListing = sysCmd.runCommand(logStatement).trim(); 80 | fileList = extractFilesFromList(logListing, fileList, 2); 81 | 82 | logStatement = logCalls.get("gc"); 83 | logStatement = logStatement.replace("{{LOGPATH}}", logDir); 84 | logListing = sysCmd.runCommand(logStatement).trim(); 85 | fileList = extractFilesFromList(logListing, fileList, 3); 86 | 87 | sysCmd.copyLogs(fileList, logDir, targetDir ); 88 | 89 | } catch (Exception e) { 90 | logger.info(Constants.CONSOLE, "An error occurred while copying the logs. It may not have completed normally {}.", Constants.CHECK_LOG); 91 | logger.error( e.getMessage(), e); 92 | } 93 | } 94 | 95 | protected List extractFilesFromList(String output, List fileList, int entries) { 96 | 97 | // Just in case, since NPE"s are a drag 98 | output = ObjectUtils.defaultIfNull(output, ""); 99 | 100 | // If there's content add it to the file list 101 | if (StringUtils.isNotEmpty(output.trim())) { 102 | try { 103 | List lines = IOUtils.readLines(new StringReader(output)); 104 | int sz = lines.size(); 105 | for (int i = 0; i < sz; i++) { 106 | fileList.add(lines.get(i)); 107 | if(i == entries){ 108 | break; 109 | } 110 | } 111 | } catch (Exception e) { 112 | logger.error( "Error getting directory listing.", e); 113 | } 114 | } 115 | 116 | return fileList; 117 | 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/CollectSystemCalls.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.ProcessProfile; 11 | import co.elastic.support.diagnostics.JavaPlatform; 12 | import co.elastic.support.diagnostics.chain.Command; 13 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 14 | import co.elastic.support.util.ResourceCache; 15 | import co.elastic.support.util.SystemCommand; 16 | import co.elastic.support.util.SystemProperties; 17 | import co.elastic.support.util.SystemUtils; 18 | import org.apache.logging.log4j.LogManager; 19 | import org.apache.logging.log4j.Logger; 20 | import java.util.Map; 21 | 22 | public class CollectSystemCalls implements Command { 23 | 24 | private static final Logger logger = LogManager.getLogger(CollectSystemCalls.class); 25 | 26 | public void execute(DiagnosticContext context) { 27 | 28 | // If we hit a snafu earlier in determining the details on where and how to run, then just get out. 29 | if(!context.runSystemCalls){ 30 | logger.info( Constants.CONSOLE, "There was an issue in setting up system call collection - bypassing. {}", Constants.CHECK_LOG); 31 | return; 32 | } 33 | 34 | // Should be cached from the PlatformDetails check. 35 | SystemCommand sysCmd = context.resourceCache.getSystemCommand(Constants.systemCommands); 36 | String targetDir = context.tempDir + SystemProperties.fileSeparator + "syscalls"; 37 | String pid = context.targetNode.pid; 38 | ProcessProfile targetNode = context.targetNode; 39 | JavaPlatform javaPlatform = targetNode.javaPlatform; 40 | String platform = ""; 41 | if (javaPlatform == null) { 42 | platform = context.targetNode.os; 43 | } else { 44 | platform = javaPlatform.platform; 45 | 46 | } 47 | Map> osCmds = context.diagsConfig.getSysCalls(platform); 48 | 49 | try { 50 | // Get the configurations for that platoform's sys calls. 51 | Map sysCalls = osCmds.get("sys"); 52 | processCalls(targetDir, sysCalls, sysCmd, pid); 53 | logger.info(Constants.CONSOLE, "First set of system calls executed."); 54 | // This should give us the full path to the java executable that 55 | // was used to start Elasticsearch 56 | Map javaCalls = osCmds.get("java"); 57 | String javaProcessString = javaCalls.get("elastic-java"); 58 | javaProcessString = javaProcessString.replace("{{PID}}", pid); 59 | String javaExecutablePath = sysCmd.runCommand(javaProcessString); 60 | 61 | // For the JDK based commoands we need to execute them using the same 62 | // JVM that's running ES. Given that it could be the bundled one or some 63 | // previously installed version we need to shell out and check before we run them. 64 | String esJavaHome = javaPlatform.extractJavaHome(javaExecutablePath); 65 | logger.info(Constants.CONSOLE, "Java Home installation at: {}", esJavaHome); 66 | 67 | // Check for the presence of a JDK - run the javac command with no arguments 68 | // and see if you get a valid return - javac 69 | 70 | String javacCheck = javaCalls.get("javac"); 71 | javacCheck = javacCheck.replace("{{JAVA_HOME}}", esJavaHome); 72 | javaCalls.put("javac", javacCheck); 73 | String javacResult = sysCmd.runCommand(javacCheck); 74 | if(javacResult.toLowerCase().contains(javaPlatform.javac)){ 75 | 76 | String jstack = javaCalls.get("jstack"); 77 | jstack = jstack.replace("{{JAVA_HOME}}", esJavaHome); 78 | jstack = jstack.replace("{{PID}}", pid); 79 | 80 | String jps = javaCalls.get("jps"); 81 | jps = jps.replace("{{JAVA_HOME}}", esJavaHome); 82 | 83 | javaCalls.put("jstack", jstack); 84 | javaCalls.put("jps", jps); 85 | processCalls(targetDir, javaCalls, sysCmd, pid); 86 | } 87 | else { 88 | logger.info( Constants.CONSOLE, "JDK not found - bypassing jstack and jps commands."); 89 | } 90 | } catch (Exception e) { 91 | logger.error( e); 92 | logger.info(Constants.CONSOLE, "Unexpected error - bypassing some or all system calls. {}", Constants.CHECK_LOG); 93 | } 94 | } 95 | 96 | public static void processCalls(String targetDir, Map commandMap, SystemCommand sysCmd, String pid) { 97 | commandMap.forEach((k, v) -> { 98 | try { 99 | String cmd = v.replace("{{PID}}", pid); 100 | String output = sysCmd.runCommand(cmd); 101 | SystemUtils.writeToFile(output, targetDir + SystemProperties.fileSeparator + k + ".txt"); 102 | } catch (Exception e) { 103 | logger.info(e.getMessage()); 104 | } 105 | } 106 | ); 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/GenerateDiagnosticManifest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.chain.Command; 11 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 12 | import co.elastic.support.util.SystemProperties; 13 | import com.fasterxml.jackson.databind.ObjectMapper; 14 | import com.fasterxml.jackson.databind.SerializationFeature; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | 18 | import java.io.File; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * Generate a manifest containing the basic runtime info for the diagnostic 24 | * runtime. 25 | */ 26 | public class GenerateDiagnosticManifest implements Command { 27 | private final Logger logger = LogManager.getLogger(GenerateDiagnosticManifest.class); 28 | 29 | public void execute(DiagnosticContext context) { 30 | logger.info(Constants.CONSOLE, "Writing [diagnostic_manifest.json]."); 31 | 32 | String product = "elasticsearch"; 33 | 34 | if (context.diagnosticInputs.diagType.startsWith("kibana")) { 35 | product = "kibana"; 36 | } else if (context.diagnosticInputs.diagType.startsWith("logstash")) { 37 | product = "logstash"; 38 | } 39 | 40 | try { 41 | ObjectMapper mapper = new ObjectMapper(); 42 | mapper.enable(SerializationFeature.INDENT_OUTPUT); 43 | 44 | Map manifest = new HashMap<>(); 45 | 46 | manifest.put("diagnostic", context.diagVersion); 47 | manifest.put("type", product + "_diagnostic"); 48 | manifest.put("product", product); 49 | // Logstash does not lookup a version currently 50 | manifest.put("version", context.version != null ? context.version.getVersion() : "0.0.0"); 51 | manifest.put("timestamp", SystemProperties.getUtcDateTimeString()); 52 | manifest.put("flags", context.diagnosticInputs.toString()); 53 | manifest.put("runner", context.diagnosticInputs.runner); 54 | manifest.put("mode", context.diagnosticInputs.mode); 55 | 56 | mapper.writeValue( 57 | new File(context.tempDir + SystemProperties.fileSeparator + "diagnostic_manifest.json"), 58 | manifest); 59 | } catch (Exception e) { 60 | logger.info(Constants.CONSOLE, "Error creating [diagnostic_manifest.json]", e); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/GenerateManifest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.chain.Command; 11 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 12 | import co.elastic.support.util.SystemProperties; 13 | import com.fasterxml.jackson.databind.ObjectMapper; 14 | import com.fasterxml.jackson.databind.SerializationFeature; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | 18 | import java.io.File; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * Generate a manifest containing the basic runtime info for the diagnostic 24 | * runtime. 25 | */ 26 | public class GenerateManifest implements Command { 27 | private final Logger logger = LogManager.getLogger(GenerateManifest.class); 28 | 29 | public void execute(DiagnosticContext context) { 30 | logger.info(Constants.CONSOLE, "Writing legacy [manifest.json]."); 31 | 32 | try { 33 | ObjectMapper mapper = new ObjectMapper(); 34 | mapper.enable(SerializationFeature.INDENT_OUTPUT); 35 | 36 | Map manifest = new HashMap<>(); 37 | 38 | manifest.put(Constants.DIAG_VERSION, context.diagVersion); 39 | manifest.put("Product Version", context.version); 40 | manifest.put("collectionDate", SystemProperties.getUtcDateTimeString()); 41 | manifest.put("diagnosticInputs", context.diagnosticInputs.toString()); 42 | manifest.put("runner", context.diagnosticInputs.runner); 43 | 44 | mapper.writeValue(new File(context.tempDir + SystemProperties.fileSeparator + "manifest.json"), manifest); 45 | } catch (Exception e) { 46 | logger.info(Constants.CONSOLE, "Error creating the manifest file", e); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/RunClusterQueries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.DiagConfig; 11 | import co.elastic.support.diagnostics.DiagnosticException; 12 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 13 | import co.elastic.support.rest.RestClient; 14 | import co.elastic.support.rest.RestEntry; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | public class RunClusterQueries extends BaseQuery { 22 | 23 | /** 24 | * Builds the list of queries for Elasticsearch version that was retrieved previously, 25 | * then executes them and saves the result to temporary storage. 26 | */ 27 | 28 | private static final Logger logger = LogManager.getLogger(RunClusterQueries.class); 29 | 30 | public void execute(DiagnosticContext context) throws DiagnosticException { 31 | 32 | try { 33 | DiagConfig diagConfig = context.diagsConfig; 34 | List entries = new ArrayList<>(); 35 | entries.addAll(context.elasticRestCalls.values()); 36 | RestClient client; 37 | client = context.resourceCache.getRestClient(Constants.restInputHost); 38 | /* if(ResourceCache.resourceExists(Constants.restTargetHost)){ 39 | client = ResourceCache.getRestClient(Constants.restTargetHost); 40 | } 41 | else { 42 | client = ResourceCache.getRestClient(Constants.restInputHost); 43 | }*/ 44 | 45 | runQueries(client, entries, context.tempDir, diagConfig.callRetries, diagConfig.pauseRetries); 46 | } catch (Throwable t) { 47 | logger.error( "Error executing REST queries", t); 48 | throw new DiagnosticException(String.format("Unrecoverable REST Query Execution error - exiting. %s", Constants.CHECK_LOG)); 49 | } 50 | 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/diagnostics/commands/RunLogstashQueries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.diagnostics.commands; 8 | 9 | import co.elastic.support.util.JsonYamlUtils; 10 | import co.elastic.support.util.LocalSystem; 11 | import co.elastic.support.util.RemoteSystem; 12 | import co.elastic.support.util.SystemCommand; 13 | import co.elastic.support.util.SystemProperties; 14 | import co.elastic.support.Constants; 15 | import co.elastic.support.diagnostics.DiagnosticException; 16 | import co.elastic.support.diagnostics.JavaPlatform; 17 | import co.elastic.support.diagnostics.ProcessProfile; 18 | import co.elastic.support.diagnostics.chain.DiagnosticContext; 19 | import co.elastic.support.rest.RestClient; 20 | import co.elastic.support.rest.RestEntry; 21 | import co.elastic.support.rest.RestEntryConfig; 22 | import co.elastic.support.util.SystemUtils; 23 | import com.fasterxml.jackson.databind.JsonNode; 24 | import org.apache.commons.lang3.StringUtils; 25 | import org.apache.logging.log4j.LogManager; 26 | import org.apache.logging.log4j.Logger; 27 | 28 | import java.util.ArrayList; 29 | import java.util.List; 30 | import java.util.Map; 31 | 32 | public class RunLogstashQueries extends BaseQuery { 33 | 34 | /** 35 | * Executes the REST calls for Logstash 36 | */ 37 | 38 | private static final Logger logger = LogManager.getLogger(BaseQuery.class); 39 | 40 | public void execute(DiagnosticContext context) throws DiagnosticException { 41 | 42 | try { 43 | RestClient client = context.resourceCache.getRestClient(Constants.restInputHost); 44 | 45 | RestEntryConfig builder = new RestEntryConfig("1.0.0"); 46 | Map restCalls = JsonYamlUtils.readYamlFromClasspath(Constants.LS_REST, true); 47 | Map entries = builder.buildEntryMap(restCalls); 48 | 49 | List queries = new ArrayList<>(); 50 | queries.addAll(entries.values()); 51 | runQueries(client, queries, context.tempDir, 0, 0); 52 | 53 | // Get the information we need to run system calls. It's easier to just get it 54 | // off disk after all the REST calls run. 55 | ProcessProfile nodeProfile = new ProcessProfile(); 56 | context.targetNode = nodeProfile; 57 | 58 | JsonNode nodeData = JsonYamlUtils.createJsonNodeFromFileName(context.tempDir, "logstash_node.json"); 59 | nodeProfile.pid = nodeData.path("jvm").path("pid").asText(); 60 | 61 | nodeProfile.os = SystemUtils.parseOperatingSystemName(nodeData.path("os").path("name").asText()); 62 | nodeProfile.javaPlatform = new JavaPlatform(nodeProfile.os); 63 | if (StringUtils.isEmpty(nodeProfile.pid) || nodeProfile.pid.equals("1")) { 64 | context.dockerPresent = true; 65 | context.runSystemCalls = false; 66 | } 67 | // Create and cache the system command type we need, local or remote... 68 | SystemCommand syscmd = null; 69 | switch (context.diagnosticInputs.diagType) { 70 | case Constants.logstashRemote: 71 | String targetOS; 72 | if (context.dockerPresent) { 73 | targetOS = Constants.linuxPlatform; 74 | } else { 75 | targetOS = nodeProfile.os; 76 | } 77 | syscmd = new RemoteSystem( 78 | targetOS, 79 | context.diagnosticInputs.remoteUser, 80 | context.diagnosticInputs.remotePassword, 81 | context.diagnosticInputs.host, 82 | context.diagnosticInputs.remotePort, 83 | context.diagnosticInputs.keyfile, 84 | context.diagnosticInputs.pkiKeystorePass, 85 | context.diagnosticInputs.knownHostsFile, 86 | context.diagnosticInputs.trustRemote, 87 | context.diagnosticInputs.isSudo); 88 | context.resourceCache.addSystemCommand(Constants.systemCommands, syscmd); 89 | break; 90 | 91 | case Constants.logstashLocal: 92 | if (context.dockerPresent) { 93 | syscmd = new LocalSystem(SystemUtils.parseOperatingSystemName(SystemProperties.osName)); 94 | } else { 95 | syscmd = new LocalSystem(nodeProfile.os); 96 | } 97 | context.resourceCache.addSystemCommand(Constants.systemCommands, syscmd); 98 | 99 | break; 100 | 101 | default: 102 | // If it's not one of the above types it shouldn't be here but try to keep 103 | // going... 104 | context.runSystemCalls = false; 105 | } 106 | 107 | } catch (Throwable t) { 108 | logger.error("Logstash Query error:", t); 109 | throw new DiagnosticException(String.format( 110 | "Error obtaining logstash output and/or process id - will bypass the rest of processing.. %s", 111 | Constants.CHECK_LOG)); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringExportApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | import co.elastic.support.util.ResourceCache; 10 | import co.elastic.support.Constants; 11 | import co.elastic.support.diagnostics.ShowHelpException; 12 | import co.elastic.support.util.SystemUtils; 13 | import co.elastic.support.util.TextIOManager; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | 17 | import java.util.List; 18 | 19 | public class MonitoringExportApp { 20 | 21 | private static final Logger logger = LogManager.getLogger(MonitoringExportApp.class); 22 | 23 | public static void main(String[] args) { 24 | 25 | try (TextIOManager textIOManager = new TextIOManager()) { 26 | MonitoringExportInputs monitoringExportInputs = new MonitoringExportInputs(); 27 | if (args.length == 0) { 28 | logger.info(Constants.CONSOLE, Constants.interactiveMsg); 29 | monitoringExportInputs.interactive = true; 30 | monitoringExportInputs.runInteractive(textIOManager); 31 | } else { 32 | List errors = monitoringExportInputs.parseInputs(textIOManager, args); 33 | if (errors.size() > 0) { 34 | for (String err : errors) { 35 | logger.error(Constants.CONSOLE, err); 36 | } 37 | monitoringExportInputs.usage(); 38 | SystemUtils.quitApp(); 39 | } 40 | } 41 | new MonitoringExportService().execExtract(monitoringExportInputs); 42 | } catch (ShowHelpException she){ 43 | SystemUtils.quitApp(); 44 | } catch (Exception e) { 45 | logger.error(Constants.CONSOLE, "FATAL ERROR occurred: {}. {}", e.getMessage(), Constants.CHECK_LOG, e); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringExportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | 10 | import co.elastic.support.util.SystemProperties; 11 | import co.elastic.support.BaseConfig; 12 | import co.elastic.support.Constants; 13 | import org.apache.logging.log4j.LogManager; 14 | import org.apache.logging.log4j.Logger; 15 | import org.semver4j.Semver; 16 | 17 | import java.io.BufferedReader; 18 | import java.io.IOException; 19 | import java.io.InputStream; 20 | import java.io.InputStreamReader; 21 | import java.util.*; 22 | 23 | public class MonitoringExportConfig extends BaseConfig { 24 | 25 | protected List monitoringStats = new ArrayList<>(); 26 | protected Map queries = new LinkedHashMap<>(); 27 | protected String monitoringStartUri; 28 | protected String monitoringUri; 29 | protected String monitoringScrollUri; 30 | protected String monitoringScrollTtl; 31 | protected int monitoringScrollSize; 32 | protected Semver semver; 33 | 34 | protected List queryFiles; 35 | protected List monitorSets; 36 | protected List logstashSets; 37 | protected List metricSets; 38 | 39 | Logger logger = LogManager.getLogger(MonitoringExportConfig.class); 40 | 41 | public MonitoringExportConfig(Map configuration) { 42 | 43 | super(configuration); 44 | 45 | monitorSets = (List) configuration.get("monitor-sets"); 46 | metricSets = (List) configuration.get("metric-sets"); 47 | logstashSets = (List) configuration.get("logstash-sets"); 48 | queryFiles = (List) configuration.get("query-files"); 49 | monitoringScrollSize = (Integer)configuration.get("monitoring-scroll-size"); 50 | queries = getQueriesFromFiles(); 51 | 52 | } 53 | 54 | public void setVersion(Semver semver){ 55 | this.semver = semver; 56 | monitoringUri = getVersionedQuery("monitoring-uri"); 57 | monitoringStartUri = getVersionedQuery("monitoring-start-scroll-uri"); 58 | monitoringScrollUri = getVersionedQuery("monitoring-scroll-uri"); 59 | } 60 | 61 | public String getMonitoringUri() { 62 | return monitoringUri; 63 | } 64 | 65 | public String getMonitoringScrollUri() { 66 | return monitoringScrollUri; 67 | } 68 | 69 | public String getMonitoringScrollTtl() { 70 | return monitoringScrollTtl; 71 | } 72 | 73 | public int getMonitoringScrollSize() { 74 | return monitoringScrollSize; 75 | } 76 | 77 | public List getMonitoringStats() { 78 | return monitoringStats; 79 | } 80 | 81 | public List getStatsByType(String type){ 82 | 83 | List stats = new ArrayList<>(); 84 | switch (type){ 85 | case "all" : 86 | stats.addAll(monitorSets); 87 | stats.addAll(logstashSets); 88 | stats.addAll(metricSets); 89 | return stats; 90 | case "metric" : 91 | return metricSets; 92 | default: 93 | stats.addAll(monitorSets); 94 | stats.addAll(logstashSets); 95 | return stats; 96 | } 97 | } 98 | 99 | private Map getQueriesFromFiles() { 100 | 101 | Map buildQueries = new LinkedHashMap<>(); 102 | for (String entry: queryFiles) { 103 | StringBuilder resultStringBuilder = new StringBuilder(); 104 | String path = Constants.QUERY_CONFIG_PACKAGE + entry + ".json"; 105 | InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(path); 106 | 107 | try { 108 | try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { 109 | String line; 110 | while ((line = br.readLine()) != null) { 111 | resultStringBuilder.append(line).append(SystemProperties.lineSeparator); 112 | } 113 | } 114 | } catch (IOException e) { 115 | logger.info(Constants.CONSOLE, "Failed to read query configuration file"); 116 | logger.info("Bad config", e); 117 | } 118 | 119 | buildQueries.put(entry, resultStringBuilder.toString()); 120 | 121 | } 122 | 123 | return buildQueries; 124 | 125 | } 126 | 127 | private String getVersionedQuery(String query ){ 128 | Map urls = (Map) configuration.get("monitoring-queries"); 129 | Map> querySet = urls.get(query); 130 | Map versions = querySet.get("versions"); 131 | 132 | for(Map.Entry urlVersion: versions.entrySet()){ 133 | if(semver.satisfies(urlVersion.getKey())){ 134 | return urlVersion.getValue(); 135 | } 136 | } 137 | 138 | return ""; 139 | 140 | } 141 | 142 | } 143 | 144 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringImportApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.diagnostics.ShowHelpException; 11 | import co.elastic.support.util.ResourceCache; 12 | import co.elastic.support.util.SystemUtils; 13 | import co.elastic.support.util.TextIOManager; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | 17 | import java.util.List; 18 | 19 | public class MonitoringImportApp { 20 | 21 | 22 | private static final Logger logger = LogManager.getLogger(MonitoringImportApp.class); 23 | 24 | public static void main(String[] args) { 25 | try( 26 | TextIOManager textIOManager = new TextIOManager(); 27 | ) { 28 | MonitoringImportInputs monitoringImportInputs = new MonitoringImportInputs(); 29 | if (args.length == 0) { 30 | logger.info(Constants.CONSOLE, Constants.interactiveMsg); 31 | monitoringImportInputs.interactive = true; 32 | monitoringImportInputs.runInteractive(textIOManager); 33 | } else { 34 | List errors = monitoringImportInputs.parseInputs(textIOManager, args); 35 | if (errors.size() > 0) { 36 | for (String err : errors) { 37 | logger.error(Constants.CONSOLE, err); 38 | } 39 | monitoringImportInputs.usage(); 40 | SystemUtils.quitApp(); 41 | } 42 | } 43 | new MonitoringImportService().execImport(monitoringImportInputs); 44 | } catch (ShowHelpException she){ 45 | SystemUtils.quitApp(); 46 | } catch (Exception e) { 47 | logger.error(Constants.CONSOLE, "Error occurred: {}. {}", e.getMessage(), Constants.CHECK_LOG); 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringImportConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | 10 | import co.elastic.support.BaseConfig; 11 | import org.apache.logging.log4j.LogManager; 12 | import org.apache.logging.log4j.Logger; 13 | 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | public class MonitoringImportConfig extends BaseConfig { 18 | 19 | protected int bulkSize ; 20 | 21 | public String monitoringExtractIndexPattern; 22 | public String logstashExtractIndexPattern; 23 | public String metricbeatExtractIndexPattern; 24 | 25 | public String metricbeatTemplate; 26 | public String logstashTemplate; 27 | public String esTemplate; 28 | 29 | public List templateList; 30 | 31 | Logger logger = LogManager.getLogger(MonitoringImportConfig.class); 32 | 33 | public MonitoringImportConfig(Map configuration) { 34 | super(configuration); 35 | bulkSize = (Integer) configuration.get("bulk-import-size"); 36 | monitoringExtractIndexPattern = (String)configuration.get("monitoring-extract-pattern"); 37 | logstashExtractIndexPattern = (String)configuration.get("logstash-extract-pattern"); 38 | metricbeatExtractIndexPattern = (String)configuration.get("metricbeat-extract-pattern"); 39 | templateList = (List)configuration.get("import-templates"); 40 | 41 | } 42 | 43 | 44 | } 45 | 46 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringImportInputs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | import co.elastic.support.util.TextIOManager; 10 | import com.beust.jcommander.Parameter; 11 | import co.elastic.support.rest.ElasticRestClientInputs; 12 | import org.apache.commons.lang3.ObjectUtils; 13 | import org.apache.commons.lang3.StringUtils; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | import org.beryx.textio.StringInputReader; 17 | 18 | import java.time.ZoneId; 19 | import java.time.ZonedDateTime; 20 | import java.time.format.DateTimeFormatter; 21 | import java.util.Collections; 22 | import java.util.List; 23 | 24 | public class MonitoringImportInputs extends ElasticRestClientInputs { 25 | 26 | private static final Logger logger = LogManager.getLogger(MonitoringImportInputs.class); 27 | 28 | // Start Input Fields 29 | 30 | @Parameter(names = {"--clustername"}, description = "Overrides the name of the imported cluster.") 31 | protected String clusterName; 32 | 33 | @Parameter(names = {"--targetsuffix"}, description = "Overrides the suffix of the import target from yyyy-MM-dd.") 34 | protected String targetSuffix = DateTimeFormatter.ofPattern("yyyy-MM-dd").format(ZonedDateTime.now(ZoneId.of("+0"))); 35 | 36 | @Parameter(names = {"-i", "--input"}, description = "Required: The archive that you wish to import into Elastic Monitoring. This must be in the format produced by the diagnostic export utility.") 37 | protected String input; 38 | 39 | // End Input Fields 40 | 41 | // Start Input Readers 42 | 43 | protected StringInputReader proxyHostReader; 44 | 45 | // End Input Readers 46 | 47 | public boolean runInteractive(TextIOManager textIOManager){ 48 | 49 | proxyHostReader = textIOManager.textIO.newStringInputReader() 50 | .withInputTrimming(true) 51 | .withValueChecker((String val, String propname) -> validateId(val)); 52 | 53 | clusterName = textIOManager.textIO.newStringInputReader() 54 | .withMinLength(0) 55 | .read("Specify an alternate name for the imported cluster or hit enter to use original cluster name:"); 56 | 57 | targetSuffix = textIOManager.textIO.newStringInputReader() 58 | .withInputTrimming(true) 59 | .withDefaultValue(targetSuffix) 60 | .read("Specify an alternate suffix for the import target or hit enter for the default generated name:"); 61 | 62 | input = textIOManager.textIO.newStringInputReader() 63 | .withInputTrimming(true) 64 | .withValueChecker((String val, String propname) -> validateRequiredFile(val)) 65 | .read("Enter the full path of the archvive you wish to import."); 66 | 67 | runHttpInteractive(textIOManager); 68 | 69 | return true; 70 | } 71 | 72 | @Override 73 | public List parseInputs(TextIOManager textIOManager, String[] args){ 74 | List errors = super.parseInputs(textIOManager, args); 75 | 76 | errors.addAll(ObjectUtils.defaultIfNull(validateId(clusterName), emptyList)); 77 | errors.addAll(ObjectUtils.defaultIfNull(validateId(targetSuffix), emptyList)); 78 | errors.addAll(ObjectUtils.defaultIfNull(validateRequiredFile(input), emptyList)); 79 | 80 | return errors; 81 | } 82 | 83 | public List validateId(String val){ 84 | if(StringUtils.isEmpty(val)){ 85 | return null; 86 | } 87 | if(val.contains(" ")){ 88 | return Collections.singletonList("Spaces not permitted in name."); 89 | } 90 | return null; 91 | } 92 | 93 | @Override 94 | public String toString() { 95 | return "MonitoringImportInputs{" + 96 | "clusterName='" + clusterName + '\'' + 97 | ", targetSuffix='" + targetSuffix + '\'' + 98 | ", input='" + input + '\'' + 99 | ", proxyHostReader=" + proxyHostReader + 100 | '}'; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/monitoring/MonitoringImportService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.monitoring; 8 | 9 | import co.elastic.support.diagnostics.commands.CheckElasticsearchVersion; 10 | import co.elastic.support.diagnostics.DiagnosticException; 11 | import co.elastic.support.rest.ElasticRestClientService; 12 | import co.elastic.support.Constants; 13 | import co.elastic.support.rest.RestClient; 14 | import co.elastic.support.util.ArchiveUtils; 15 | import co.elastic.support.util.JsonYamlUtils; 16 | import co.elastic.support.util.SystemProperties; 17 | import co.elastic.support.util.SystemUtils; 18 | import org.apache.commons.io.FileUtils; 19 | import org.apache.logging.log4j.LogManager; 20 | import org.apache.logging.log4j.Logger; 21 | import org.semver4j.Semver; 22 | 23 | import java.io.File; 24 | import java.util.Map; 25 | import java.util.Vector; 26 | 27 | public class MonitoringImportService extends ElasticRestClientService { 28 | private Logger logger = LogManager.getLogger(MonitoringImportService.class); 29 | 30 | void execImport(MonitoringImportInputs inputs) throws DiagnosticException { 31 | Map configMap = JsonYamlUtils.readYamlFromClasspath(Constants.DIAG_CONFIG, true); 32 | MonitoringImportConfig config = new MonitoringImportConfig(configMap); 33 | 34 | try (RestClient client = getClient(inputs, config)){ 35 | 36 | String tempDir = SystemProperties.userDir + SystemProperties.fileSeparator + Constants.MONITORING_DIR; 37 | String extractDir = tempDir + SystemProperties.fileSeparator +"extract"; 38 | 39 | // Create the temp directory - delete if first if it exists from a previous run 40 | SystemUtils.nukeDirectory(tempDir); 41 | logger.info(Constants.CONSOLE, "Creating temporary directory {}", tempDir); 42 | new File(extractDir).mkdirs(); 43 | 44 | // Set up the log file manually since we're going to package it with the diagnostic. 45 | // It will go to wherever we have the temp dir set up. 46 | logger.info(Constants.CONSOLE, "Configuring log file."); 47 | createFileAppender(tempDir, "import.log"); 48 | 49 | // Check the version. 50 | Semver version = CheckElasticsearchVersion.getElasticsearchVersion(client); 51 | if (version.getMajor() < 7) { 52 | throw new DiagnosticException("Target cluster must be at least 7.x"); 53 | } 54 | 55 | ArchiveUtils.extractArchive(inputs.input, extractDir); 56 | MonitoringImportProcessor processor = new MonitoringImportProcessor(config, inputs, client); 57 | processor.exec(getDirectoryEntries(extractDir)); 58 | 59 | }catch (Exception e){ 60 | logger.error( "Error extracting archive or indexing results", e); 61 | logger.info(Constants.CONSOLE, "Cannot continue processing. {} \n {}", e.getMessage(), Constants.CHECK_LOG); 62 | } 63 | finally { 64 | closeLogs(); 65 | } 66 | } 67 | 68 | private Vector getDirectoryEntries(String dir) { 69 | File targetDir = new File(dir); 70 | Vector files = new Vector<>(); 71 | files.addAll(FileUtils.listFiles(targetDir, null, true)); 72 | return files; 73 | } 74 | 75 | private RestClient getClient(MonitoringImportInputs inputs, MonitoringImportConfig config){ 76 | 77 | return RestClient.getClient( 78 | inputs.host, 79 | inputs.port, 80 | inputs.scheme, 81 | inputs.user, 82 | inputs.password, 83 | inputs.proxyHost, 84 | inputs.proxyPort, 85 | inputs.proxyUser, 86 | inputs.proxyPassword, 87 | inputs.pkiKeystore, 88 | inputs.pkiKeystorePass, 89 | inputs.skipVerification, 90 | config.extraHeaders, 91 | config.connectionTimeout, 92 | config.connectionRequestTimeout, 93 | config.socketTimeout 94 | ); 95 | 96 | } 97 | 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/rest/ElasticRestClientService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.rest; 8 | 9 | import co.elastic.support.util.SystemProperties; 10 | import co.elastic.support.util.SystemUtils; 11 | import co.elastic.support.BaseService; 12 | import co.elastic.support.Constants; 13 | import org.apache.commons.lang3.StringUtils; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | 17 | public class ElasticRestClientService extends BaseService { 18 | 19 | private static final Logger logger = LogManager.getLogger(ElasticRestClientInputs.class); 20 | 21 | protected void checkAuthLevel(String user, boolean isAuth){ 22 | 23 | if(StringUtils.isNotEmpty(user) && !isAuth){ 24 | String border = SystemUtils.buildStringFromChar(60, '*'); 25 | logger.info(Constants.CONSOLE, SystemProperties.lineSeparator); 26 | logger.info(Constants.CONSOLE, border); 27 | logger.info(Constants.CONSOLE, border); 28 | logger.info(Constants.CONSOLE, border); 29 | logger.info(Constants.CONSOLE, "The elasticsearch user entered: {} does not appear to have sufficient authorization to access all collected information", user); 30 | logger.info(Constants.CONSOLE, "Some of the calls may not have completed successfully."); 31 | logger.info(Constants.CONSOLE, "If you are using a custom role please verify that it has the admin role for versions prior to 5.x or the superuser role for subsequent versions."); 32 | logger.info(Constants.CONSOLE, border); 33 | logger.info(Constants.CONSOLE, border); 34 | logger.info(Constants.CONSOLE, border); 35 | } 36 | 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/rest/RestEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.rest; 8 | 9 | import lombok.Getter; 10 | 11 | @Getter 12 | public class RestEntry { 13 | public static final String MISSING = "missing"; 14 | 15 | private final String name; 16 | private final String url; 17 | private final String subdir; 18 | private final String extension; 19 | private final boolean retry; 20 | private final boolean showErrors; 21 | private final String pageableFieldName; 22 | private final boolean pageable; 23 | private final boolean spaceAware; 24 | 25 | public RestEntry(String name, String subdir, String extension, boolean retry, String url, boolean showErrors) { 26 | this(name, subdir, extension, retry, url, showErrors, null, false); 27 | } 28 | 29 | public RestEntry( 30 | String name, 31 | String subdir, 32 | String extension, 33 | boolean retry, 34 | String url, 35 | boolean showErrors, 36 | String pageableFieldName, 37 | boolean spaceAware 38 | ) { 39 | this.name = name; 40 | this.subdir = subdir; 41 | this.extension = extension; 42 | this.retry = retry; 43 | this.url = url; 44 | this.showErrors = showErrors; 45 | this.pageableFieldName = pageableFieldName; 46 | this.pageable = pageableFieldName != null; 47 | this.spaceAware = spaceAware; 48 | } 49 | 50 | public RestEntry copyWithNewUrl(String url, String subdir) { 51 | return new RestEntry(name, subdir, extension, retry, url, showErrors, pageableFieldName, spaceAware); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/rest/RestEntryConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.rest; 8 | 9 | import org.apache.commons.lang3.ObjectUtils; 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | import org.semver4j.Semver; 13 | 14 | import java.util.LinkedHashMap; 15 | import java.util.Map; 16 | 17 | public class RestEntryConfig { 18 | 19 | private static final Logger logger = LogManager.getLogger(RestEntryConfig.class); 20 | 21 | private final Semver semver; 22 | private final String mode; 23 | 24 | public RestEntryConfig(String version) { 25 | this(version, "full"); 26 | } 27 | 28 | public RestEntryConfig(String version, String mode) { 29 | this.semver = new Semver(version); 30 | this.mode = mode; 31 | } 32 | 33 | public Map buildEntryMap(Map config) { 34 | Map entries = new LinkedHashMap<>(); 35 | for (Map.Entry entry : config.entrySet()) { 36 | final String name = entry.getKey(); 37 | final RestEntry re = build(entry); 38 | 39 | if (re == null) { 40 | logger.warn("{} was bypassed due to mode", name); 41 | } else if (re.getUrl().equals(RestEntry.MISSING)) { 42 | logger.warn("{} was bypassed due to version check.", name); 43 | } else { 44 | entries.put(name, re); 45 | } 46 | } 47 | return entries; 48 | } 49 | 50 | @SuppressWarnings("unchecked") 51 | private RestEntry build(Map.Entry entry) { 52 | Map values = (Map) entry.getValue(); 53 | 54 | // only some diagnostics provide a mode (currently only Elasticsearch) 55 | // currently "tags" is a simple string, but if we ever need it to be an 56 | // array, then naturally this will need to change 57 | if ("full".equals(mode) == false && mode.equals(values.get("tags")) == false) { 58 | return null; 59 | } 60 | 61 | return buildRestEntryForVersion(entry.getKey(), values); 62 | } 63 | 64 | @SuppressWarnings("unchecked") 65 | private RestEntry buildRestEntryForVersion(String name, Map entry) { 66 | String subdir = (String) ObjectUtils.defaultIfNull(entry.get("subdir"), ""); 67 | String extension = (String) ObjectUtils.defaultIfNull(entry.get("extension"), ".json"); 68 | boolean retry = (boolean) ObjectUtils.defaultIfNull(entry.get("retry"), false); 69 | boolean showErrors = (boolean) ObjectUtils.defaultIfNull(entry.get("showErrors"), true); 70 | 71 | Map versions = (Map) entry.get("versions"); 72 | 73 | for (Map.Entry urlVersion : versions.entrySet()) { 74 | if (semver.satisfies(urlVersion.getKey())) { 75 | if (urlVersion.getValue() instanceof String) { 76 | return new RestEntry(name, subdir, extension, retry, (String) urlVersion.getValue(), showErrors); 77 | // We allow it to be String,String or String,Map(url,paginate,spaceaware) 78 | } else if (urlVersion.getValue() instanceof Map) { 79 | Map info = (Map) urlVersion.getValue(); 80 | 81 | String url = (String) ObjectUtils.defaultIfNull(info.get("url"), null); 82 | 83 | if (url == null) { 84 | throw new RuntimeException("Undefined URL for REST entry (route)"); 85 | } 86 | 87 | String pageableFieldName = (String) ObjectUtils.defaultIfNull(info.get("paginate"), null); 88 | boolean spaceAware = (boolean) ObjectUtils.defaultIfNull(info.get("spaceaware"), false); 89 | 90 | return new RestEntry(name, subdir, extension, retry, url, showErrors, pageableFieldName, spaceAware); 91 | } 92 | } 93 | } 94 | 95 | return null; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/rest/RestResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.rest; 8 | 9 | import co.elastic.support.util.SystemProperties; 10 | import co.elastic.support.Constants; 11 | import org.apache.commons.io.FileUtils; 12 | import org.apache.commons.io.IOUtils; 13 | import org.apache.commons.lang3.StringUtils; 14 | import org.apache.http.HttpResponse; 15 | import org.apache.http.client.utils.HttpClientUtils; 16 | import org.apache.http.util.EntityUtils; 17 | import org.apache.logging.log4j.LogManager; 18 | import org.apache.logging.log4j.Logger; 19 | 20 | import java.io.File; 21 | import java.io.FileOutputStream; 22 | import java.io.OutputStream; 23 | 24 | public class RestResult implements Cloneable { 25 | 26 | private static final Logger logger = LogManager.getLogger(RestResult.class); 27 | 28 | String responseString = "Undetermined error = check logs"; 29 | int status = -1; 30 | String reason; 31 | boolean isRetryable; 32 | String url = ""; 33 | 34 | // Sending in a response object to be processed implicitly 35 | // closes the response as a result. The body is either streamed directly 36 | // to disk or the body is stored as a string and the status retained as well. 37 | public RestResult(HttpResponse response, String url) { 38 | this.url = url; 39 | try { 40 | processCodes(response); 41 | responseString = EntityUtils.toString(response.getEntity()); 42 | } catch (Exception e) { 43 | logger.error("Error Processing Response", e); 44 | throw new RuntimeException(); 45 | } finally { 46 | HttpClientUtils.closeQuietly(response); 47 | } 48 | } 49 | 50 | public RestResult(HttpResponse response, String fileName, String url) { 51 | 52 | this.url = url; 53 | 54 | // If the query got a success status stream the result immediately to the target 55 | // file. 56 | // If not, the result should be small and contain diagnostic info so stgre it in 57 | // the response string. 58 | File output = new File(fileName); 59 | if (output.exists()) { 60 | FileUtils.deleteQuietly(output); 61 | } 62 | 63 | try (OutputStream out = new FileOutputStream(fileName)) { 64 | processCodes(response); 65 | response.getEntity().writeTo(out); 66 | } catch (Exception e) { 67 | logger.error("Error Streaming Response To OutputStream", e); 68 | throw new RuntimeException(); 69 | } finally { 70 | HttpClientUtils.closeQuietly(response); 71 | } 72 | } 73 | 74 | private void processCodes(HttpResponse response) { 75 | status = response.getStatusLine().getStatusCode(); 76 | if (status == 400) { 77 | reason = "Bad Request. Rejected"; 78 | isRetryable = true; 79 | } else if (status == 401) { 80 | reason = "Authentication failure. Invalid login credentials."; 81 | isRetryable = false; 82 | } else if (status == 403) { 83 | reason = "Authorization failure or invalid license."; 84 | isRetryable = false; 85 | } else if (status == 404) { 86 | reason = "Endpoint does not exist."; 87 | isRetryable = true; 88 | } else { 89 | reason = response.getStatusLine().getReasonPhrase(); 90 | isRetryable = true; 91 | } 92 | } 93 | 94 | public String formatStatusMessage(String msg) { 95 | 96 | if (StringUtils.isNotEmpty(msg)) { 97 | msg = msg + " "; 98 | } 99 | 100 | return String.format("%sStatus: %d Reason: %s", 101 | msg, 102 | status, 103 | reason); 104 | } 105 | 106 | public int getStatus() { 107 | return status; 108 | } 109 | 110 | public String getReason() { 111 | return reason; 112 | } 113 | 114 | public String toString() { 115 | return responseString; 116 | } 117 | 118 | public void toFile(String fileName) { 119 | File output = new File(fileName); 120 | if (output.exists()) { 121 | FileUtils.deleteQuietly(output); 122 | } 123 | 124 | try (FileOutputStream fs = new FileOutputStream(output)) { 125 | IOUtils.write(reason + SystemProperties.lineSeparator + responseString, fs, Constants.UTF_8); 126 | } catch (Exception e) { 127 | logger.error("Error writing Response To OutputStream", e); 128 | } 129 | } 130 | 131 | public boolean isRetryable() { 132 | return isRetryable; 133 | } 134 | 135 | public boolean isValid() { 136 | if (status == 200) { 137 | return true; 138 | } 139 | return false; 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/ScrubApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | import co.elastic.support.diagnostics.commands.GenerateManifest; 10 | import co.elastic.support.Constants; 11 | import co.elastic.support.diagnostics.ShowHelpException; 12 | import co.elastic.support.util.ResourceCache; 13 | import co.elastic.support.util.SystemUtils; 14 | import co.elastic.support.util.TextIOManager; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | 18 | import java.util.List; 19 | 20 | 21 | public class ScrubApp { 22 | 23 | private static Logger logger = LogManager.getLogger(ScrubApp.class); 24 | 25 | public static void main(String[] args) { 26 | try( 27 | TextIOManager textIOManager = new TextIOManager(); 28 | ) { 29 | ScrubInputs scrubInputs = new ScrubInputs(); 30 | if (args.length == 0) { 31 | logger.error(Constants.CONSOLE, Constants.interactiveMsg); 32 | scrubInputs.interactive = true; 33 | scrubInputs.runInteractive(textIOManager); 34 | } else { 35 | List errors = scrubInputs.parseInputs(textIOManager, args); 36 | if (errors.size() > 0) { 37 | for (String err : errors) { 38 | logger.error(Constants.CONSOLE, err); 39 | } 40 | scrubInputs.usage(); 41 | SystemUtils.quitApp(); 42 | } 43 | } 44 | logger.info(Constants.CONSOLE, "Using version: {} of diagnostic-utiliy", GenerateManifest.class.getPackage().getImplementationVersion()); 45 | new ScrubService().exec(scrubInputs); 46 | } catch (ShowHelpException she){ 47 | SystemUtils.quitApp(); 48 | } catch (Exception e) { 49 | logger.error(Constants.CONSOLE, "FATAL ERROR occurred: {}. {}", e.getMessage(), Constants.CHECK_LOG, e); 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/ScrubConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | 13 | import java.util.Vector; 14 | 15 | public class ScrubConfig { 16 | 17 | private static final Logger logger = LogManager.getLogger((ScrubConfig.class)); 18 | 19 | private Vector remove = new Vector(); 20 | private Vector tokens = new Vector<>(); 21 | private Vector autoScrub = new Vector<>(); 22 | 23 | public ScrubConfig(){ 24 | 25 | 26 | 27 | 28 | 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/ScrubInputs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | import co.elastic.support.BaseInputs; 10 | import co.elastic.support.util.ResourceCache; 11 | import co.elastic.support.util.SystemProperties; 12 | import co.elastic.support.util.TextIOManager; 13 | import com.beust.jcommander.Parameter; 14 | import co.elastic.support.Constants; 15 | import org.apache.commons.lang3.StringUtils; 16 | import org.apache.logging.log4j.LogManager; 17 | import org.apache.logging.log4j.Logger; 18 | 19 | import java.io.File; 20 | import java.util.Collections; 21 | import java.util.List; 22 | 23 | public class ScrubInputs extends BaseInputs { 24 | 25 | private static final Logger logger = LogManager.getLogger(ScrubInputs.class); 26 | 27 | // Start Input Fields 28 | @Parameter( 29 | names = { "-i", "--input" }, 30 | description = "Required field. Full path to the archive file, directory, or individual file to be scrubbed." 31 | ) 32 | public String scrub; 33 | 34 | @Parameter( 35 | names = { "--workers" }, 36 | description = "Optional field. How many processing instances to run. Defaults to the number of detected cores." 37 | ) 38 | public int workers = Runtime.getRuntime().availableProcessors(); 39 | 40 | // End Input Fields 41 | 42 | public String type = "zip"; 43 | public boolean isArchive = true; 44 | public String scrubbedFileBaseName; 45 | 46 | public boolean runInteractive(TextIOManager textIOManager) { 47 | 48 | scrub = textIOManager.textIO.newStringInputReader() 49 | .withInputTrimming(true) 50 | .withValueChecker((String val, String propname) -> validateScrubInput(val)) 51 | .read("Enter the full path of the archive you wish to import."); 52 | 53 | workers = textIOManager.textIO.newIntInputReader() 54 | .withMinVal(0) 55 | .withDefaultValue(workers) 56 | .read("Enter the number of workers to run in parallel. Defaults to the detected number of processors: " + workers); 57 | 58 | logger.info(Constants.CONSOLE, ""); 59 | logger.info( 60 | Constants.CONSOLE, 61 | "The utility will obfuscate IP and MAC addresses by default. You do NOT need to configure that functionality." 62 | ); 63 | logger.info(Constants.CONSOLE, "If you wish to extend for additional masking you MUST explicitly enter a file to input."); 64 | logger.info(Constants.CONSOLE, "Note that for docker containers this must be a file within the configured volume."); 65 | 66 | if (runningInDocker) { 67 | logger.info(Constants.CONSOLE, "Result will be written to the configured Docker volume."); 68 | } else { 69 | runOutputDirInteractive(textIOManager); 70 | } 71 | 72 | return true; 73 | } 74 | 75 | @Override 76 | public List parseInputs(TextIOManager textIOManager, String[] args) { 77 | List errors = super.parseInputs(textIOManager, args); 78 | 79 | List valid = validateScrubInput(scrub); 80 | if (valid != null) { 81 | errors.addAll(valid); 82 | } 83 | 84 | return errors; 85 | } 86 | 87 | public List validateScrubInput(String path) { 88 | if (StringUtils.isEmpty(path.trim())) { 89 | return Collections.singletonList("Input archive, directory, or single file is required."); 90 | } 91 | 92 | File file = new File(path); 93 | 94 | if (!file.exists()) { 95 | return Collections.singletonList("Specified required file location could not be located or is a directory."); 96 | } 97 | int filePosition = path.lastIndexOf(SystemProperties.fileSeparator); 98 | 99 | if (path.endsWith(".zip")) { 100 | this.type = "zip"; 101 | scrubbedFileBaseName = path.substring(filePosition + 1).replace(".zip", ""); 102 | } else if (path.endsWith(".tar.gz")) { 103 | this.type = "tar.gz"; 104 | scrubbedFileBaseName = path.substring(filePosition + 1).replace(".tar.gz", ""); 105 | } else if (path.endsWith(".tar")) { 106 | this.type = "tar"; 107 | scrubbedFileBaseName = path.substring(filePosition + 1).replace(".tar", ""); 108 | } else if (file.isDirectory()) { 109 | this.type = "dir"; 110 | isArchive = false; 111 | scrubbedFileBaseName = path.substring(filePosition + 1); 112 | } else { 113 | this.type = "file"; 114 | isArchive = false; 115 | scrubbedFileBaseName = path.substring(filePosition + 1, path.lastIndexOf(".")); 116 | } 117 | 118 | return null; 119 | 120 | } 121 | 122 | @Override 123 | public String toString() { 124 | return "ScrubInputs{" + "input='" + scrub + '\'' + '}'; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/ScrubTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | import co.elastic.support.Constants; 10 | import co.elastic.support.util.SystemProperties; 11 | import co.elastic.support.util.TaskEntry; 12 | import org.apache.commons.io.FileUtils; 13 | import org.apache.logging.log4j.LogManager; 14 | import org.apache.logging.log4j.Logger; 15 | 16 | import java.io.File; 17 | import java.util.concurrent.Callable; 18 | 19 | public class ScrubTask implements Callable { 20 | 21 | private static Logger logger = LogManager.getLogger(ScrubTask.class); 22 | ScrubProcessor processor; 23 | TaskEntry entry; 24 | String dir; 25 | 26 | public ScrubTask(ScrubProcessor processor, TaskEntry entry, String dir){ 27 | this.entry = entry; 28 | this.processor = processor; 29 | this.dir = dir; 30 | } 31 | 32 | @Override 33 | public String call() { 34 | String result; 35 | try { 36 | logger.debug(entry.entryName() + " started"); 37 | 38 | // If it's in remove we not only don't process it we don't write it to the scrubbed archive either 39 | if (processor.isRemove(entry.entryName())) { 40 | logger.info(Constants.CONSOLE, "Removing entry: {}", entry.entryName()); 41 | return entry.entryName() + ":removed"; 42 | } 43 | 44 | String content = entry.content(); 45 | 46 | if (processor.isExclude(entry.entryName())) { 47 | result = entry.entryName() + ":excluded"; 48 | logger.info(Constants.CONSOLE, "Excluded from sanitization: {}", entry.entryName()); 49 | 50 | } else { 51 | content = processor.processAutoscrub(content); 52 | content = processor.processContentWithTokens(content, entry.entryName()); 53 | result = entry.entryName() + ":sanitized"; 54 | logger.info(Constants.CONSOLE, "Processed entry: {}", entry.entryName()); 55 | 56 | } 57 | 58 | String targetFileName = dir + SystemProperties.fileSeparator + entry.entryName(); 59 | FileUtils.writeStringToFile(new File(targetFileName), content, "UTF-8"); 60 | 61 | } catch (Exception e) { 62 | logger.error("Error occurrred processing: {}", entry.entryName(), e); 63 | result = "error:" + entry.entryName() + " " + e.getMessage(); 64 | } 65 | 66 | logger.debug(entry.entryName() + " complete"); 67 | 68 | return result; 69 | 70 | } 71 | 72 | @Override 73 | public String toString() { 74 | return super.toString() + entry.entryName() + ":incomplete"; 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/ScrubTokenEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | import java.util.List; 10 | import java.util.regex.Pattern; 11 | 12 | public class ScrubTokenEntry { 13 | 14 | public ScrubTokenEntry(String token, List include, List exclude){ 15 | this.token = token; 16 | this.include = include; 17 | this.exclude = exclude; 18 | this.pattern = Pattern.compile(token); 19 | } 20 | public final String token; 21 | public final List include; 22 | public final List exclude; 23 | public final Pattern pattern; 24 | 25 | @Override 26 | public String toString() { 27 | return "\nScrubTokenEntry{" + 28 | "token='" + token + '\'' + 29 | ", include=" + include + 30 | ", exclude=" + exclude + 31 | ", pattern=" + pattern + 32 | '}'; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/scrub/TokenGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.scrub; 8 | 9 | public interface TokenGenerator { 10 | 11 | public String generate(String input); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/ArchiveEntryProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import org.apache.commons.compress.archivers.zip.ZipFile; 10 | import java.io.InputStream; 11 | 12 | public interface ArchiveEntryProcessor { 13 | public void process(InputStream is, String name); 14 | public void init(ZipFile zipFile); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/FileTaskEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import org.apache.commons.io.FileUtils; 10 | import org.apache.commons.io.IOUtils; 11 | import org.apache.logging.log4j.LogManager; 12 | import org.apache.logging.log4j.Logger; 13 | 14 | import java.io.File; 15 | import java.io.FileInputStream; 16 | import java.io.IOException; 17 | import java.util.zip.GZIPInputStream; 18 | 19 | public class FileTaskEntry implements TaskEntry { 20 | private File file; 21 | private String rootDir; 22 | private static final Logger logger = LogManager.getLogger(FileTaskEntry.class); 23 | 24 | public FileTaskEntry(File file, String rootDir){ 25 | this.rootDir = rootDir; 26 | this.file = file; 27 | } 28 | 29 | @Override 30 | public String content() { 31 | String contents = ""; 32 | try { 33 | if(file.getName().endsWith(".gz")){ 34 | GZIPInputStream gzi = new GZIPInputStream( new FileInputStream(file)); 35 | return IOUtils.toString(gzi, "UTF-8"); 36 | } 37 | 38 | return FileUtils.readFileToString(file, "UTF-8"); 39 | } catch (IOException e) { 40 | logger.error("Error retrieving file: {}", file.getName(), e); 41 | } 42 | 43 | return file.getName() + ":error"; 44 | } 45 | 46 | @Override 47 | public String entryName() { 48 | String name = file.getAbsolutePath().replaceFirst(rootDir + SystemProperties.fileSeparator, ""); 49 | return name.replaceFirst(".gz", ""); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/JsonYamlUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import co.elastic.support.diagnostics.DiagnosticException; 10 | import com.fasterxml.jackson.databind.JsonNode; 11 | import com.fasterxml.jackson.databind.ObjectMapper; 12 | import org.apache.commons.io.FileUtils; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import org.yaml.snakeyaml.DumperOptions; 16 | import org.yaml.snakeyaml.Yaml; 17 | 18 | import java.io.File; 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.util.*; 22 | 23 | public class JsonYamlUtils { 24 | 25 | private static final Logger logger = LoggerFactory.getLogger(JsonYamlUtils.class); 26 | 27 | public static ObjectMapper mapper = new ObjectMapper(); 28 | 29 | public static JsonNode createJsonNodeFromFileName(String dir, String fileName) { 30 | File jsonFile = FileUtils.getFile(dir, fileName); 31 | 32 | try { 33 | String fileString = FileUtils.readFileToString(jsonFile, "UTF8"); 34 | 35 | return JsonYamlUtils.createJsonNodeFromString(fileString); 36 | } catch (IOException e) { 37 | logger.info("Error reading in JSON string from file: {}", jsonFile); 38 | throw new RuntimeException(e); 39 | } 40 | } 41 | 42 | public static JsonNode createJsonNodeFromString(String nodeString) { 43 | try { 44 | return new ObjectMapper().readTree(nodeString); 45 | } catch (IOException e) { 46 | logger.info("Error creating JSON node from input string: {}", nodeString); 47 | throw new RuntimeException(e); 48 | } 49 | } 50 | 51 | public static Map readYamlFromClasspath(String path, boolean isBlock) throws DiagnosticException { 52 | try ( 53 | InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(path); 54 | ) { 55 | return JsonYamlUtils.readYaml(inputStream, isBlock); 56 | } 57 | catch (IOException e) { 58 | logger.info("Error reading YAML from {}", path); 59 | throw new DiagnosticException("Error reading YAML file",e); 60 | } 61 | } 62 | 63 | private static Map readYaml(InputStream in, boolean isBlock) { 64 | DumperOptions options = new DumperOptions(); 65 | 66 | if (isBlock) { 67 | options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); 68 | } 69 | 70 | Yaml yaml = new Yaml(options); 71 | Map doc = yaml.load(in); 72 | 73 | if (doc == null){ 74 | return new HashMap<>(); 75 | } 76 | 77 | return doc; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/LocalSystem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import co.elastic.support.Constants; 10 | import org.apache.commons.io.FileUtils; 11 | import org.apache.logging.log4j.LogManager; 12 | import org.apache.logging.log4j.Logger; 13 | 14 | import java.io.*; 15 | 16 | import java.util.List; 17 | 18 | 19 | public class LocalSystem extends SystemCommand { 20 | 21 | Logger logger = LogManager.getLogger(LocalSystem.class); 22 | private ProcessBuilder pb; 23 | private static final String[] wincCmds = {"cmd", "/c"}; 24 | private static final String[] nixcCdms = {"/bin/sh", "-c"}; 25 | private String encoding = Constants.UTF_8; 26 | 27 | public LocalSystem(String osName) { 28 | 29 | this.osName = osName; 30 | 31 | switch (osName) { 32 | case Constants.linuxPlatform: 33 | case Constants.macPlatform: 34 | pb = new ProcessBuilder(nixcCdms); 35 | break; 36 | case Constants.winPlatform: 37 | pb = new ProcessBuilder(wincCmds); 38 | // Windows hack - wmmic uses UTF-16 39 | encoding = Constants.UTF_16; 40 | break; 41 | default: 42 | pb = new ProcessBuilder(nixcCdms); 43 | logger.info("Unrecognized OS: {} - using Linux as default.", osName); 44 | } 45 | pb.redirectErrorStream(true); 46 | 47 | } 48 | 49 | public String runCommand(String cmd) { 50 | 51 | StringBuffer sb = new StringBuffer(); 52 | 53 | try { 54 | List current = pb.command(); 55 | if (current.size() == 2) { 56 | current.add(cmd); 57 | } else { 58 | current.set(2, cmd); 59 | } 60 | 61 | pb.redirectErrorStream(true); 62 | Process pr = pb.start(); 63 | BufferedReader reader = 64 | new BufferedReader(new InputStreamReader(pr.getInputStream())); 65 | String line; 66 | while ((line = reader.readLine()) != null) { 67 | sb.append(line + SystemProperties.lineSeparator); 68 | } 69 | pr.waitFor(); 70 | 71 | } catch (Exception e) { 72 | logger.info("Error encountered running {}", cmd); 73 | logger.error( "System command error", e); 74 | } 75 | 76 | return sb.toString(); 77 | } 78 | 79 | 80 | @Override 81 | public void copyLogs(List entries, String logDir, String targetDir) { 82 | 83 | for(String entry: entries){ 84 | try { 85 | String source = logDir + SystemProperties.fileSeparator + entry; 86 | String target = targetDir + SystemProperties.fileSeparator + entry; 87 | FileUtils.copyFile(new File(source), new File(target)); 88 | } catch (IOException e) { 89 | logger.info("Error retrieving log: {}. Bypassing.", entry); 90 | logger.error( e); 91 | } 92 | } 93 | } 94 | 95 | /** 96 | * On this function we will try to collect the journalctl logs, 97 | * Some services as Kibana installed with the RPM package will give the access to the logs using the journalctl command 98 | * 99 | * @param serviceName service name defined by RPM 100 | * @param targetDir temporary path where the data need to be stored 101 | */ 102 | @Override 103 | public void copyLogsFromJournalctl(String serviceName, String targetDir) { 104 | 105 | String tempDir = "templogs"; 106 | String mkdir = "mkdir templogs"; 107 | String journalctl = "journalctl -u {{SERVICE}} > '{{TEMP}}/{{SERVICE}}.log'"; 108 | journalctl = journalctl.replace("{{SERVICE}}", serviceName); 109 | journalctl = journalctl.replace("{{TEMP}}", tempDir); 110 | 111 | try { 112 | runCommand(mkdir); 113 | runCommand(journalctl); 114 | String source = "{{TEMP}}/{{SERVICE}}.log"; 115 | source = source.replace("{{SERVICE}}", serviceName); 116 | source = source.replace("{{TEMP}}", tempDir); 117 | String target = targetDir + SystemProperties.fileSeparator + serviceName; 118 | FileUtils.copyFile(new File(source), new File(target)); 119 | // clean up the temp logs on the remote host 120 | runCommand("rm -Rf templogs"); 121 | } catch (IOException e) { 122 | logger.info("Error retrieving log: {}. Bypassing.", serviceName); 123 | logger.error( e); 124 | } 125 | } 126 | 127 | 128 | @Override 129 | public void close() throws IOException { 130 | // Nothing to do for this one. 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/RemoteUserInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import com.jcraft.jsch.UserInfo; 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | 13 | public class RemoteUserInfo implements UserInfo { 14 | 15 | private static final Logger logger = LogManager.getLogger(RemoteUserInfo.class); 16 | 17 | public RemoteUserInfo( String name, String password, String passphrase){ 18 | 19 | this.password = password; 20 | this.passphrase = passphrase; 21 | 22 | } 23 | 24 | private String password = null; 25 | public String getPassword() { 26 | return password; 27 | } 28 | 29 | private String passphrase = null; 30 | public String getPassphrase() { 31 | return passphrase; 32 | } 33 | 34 | public boolean promptPassphrase(String message) { 35 | return true; 36 | } 37 | 38 | public boolean promptPassword(String passwordPrompt) { 39 | return true; 40 | } 41 | 42 | public boolean promptYesNo(String message) { 43 | return true; 44 | } 45 | 46 | public void showMessage(String message) { 47 | logger.debug(message); 48 | } 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/ResourceCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import co.elastic.support.rest.RestClient; 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | 13 | import java.io.Closeable; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | 16 | public class ResourceCache implements AutoCloseable { 17 | 18 | private final Logger logger = LogManager.getLogger(ResourceCache.class); 19 | private ConcurrentHashMap resources = new ConcurrentHashMap<>(); 20 | 21 | public void addSystemCommand(String name, SystemCommand systemCommand) { 22 | // Log the error if they tried to overlay with a dupe but don't throw an 23 | // exception. 24 | resources.putIfAbsent(name, systemCommand); 25 | } 26 | 27 | public SystemCommand getSystemCommand(String name) { 28 | if (resources.containsKey(name)) { 29 | return (SystemCommand) resources.get(name); 30 | } 31 | 32 | throw new IllegalStateException("SystemCommand instance requested does not exist"); 33 | } 34 | 35 | public void addRestClient(String name, RestClient client) { 36 | resources.putIfAbsent(name, client); 37 | } 38 | 39 | public RestClient getRestClient(String name) { 40 | if (resources.containsKey(name)) { 41 | return (RestClient) resources.get(name); 42 | } 43 | throw new IllegalStateException("RestClient instance does not exist"); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | resources.forEach((name, resource) -> { 49 | try { 50 | resource.close(); 51 | } catch (Exception e) { 52 | logger.error("Failed to close resource {}", name); 53 | } 54 | }); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/SystemCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | 10 | import java.io.Closeable; 11 | import java.util.List; 12 | 13 | public abstract class SystemCommand implements Closeable { 14 | 15 | public String osName; 16 | public abstract String runCommand(String command); 17 | public abstract void copyLogs(List entries, String logDir, String targetDir); 18 | public abstract void copyLogsFromJournalctl(String serviceName, String targetDir); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/SystemProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | 10 | import org.apache.logging.log4j.Level; 11 | 12 | import java.text.SimpleDateFormat; 13 | import java.util.Date; 14 | import java.util.TimeZone; 15 | 16 | public class SystemProperties { 17 | 18 | public static Level DIAG = Level.forName("DIAG", 250); 19 | public static Level CONSOLE = Level.forName("CONSOLE", 50); 20 | 21 | public static final String osName = System.getProperty("os.name"); 22 | 23 | public static final String javaHome = System.getenv("JAVA_HOME"); 24 | 25 | public static final String pathSeparator = System.getProperty("path.separator"); 26 | 27 | public static final String fileSeparator = System.getProperty("file.separator"); 28 | 29 | public static final String lineSeparator = System.getProperty("line.separator"); 30 | 31 | public static final String userDir = System.getProperty("user.dir"); 32 | 33 | public static final String userHome = System.getProperty("user.home"); 34 | 35 | public static final String UTC_DATE_FORMAT = "yyyy-MM-dd"; 36 | 37 | public static final String UTC_DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSX"; 38 | 39 | public static final String FILE_DATE_FORMAT = "yyyyMMdd-HHmmss"; 40 | 41 | public static String getUtcDateString() { 42 | Date curDate = new Date(); 43 | SimpleDateFormat format = new SimpleDateFormat(UTC_DATE_FORMAT); 44 | format.setTimeZone(TimeZone.getTimeZone("UTC")); 45 | return format.format(curDate); 46 | } 47 | 48 | public static String getUtcDateTimeString() { 49 | Date curDate = new Date(); 50 | SimpleDateFormat format = new SimpleDateFormat(UTC_DATE_TIME_FORMAT); 51 | format.setTimeZone(TimeZone.getTimeZone("UTC")); 52 | return format.format(curDate); 53 | } 54 | 55 | public static String getFileDateString() { 56 | Date curDate = new Date(); 57 | SimpleDateFormat format = new SimpleDateFormat(FILE_DATE_FORMAT); 58 | format.setTimeZone(TimeZone.getTimeZone("UTC")); 59 | return format.format(curDate); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/TaskEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | public interface TaskEntry { 10 | public String content(); 11 | public String entryName(); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/TextIOManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | /* 9 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 10 | * or more contributor license agreements. Licensed under the Elastic License; 11 | * you may not use this file except in compliance with the Elastic License. 12 | */ 13 | import jline.console.ConsoleReader; 14 | import org.apache.commons.lang3.StringUtils; 15 | import org.beryx.textio.*; 16 | import org.beryx.textio.jline.JLineTextTerminal; 17 | 18 | import java.io.File; 19 | import java.util.Collections; 20 | import java.util.List; 21 | 22 | public class TextIOManager implements AutoCloseable { 23 | 24 | public TextIO textIO; 25 | 26 | public StringInputReader standardStringReader; 27 | public BooleanInputReader standardBooleanReader; 28 | public StringInputReader standardPasswordReader; 29 | public StringInputReader standardFileReader; 30 | 31 | public TextIOManager() { 32 | textIO = TextIoFactory.getTextIO(); 33 | TextTerminal terminal = textIO.getTextTerminal(); 34 | 35 | if(terminal instanceof JLineTextTerminal){ 36 | JLineTextTerminal jltt = (JLineTextTerminal)terminal; 37 | ConsoleReader reader = jltt.getReader(); 38 | reader.setExpandEvents(false); 39 | } 40 | 41 | // Input Readers 42 | // Generic - change the read label only 43 | // Warning: Setting default values may leak into later prompts if not reset. Better to use a new Reader. 44 | standardStringReader = textIO.newStringInputReader() 45 | .withMinLength(0) 46 | .withInputTrimming(true); 47 | standardBooleanReader = textIO.newBooleanInputReader(); 48 | standardPasswordReader = textIO.newStringInputReader() 49 | .withInputMasking(true) 50 | .withInputTrimming(true) 51 | .withMinLength(0); 52 | standardFileReader = textIO.newStringInputReader() 53 | .withInputTrimming(true) 54 | .withValueChecker((String val, String propname) -> validateFile(val)); 55 | // End Input Readers 56 | } 57 | 58 | static public List validateFile(String val) { 59 | if (StringUtils.isEmpty(val.trim())) { 60 | return null; 61 | } 62 | 63 | File file = new File(val); 64 | 65 | if (!file.exists()) { 66 | return Collections.singletonList( 67 | String.format("Specified file [%s] could not be located.", file.getPath()) 68 | ); 69 | } 70 | 71 | return null; 72 | } 73 | 74 | @Override 75 | public void close() { 76 | textIO.dispose(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/UrlUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import java.net.URLEncoder; 10 | import java.nio.charset.StandardCharsets; 11 | import java.io.UnsupportedEncodingException; 12 | 13 | /** 14 | * {@code UrlUtils} contains helpful methods for dealing with URLs. 15 | */ 16 | public class UrlUtils { 17 | /** 18 | * URL Encode the {@code value}. 19 | * 20 | * @param value The value to URL encode. 21 | * @return Never {@code null}. 22 | * @throws RuntimeException if encoding throws an exception. 23 | */ 24 | public static String encodeValue(String value) { 25 | try { 26 | return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); 27 | } catch (UnsupportedEncodingException e) { 28 | throw new RuntimeException(e); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/co/elastic/support/util/ZipFileTaskEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.util; 8 | 9 | import co.elastic.support.Constants; 10 | import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; 11 | import org.apache.commons.compress.archivers.zip.ZipFile; 12 | import org.apache.commons.io.IOUtils; 13 | import org.apache.logging.log4j.LogManager; 14 | import org.apache.logging.log4j.Logger; 15 | import java.util.zip.GZIPInputStream; 16 | 17 | public class ZipFileTaskEntry implements TaskEntry { 18 | 19 | private static final Logger logger = LogManager.getLogger(ZipFileTaskEntry.class); 20 | private ZipFile zipFile; 21 | private ZipArchiveEntry zipEntry; 22 | private String archiveName; 23 | 24 | public ZipFileTaskEntry(ZipFile zipFile, ZipArchiveEntry zipEntry, String archiveName){ 25 | this.zipEntry = zipEntry; 26 | this .zipFile = zipFile; 27 | this.archiveName = archiveName; 28 | } 29 | 30 | @Override 31 | public String content() { 32 | 33 | String contents = ""; 34 | try { 35 | if(zipEntry.getName().endsWith(".gz")){ 36 | GZIPInputStream gzi = new GZIPInputStream(zipFile.getInputStream(zipEntry)); 37 | contents = IOUtils.toString(gzi, "UTF-8"); 38 | } 39 | else { 40 | contents = IOUtils.toString(zipFile.getInputStream(zipEntry), "UTF-8"); 41 | } 42 | 43 | } catch (Exception e) { 44 | logger.error(Constants.CONSOLE, "Could not extract entry {}", zipEntry.getName()); 45 | } 46 | 47 | return contents; 48 | 49 | } 50 | 51 | @Override 52 | public String entryName() { 53 | String name = zipEntry.getName().replaceFirst(archiveName, ""); 54 | return name.replace(".gz", ""); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/resources/diags.yml: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 4 | # 5 | # The REST call settings have been moved to elastic-rest.yml and logstash-rest.yml 6 | # 7 | # 8 | # 9 | # 10 | 11 | ## Should only need to change these if Github modifies something 12 | ## github-settings: 13 | ## diagReleaseHost: "api.github.com" 14 | ## diagReleaseDest: "/repos/elastic/support-diagnostics/releases/latest" 15 | ## diagReleaseScheme: "https" 16 | ## diagLatestRelease: "https://api.github.com/repos/elastic/support-diagnostics/releases/latest" 17 | 18 | # Maximum number of log files including zip archives 19 | log-settings: 20 | maxLogs: 3 21 | maxGcLogs: 3 22 | 23 | # Uncomment only if modifying defaults 24 | rest-config: 25 | # timeouts in seconds 26 | requestTimeout: 60 27 | connectTimeout: 60 28 | socketTimeout: 120 29 | maxTotalConn: 100 30 | maxConnPerRoute: 10 31 | 32 | # Number of tiems to re-attempt a rest call 33 | call-retries: 3 34 | 35 | # Time before re-attempts in milliseconnds 36 | pause-retries: 5000 37 | 38 | thread-dump: 39 | jstack: "jstack {{PID}}" 40 | 41 | linuxOS: 42 | sys: 43 | top: "top -b -n1" 44 | netstat: "netstat -an" 45 | ss: "ss -an" 46 | process-list: "ps -ef" 47 | top_threads: "top -b -n1 -H" 48 | uname: "uname -a -r" 49 | cpu: "cat /proc/cpuinfo" 50 | iostat: "iostat -c -d -x -t -m 1 5" 51 | sar: "sar -A" 52 | sysctl: "sysctl -a" 53 | dmesg: "dmesg" 54 | dmesg_t: "dmesg -T" 55 | huge_pages: "cat /sys/kernel/mm/transparent_hugepage/enabled" 56 | cpu_governor: "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" 57 | limits: "cat /etc/security/limits.conf" 58 | proc-limit: "cat /proc/{{PID}}/limits" 59 | readahead: "lsblk -o NAME,RA,MOUNTPOINT,TYPE,SIZE" 60 | 61 | java: 62 | elastic-java: "ps -fp {{PID}} | awk '{ if (NR!=1) print $8}'" 63 | jps: "{{JAVA_HOME}}/bin/jps -l -m -v" 64 | jstack: "{{JAVA_HOME}}/bin/jstack {{PID}}" 65 | javac: "{{JAVA_HOME}}/bin/javac -version" 66 | 67 | logs: 68 | elastic: "ls -alt {{LOGPATH}} | grep -e '{{CLUSTERNAME}}.log' -e '{{CLUSTERNAME}}_server.json' | awk '{print $9}'" 69 | elastic-arc: "ls -alt {{LOGPATH}} | grep '{{CLUSTERNAME}}-.*-1.log.gz' | awk '{print $9}'" 70 | gc: "ls -alt {{LOGPATH}} | awk '/gc/ {print $9}'" 71 | kibana: "ls -alt /var/log/kibana/ | awk '/.json|.log/ {print $9}'" 72 | kibana-default-path: "/var/log/kibana/" 73 | 74 | macOS: 75 | sys: 76 | top: "top -l 1" 77 | netstat: "netstat -an" 78 | process-list: "ps -ef" 79 | ulimit: "ulimit -a" 80 | 81 | java: 82 | elastic-java: "ps -fp {{PID}} | awk '{ if (NR!=1) print $8}'" 83 | jps: "{{JAVA_HOME}}/bin/jps -l -m -v" 84 | jstack: "{{JAVA_HOME}}/bin/jstack {{PID}}" 85 | javac: "{{JAVA_HOME}}/bin/javac -version" 86 | 87 | logs: 88 | elastic: "ls -alt {{LOGPATH}} | grep -e '{{CLUSTERNAME}}.log' -e '{{CLUSTERNAME}}_server.json' | awk '{print $9}'" 89 | elastic-arc: "ls -alt {{LOGPATH}} | grep '{{CLUSTERNAME}}-.*-1.log.gz' | awk '{print $9}'" 90 | gc: "ls -alt {{LOGPATH}} | awk '/gc/ {print $9}'" 91 | kibana: "ls -alt /var/log/kibana/ | awk '/.json|.log/ {print $9}'" 92 | kibana-default-path: "/var/log/kibana/" 93 | 94 | winOS: 95 | sys: 96 | process-list: "tasklist /v" 97 | netstat: "netstat -ano" 98 | cpu: "wmic CPU" 99 | 100 | java: 101 | elastic-java: "wmic process where processId={{PID}} GET CommandLine" 102 | jps: "{{JAVA_HOME}}\\bin\\jps -l -m -v" 103 | jstack: "{{JAVA_HOME}}\\bin\\jstack {{PID}}" 104 | javac: "{{JAVA_HOME}}\\bin\\javac -version" 105 | 106 | logs: 107 | elastic: "dir /b /l /o:-d /a:-d {{LOGPATH}}\\{{CLUSTERNAME}}.log" 108 | elastic-arc: "dir /b /l /o:-d /a:-d {{LOGPATH}}\\{{CLUSTERNAME}}-*.log.gz" 109 | gc: "dir /l /b /o:-d /a:-d {{LOGPATH}}\\gc.log.*" 110 | 111 | docker-container-ids: "{{dockerPath}} ps -q" 112 | 113 | docker-global: 114 | docker-info: "bash -c '{{dockerPath}} info'" 115 | docker-ps-all: "bash -c '{{dockerPath}} ps -a --no-trunc'" 116 | 117 | docker-container: 118 | docker-logs: "{{dockerPath}} logs {{CONTAINER_ID}}" 119 | docker-top: "{{dockerPath}} top {{CONTAINER_ID}}" 120 | docker-inspect: "{{dockerPath}} inspect {{CONTAINER_ID}}" 121 | 122 | monitoring-scroll-size: 1000 123 | bulk-import-size: 500 124 | 125 | import-templates: 126 | - "monitoring-es-diag" 127 | - "monitoring-logstash-diag" 128 | - "metricbeat-system-diag" 129 | 130 | monitoring-extract-pattern: ".monitoring-es-{{version}}-diag-import-{{suffix}}" 131 | logstash-extract-pattern: ".monitoring-logstash-{{version}}-diag-import-{{suffix}}" 132 | metricbeat-extract-pattern: "metricbeat-{{version}}-diag-import-{{suffix}}" 133 | 134 | query-files: 135 | - general 136 | - index_stats 137 | - index_all 138 | - cluster_id_check 139 | - cluster_ids 140 | - metricbeat 141 | 142 | monitor-sets: 143 | - cluster_state 144 | - cluster_stats 145 | - node_stats 146 | - indices_stats 147 | - index_stats 148 | - shards 149 | - job_stats 150 | - ccr_stats 151 | - ccr_auto_follow_stats 152 | 153 | logstash-sets: 154 | - logstash_stats 155 | - logstash_state 156 | 157 | metric-sets: 158 | - cpu 159 | - load 160 | - memory 161 | #- network 162 | #- process 163 | #- process_summary 164 | #- socket_summary 165 | #- entropy 166 | - core 167 | - diskio 168 | #- socket 169 | #- service 170 | #- users 171 | 172 | flight-recorder-es: 173 | complete: 174 | - allocation 175 | - allocation_explain 176 | - allocation_explain_disk 177 | - cluster_settings 178 | - cluster_settings_defaults 179 | - cluster_stats 180 | - fielddata 181 | - fielddata_stats 182 | - indices 183 | - indices_stats 184 | - licenses 185 | - mapping 186 | - master 187 | - nodes 188 | - nodes_stats 189 | - nodes_usage 190 | - pipelines 191 | - recovery 192 | - segments 193 | - settings 194 | - shards 195 | - tasks 196 | - templates 197 | - ccr_stats 198 | - enrich_stats 199 | - ilm_status 200 | - rollup_jobs 201 | stability: 202 | - nodes 203 | - shards 204 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/main/resources/logstash-rest.yml: -------------------------------------------------------------------------------- 1 | # The new settings format: 2 | # At the top level the label descring the query that will also be used as the 3 | # file name for its output. 4 | # Below the query label are the following attributes: 5 | # * extension - the file extension to be used for output. Optional, defaults to .json. 6 | # * subdir - some api's are now grouped in a subdirectory of the output directory to lessen clutter. Optional, defaults to root dir. 7 | # * retry - whether if a query fails it will be retried for the configured number of attempts. Optional, defaults to false. 8 | # * versions - one or more attributes of the format "version rule: "query string". Each set of version/query key pairs 9 | # should evaluate to exactly one that is appropriate for the version of the server being queried. Therefor rules should 10 | # be structured in such a way that only a valid query can be executed against a given version. Required. 11 | # NPM mode is the only one used: https://github.com/vdurmont/semver4j 12 | # NPM Versioning rules are documented here: https://github.com/npm/node-semver 13 | # 14 | # Note to those adding entries: within each group, cat API's, json API's, and commercial, they are in alphabetical order. 15 | # Please adhere to this convention when submitting pull requests. 16 | 17 | logstash_health_report: 18 | versions: 19 | ">= 8.16.0": "/_health_report" 20 | 21 | logstash_node: 22 | versions: 23 | "> 0.0.0": "/_node" 24 | 25 | logstash_nodes_hot_threads: 26 | versions: 27 | "> 0.0.0": "/_node/hot_threads?threads=10000" 28 | 29 | logstash_nodes_hot_threads_human: 30 | extension: .txt 31 | versions: 32 | "> 0.0.0": "/_node/hot_threads?human&threads=10000" 33 | 34 | logstash_node_stats: 35 | versions: 36 | "> 0.0.0": "/_node/stats" 37 | 38 | logstash_plugins: 39 | versions: 40 | "> 0.0.0": "/_node/plugins" 41 | 42 | logstash_version: 43 | versions: 44 | "> 0.0.0": "/" 45 | -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/cluster_id_check.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "bool": { 4 | "filter": [ { "term": { "cluster_uuid": "{{clusterId}}" } } ] 5 | } 6 | }, 7 | "collapse": { 8 | "field": "cluster_uuid" 9 | }, 10 | "sort": { 11 | "timestamp": "asc" 12 | }, 13 | "size": 100, 14 | "_source": ["cluster_uuid", "cluster_name"] 15 | } -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/cluster_ids.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "bool": { 4 | "filter": [ 5 | {"bool": {"should": [ 6 | {"term": {"type": "cluster_stats"}}, 7 | {"exists": {"field": "cluster_stats"}} 8 | ]}} 9 | ] 10 | } 11 | }, 12 | "collapse": { 13 | "field": "cluster_uuid" 14 | }, 15 | "sort": { 16 | "timestamp": "asc" 17 | }, 18 | "size": 100, 19 | "_source": ["cluster_uuid", "cluster_name", "cluster_settings.cluster.metadata.display_name", "elasticsearch.cluster.name"] 20 | } 21 | -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/general.json: -------------------------------------------------------------------------------- 1 | { 2 | "size": {{size}}, 3 | "query":{ 4 | "bool": { 5 | "filter": [ 6 | {"bool": {"should": [ 7 | {"term": {"type": "{{type}}"}}, 8 | {"exists": {"field": "{{field}}"}} 9 | ]}}, 10 | {"term": { "cluster_uuid" : "{{clusterId}}"} }, 11 | { "range": 12 | { "timestamp": 13 | { 14 | "gte": "{{start}}", 15 | "lte": "{{stop}}" 16 | } 17 | } 18 | } 19 | 20 | ] 21 | } 22 | }, 23 | "sort": [ 24 | { 25 | "timestamp": 26 | { 27 | "order": "asc" 28 | } 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/index_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "size": {{size}}, 3 | "query":{ 4 | "bool": { 5 | "filter": [ 6 | {"bool": {"should": [ 7 | {"term": {"type": "index_stats"}}, 8 | {"exists": {"field": "index_stats"}} 9 | ]}}, 10 | {"term": { "cluster_uuid" : "{{clusterId}}"} }, 11 | { "range": 12 | { "timestamp": 13 | { 14 | "gte": "{{start}}", 15 | "lte": "{{stop}}" 16 | } 17 | } 18 | } 19 | ] 20 | } 21 | }, 22 | "sort": [ 23 | { 24 | "index_stats.index": { 25 | "order": "asc" 26 | } 27 | }, 28 | { 29 | "timestamp": { 30 | "order": "asc" 31 | } 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/index_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "size": {{size}}, 3 | "query":{ 4 | "bool": { 5 | "must_not": [ 6 | {"wildcard": { 7 | "index_stats.index": { 8 | "value": ".*" 9 | } 10 | }} 11 | ], 12 | "filter": [ 13 | {"bool": {"should": [ 14 | {"term": {"type": "{{type}}"}}, 15 | {"exists": {"field": "{{field}}"}} 16 | ]}}, 17 | {"term": { "cluster_uuid" : "{{clusterId}}"} }, 18 | { "range": 19 | { "timestamp": 20 | { 21 | "gte": "{{start}}", 22 | "lte": "{{stop}}" 23 | } 24 | } 25 | } 26 | ] 27 | } 28 | }, 29 | "sort": [ 30 | { 31 | "index_stats.index": { 32 | "order": "asc" 33 | } 34 | }, 35 | { 36 | "timestamp": { 37 | "order": "asc" 38 | } 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /src/main/resources/monitoring-extract/metricbeat.json: -------------------------------------------------------------------------------- 1 | { 2 | "size": {{size}}, 3 | "query":{ 4 | "bool": { 5 | "filter": [ 6 | { "range": 7 | { "@timestamp": 8 | { 9 | "gte": "{{start}}", 10 | "lte": "{{stop}}" 11 | } 12 | } 13 | }, 14 | { 15 | "term": {"metricset.name": "{{type}}"} 16 | } 17 | ] 18 | } 19 | }, 20 | "sort": [ 21 | { 22 | "@timestamp": 23 | { 24 | "order": "asc" 25 | } 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /src/main/resources/monitoring-rest.yml: -------------------------------------------------------------------------------- 1 | monitoring-uri: 2 | versions: 3 | "< 7.0.0": "/.monitoring-es*/_search" 4 | ">= 7.0.0": "/.monitoring-es*/_search?rest_total_hits_as_int=true" 5 | 6 | monitoring-start-scroll-uri: 7 | versions: 8 | "< 7.0.0": "/.monitoring-{{type}}*/_search?scroll=2m" 9 | ">= 7.0.0": "/.monitoring-{{type}}*/_search?scroll=2m&rest_total_hits_as_int=true" 10 | 11 | metricbeat-start-scroll-uri: 12 | versions: 13 | "< 7.0.0": "/metricbeat*/_search?scroll=2m" 14 | ">= 7.0.0": "/metricbeat**/_search?scroll=2m&rest_total_hits_as_int=true" 15 | 16 | monitoring-scroll-uri: 17 | versions: 18 | "< 7.0.0": "/_search/scroll?scroll=2m" 19 | ">= 7.0.0": "/_search/scroll?scroll=2m&rest_total_hits_as_int=true" 20 | -------------------------------------------------------------------------------- /src/main/resources/scrub.yml: -------------------------------------------------------------------------------- 1 | # Comment out to remove from default processing 2 | auto-scrub: 3 | - "ipv4" 4 | - "ipv6" 5 | - "mac" 6 | - "nodeName" 7 | - "nodeId" 8 | - "clusterName" 9 | 10 | # Files matching any regex here will be excluded from 11 | # any sanitization. If you know there's nothing sensitive 12 | # in there you can save some processing. By default GC 13 | # logs are not sanitized. 14 | global-exclude: 15 | - ".*.gc.*" 16 | 17 | # Files matching any regex here will be removed from 18 | # the final sanitized product. For instance, if you 19 | # don't need slow logs and they have a lot of sensitive 20 | # info you can bypass them completely here. 21 | remove: 22 | - ".*.gz" 23 | # - ".*.zip" 24 | 25 | tokens: 26 | ## Example tokens - regex or literal. 27 | ## Checked in every file 28 | # - token: '\w*.sisyphus.rock.org' 29 | # - token: '\w*IGOTMP3s*' 30 | # - token: 'unknown' 31 | 32 | # For individual tokens you can also specify 33 | # an include and/or exclude paramater as well. 34 | # Depending on which you use, it will only check for that 35 | # token in a file matching the include pattern or 36 | # will bypass that check for files marked with exclude 37 | 38 | # Example 39 | # Removes the query section between the brackets 40 | # ONLY for slow log files 41 | # tokens: 42 | # - token: "\[.*\]" 43 | # include: ["*slowlog*.log"] 44 | 45 | # Example 46 | # Removes this token everywhere except for 47 | # log files and manifest.json 48 | # tokens: 49 | # - token: '\w*IGOTMP3s*' 50 | # exclude: ["*.log", "manifest.json"] 51 | # 52 | 53 | -------------------------------------------------------------------------------- /src/test/java/co/elastic/support/rest/TestRestConfigFileValidity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one 3 | * or more contributor license agreements. Licensed under the Elastic License 4 | * 2.0; you may not use this file except in compliance with the Elastic License 5 | * 2.0. 6 | */ 7 | package co.elastic.support.rest; 8 | 9 | import co.elastic.support.diagnostics.DiagnosticException; 10 | import co.elastic.support.util.JsonYamlUtils; 11 | import org.junit.jupiter.api.Test; 12 | import org.semver4j.Semver; 13 | 14 | import java.util.Arrays; 15 | import java.util.Map; 16 | 17 | import static org.junit.jupiter.api.Assertions.assertNotNull; 18 | import static org.junit.jupiter.api.Assertions.assertTrue; 19 | 20 | public class TestRestConfigFileValidity { 21 | protected static Semver sem = new Semver("9.9.999"); 22 | 23 | @Test 24 | public void validateElasticConfigVersioning() throws DiagnosticException { 25 | // validates each set of version entries. 26 | for (String yamlFile : Arrays.asList("elastic-rest.yml", "logstash-rest.yml", "kibana-rest.yml", "monitoring-rest.yml")) { 27 | Map restEntriesConfig = JsonYamlUtils.readYamlFromClasspath(yamlFile, true); 28 | validateEntries(yamlFile, restEntriesConfig); 29 | } 30 | } 31 | 32 | @SuppressWarnings("unchecked") 33 | private void validateEntries(String file, Map config) { 34 | for (Map.Entry entry : config.entrySet()) { 35 | Map values = (Map) entry.getValue(); 36 | Map versions = (Map) values.get("versions"); 37 | 38 | int valid = 0; 39 | 40 | // Urls should have a leading / 41 | // For each entry there should be at most 1 valid url. 42 | for (Map.Entry versionNode : versions.entrySet()) { 43 | if (sem.satisfies(versionNode.getKey())) { 44 | valid++; 45 | } 46 | 47 | if (versionNode.getValue() instanceof String) { 48 | String url = (String) versionNode.getValue(); 49 | assertTrue(url.startsWith("/"), url); 50 | } else if (versionNode.getValue() instanceof Map) { 51 | Map entryVersion = (Map) versionNode.getValue(); 52 | String url = (String) entryVersion.get("url"); 53 | Object spaceaware = entryVersion.get("spaceaware"); 54 | Object paginate = entryVersion.get("paginate"); 55 | 56 | assertNotNull(url, entry.getKey() + "[" + versionNode.getKey() + "]"); 57 | assertTrue(url.startsWith("/"), url); 58 | assertTrue(spaceaware == null || spaceaware instanceof Boolean, "spaceaware is not a Boolean"); 59 | assertTrue(paginate == null || paginate instanceof String, "paginate is not a String"); 60 | } 61 | } 62 | 63 | // should be at most 1 valid URL (0 if it's not available anymore) 64 | assertTrue(valid <= 1, "[" + file + "][" + entry.getKey() + "] matches " + valid); 65 | } 66 | 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/test/resources/diags-test.yml: -------------------------------------------------------------------------------- 1 | # REST timeout settings - note:values are in second 2 | 3 | github-settings: 4 | diagReleaseHost: "api.github.com" 5 | diagReleaseDest: "/repos/elastic/support-diagnostics/releases/latest" 6 | diagReleaseScheme: "https" 7 | diagLatestRelease: "https://api.github.com/repos/elastic/support-diagnostics/releases/latest" 8 | 9 | log-settings: 10 | maxLogs: 2 11 | maxGcLogs: 3 12 | 13 | network-cache-settings: 14 | networkaddress.cache.ttl: 15 | networkaddress.cache.negative.ttl: 16 | 17 | password-keys: 18 | - password 19 | - secret 20 | - access 21 | - key 22 | 23 | textFileExtensions: 24 | - allocation 25 | - cat_aliases 26 | - cat_count 27 | - cat_fielddata 28 | - cat_health 29 | - cat_indices 30 | - cat_master 31 | - cat_nodeattrs 32 | - cat_nodes 33 | - cat_pending_tasks 34 | - cat_recovery 35 | - cat_repositories 36 | - cat_segments 37 | - cat_shards 38 | - cat_thread_pool 39 | - logstash_nodes_hot_threads 40 | - nodes_hot_threads 41 | 42 | 43 | rest-config: 44 | requestTimeout: 30000 45 | connectTimeout: 30000 46 | socketTimeout: 30000 47 | 48 | retries: 49 | nodes: 3 50 | nodes-stats: 3 51 | shards: 3 52 | 53 | rest-calls: 54 | 55 | common: 56 | alias: "/_alias?human" 57 | cat_indices: "/_cat/indices?v" 58 | 59 | versions: 60 | major-1: 61 | minor-0: 62 | licenses: "/_licenses" 63 | 64 | major-2: 65 | minor-0: 66 | cat_nodeattrs: "/_cat/nodeattrs?v&h=node,id,pid,host,ip,port,attr,value" 67 | 68 | major-5: 69 | minor-0: 70 | allocation_explain: "/_cluster/allocation/explain" 71 | minor-2: 72 | cat_indices: "_cat/indices?v&s=index" 73 | 74 | major-6: 75 | minor-0: 76 | nodes_usage: "/_nodes/usage" 77 | minor-5: 78 | security_priv: "/_xpack/security/privilege" 79 | 80 | elastic-threads: 81 | nodes: "/_nodes?human" 82 | nodes_hot_threads: "/_nodes/hot_threads?threads=10000" 83 | 84 | thread-dump: 85 | jstack: "jstack PID" 86 | 87 | logstash: 88 | logstash_version: "/" 89 | logstash_node: "/_node" 90 | logstash_node_stats: "/_node/stats" 91 | logstash_nodes_hot_threads: "/_node/hot_threads?human=true&threads=10000" 92 | logstash_plugins: "/_node/plugins" 93 | 94 | linuxOS: 95 | top: "top -b -n1" 96 | netstat: "netstat -an" 97 | ss: "ss -an" 98 | process-list: "ps -ef" 99 | top_threads: "top -b -n1 -H" 100 | uname: "uname -a -r" 101 | cpu: "cat /proc/cpuinfo" 102 | iostat: "iostat -c -d -x -t -m 1 5" 103 | sar: "sar -A" 104 | sysctl: "sysctl -a" 105 | dmesg: "dmesg" 106 | huge_pages: "cat /sys/kernel/mm/transparent_hugepage/enabled" 107 | cpu_governor: "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" 108 | limits: "cat /etc/security/limits.conf" 109 | 110 | linuxOS-dep: 111 | proc-limit: "cat /proc/PID/limits" 112 | jps: "JAVA_HOME/bin/jps -l -m -v" 113 | jstack: "JAVA_HOME/bin/jstack PID" 114 | 115 | macOS: 116 | top: "top -l 1" 117 | netstat: "netstat -an" 118 | process-list: "ps -ef" 119 | ulimit: "ulimit -a" 120 | jps: "jps -l -m -v" 121 | 122 | macOS-dep: 123 | jstack: "jstack PID" 124 | jps: "jps -l -m -v" 125 | 126 | winOS: 127 | process-list: "tasklist /v" 128 | netstat: "netstat -ano" 129 | cpu: "wmic CPU" 130 | 131 | winOS-dep: 132 | jps: "JAVA_HOME\\bin\\jps -l -m -v" 133 | jstack: "JAVA_HOME\\bin\\jstack PID" 134 | -------------------------------------------------------------------------------- /src/test/resources/linux-process-list.txt: -------------------------------------------------------------------------------- 1 | UID PID PPID C STIME TTY TIME CMD 2 | gnieman 90409 90408 18 16:44 pts/1 00:00:02 /home/gnieman/Downloads/jdk/bin/java -Xms256m -Xmx2000m -cp .:./lib/* co.elastic.support.diagnostics.DiagnosticApp -h localhost 3 | elastic+ 90046 1 19 16:43 ? 00:00:15 /usr/share/elasticsearch/jdk/bin/java -Xms1g -Xmx1g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Des.networkaddress.cache.ttl=60 -Des.networkaddress.cache.negative.ttl=10 -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastThrow -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Djava.io.tmpdir=/tmp/elasticsearch-1445746514332098052 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/lib/elasticsearch -XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log -Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m -Djava.locale.providers=COMPAT -Dio.netty.allocator.type=unpooled -Des.path.home=/usr/share/elasticsearch -Des.path.conf=/etc/elasticsearch -Des.distribution.flavor=default -Des.distribution.type=rpm -Des.bundled_jdk=true -cp /usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -p /var/run/elasticsearch/elasticsearch.pid --quiet 4 | elastic+ 90128 90046 0 16:43 ? 00:00:00 /usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/bin/controller 5 | root 90272 9217 0 16:43 ? 00:00:00 sleep 60 6 | gnieman 90408 28835 0 16:44 pts/1 00:00:00 bash ./diagnostics.sh -h localhost 7 | gnieman 90489 90409 0 16:44 pts/1 00:00:00 ps -ef 8 | -------------------------------------------------------------------------------- /src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/test/resources/proc/1/cgroup-no: -------------------------------------------------------------------------------- 1 | 11:hugetlb:/ 2 | 10:freezer:/ 3 | 9:memory:/ 4 | 8:perf_event:/ 5 | 7:pids:/ 6 | 6:blkio:/ 7 | 5:cpuset:/ 8 | 4:net_prio,net_cls:/ 9 | 3:cpuacct,cpu:/ 10 | 2:devices:/ 11 | 1:name=systemd:/ -------------------------------------------------------------------------------- /src/test/resources/proc/1/cgroup-yes: -------------------------------------------------------------------------------- 1 | 11:hugetlb:/ 2 | 10:freezer:/ 3 | 9:memory:/ 4 | 8:perf_event:/ 5 | 7:pids:/docker 6 | 6:blkio:/ 7 | 5:cpuset:/ 8 | 4:net_prio,net_cls:/ 9 | 3:cpuacct,cpu:/ 10 | 2:devices:/ 11 | 1:name=systemd:/ -------------------------------------------------------------------------------- /src/test/resources/rest-diags.yml: -------------------------------------------------------------------------------- 1 | # REST timeout settings - note:values are in second 2 | 3 | github-settings: 4 | diagReleaseHost: "api.github.com" 5 | diagReleaseDest: "/repos/elastic/support-diagnostics/releases/latest" 6 | diagReleaseScheme: "https" 7 | diagLatestRelease: "https://api.github.com/repos/elastic/support-diagnostics/releases/latest" 8 | 9 | log-settings: 10 | maxLogs: 2 11 | maxGcLogs: 3 12 | 13 | password-keys: 14 | - password 15 | - secret 16 | - access 17 | - key 18 | 19 | text-file-extensions: 20 | - allocation 21 | - cat_aliases 22 | - cat_count 23 | - cat_fielddata 24 | - cat_health 25 | - cat_indices 26 | - cat_master 27 | - cat_nodeattrs 28 | - cat_nodes 29 | - cat_pending_tasks 30 | - cat_recovery 31 | - cat_repositories 32 | - cat_segments 33 | - cat_shards 34 | - cat_thread_pool 35 | - logstash_nodes_hot_threads 36 | - nodes_hot_threads 37 | 38 | # Uncomment only if modifying defaults 39 | rest-config: 40 | requestTimeout: 10000 41 | connectTimeout: 5000 42 | socketTimeout: 5000 43 | maxTotalConn: 100 44 | maxConnPerRoute: 10 45 | 46 | # Number of tiems to reattempt a rest call 47 | call-retries: 3 48 | # Time before reattempts in milliseconnds 49 | pause-retries: 5000 50 | # Calls to reattempt 51 | require-retry: 52 | - nodes 53 | - nodes-stats 54 | - shards 55 | - cluster_state 56 | 57 | rest-calls: 58 | common: 59 | nodes: "/_nodes?human" 60 | 61 | versions: 62 | major-1: 63 | major-2: 64 | major-5: 65 | major-6: 66 | -------------------------------------------------------------------------------- /src/test/resources/win-process-list.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elastic/support-diagnostics/9033eef1233c7fbe533b8f5215e1beada4766da4/src/test/resources/win-process-list.txt --------------------------------------------------------------------------------