├── .github └── workflows │ └── go.yml ├── .gitignore ├── .gitlab-ci.yml ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── build.sh ├── config └── config.go ├── current_version_agent ├── docker ├── entrypoint.sh └── releem.conf.tpl ├── errors └── releemErrors.go ├── go.mod ├── go.sum ├── install.sh ├── main.go ├── metrics ├── agent.go ├── awsrdsdiscovery.go ├── awsrdsenhancedmetrics.go ├── awsrdsmetrics.go ├── dbCollectQueries.go ├── dbConf.go ├── dbInfo.go ├── dbInfoBase.go ├── dbMetrics.go ├── dbMetricsBase.go ├── os.go └── runner.go ├── models └── models.go ├── mysqlconfigurer.sh ├── releem-agent ├── install_on_primise.sh └── mysqlconfigurer.sh ├── releem.conf ├── repeater ├── logger.go └── releemConfiguration.go ├── tasks └── tasks.go ├── update_releem_docker.sh └── utils └── utils.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | tags: 9 | - '**' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v4 20 | with: 21 | go-version: '1.22.0' 22 | 23 | - name: Build 24 | run: bash build.sh 25 | 26 | - name: Create upload dir 27 | run: | 28 | mkdir -p ../v2 29 | cp current_version_agent install.sh mysqlconfigurer.sh releem-agent-x86_64 releem-agent-amd64 releem-agent-aarch64 releem-agent-freebsd-amd64 releem-agent-i686 releem-agent.exe ../v2/ 30 | 31 | - uses: shallwefootball/s3-upload-action@master 32 | name: Upload S3 current_version_agent 33 | with: 34 | aws_key_id: ${{ secrets.AWS_KEY_ID }} 35 | aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} 36 | aws_bucket: ${{ secrets.AWS_BUCKET }} 37 | source_dir: '../v2' 38 | destination_dir: 'v2' 39 | 40 | - name: Log in to Docker Hub 41 | uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a 42 | with: 43 | username: ${{ secrets.DOCKER_USERNAME }} 44 | password: ${{ secrets.DOCKER_PASSWORD }} 45 | 46 | - name: Set up QEMU 47 | uses: docker/setup-qemu-action@v3 48 | 49 | - name: Set up Docker Buildx 50 | uses: docker/setup-buildx-action@v3 51 | with: 52 | platforms: linux/amd64,linux/386, linux/arm64 53 | 54 | - name: Extract metadata (tags, labels) for Docker 55 | id: meta 56 | uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 57 | with: 58 | images: releem/releem-agent 59 | 60 | - name: Build and push Docker images 61 | uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 62 | with: 63 | platforms: linux/amd64,linux/386, linux/arm64 64 | context: . 65 | push: true 66 | tags: ${{ steps.meta.outputs.tags }} 67 | labels: ${{ steps.meta.outputs.labels }} 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .devcontainer/* 2 | test-project/test.sh 3 | .DS_Store 4 | releem-install.log 5 | releem-agent* 6 | releem1.conf 7 | releem_rds.conf -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: ubuntu:20.04 2 | 3 | stages: 4 | - testing 5 | 6 | test_preprod_mysql55: 7 | stage: testing 8 | services: 9 | - name: mysql:5.5 10 | alias: mysql55 11 | variables: 12 | MYSQL_ROOT_PASSWORD: mysql 13 | before_script: 14 | - apt-get update 15 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 16 | - mkdir -p /opt/releem 17 | script: 18 | - echo "[client]" > ~/.my.cnf 19 | - echo "user=root" >> ~/.my.cnf 20 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 21 | - echo "host=mysql55" >> ~/.my.cnf 22 | - cat ~/.my.cnf 23 | #Creating test db 24 | - git clone https://github.com/datacharmer/test_db.git 25 | - cd test_db 26 | - mysql < employees.sql 27 | #Execute MySQLConfigurer 28 | - cd .. 29 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 30 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 31 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 32 | 33 | when: manual 34 | 35 | 36 | test_preprod_mysql56: 37 | stage: testing 38 | services: 39 | - name: mysql:5.6 40 | alias: mysql56 41 | variables: 42 | MYSQL_ROOT_PASSWORD: mysql 43 | before_script: 44 | - apt-get update 45 | - mkdir -p /opt/releem 46 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 47 | script: 48 | - echo "[client]" > ~/.my.cnf 49 | - echo "user=root" >> ~/.my.cnf 50 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 51 | - echo "host=mysql56" >> ~/.my.cnf 52 | - cat ~/.my.cnf 53 | #Creating test db 54 | - git clone https://github.com/datacharmer/test_db.git 55 | - cd test_db 56 | - mysql < employees.sql 57 | #Execute MySQLConfigurer 58 | - cd .. 59 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 60 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 61 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 62 | 63 | when: manual 64 | 65 | 66 | test_preprod_mysql57: 67 | stage: testing 68 | services: 69 | - name: mysql:5.7 70 | alias: mysql57 71 | variables: 72 | MYSQL_ROOT_PASSWORD: mysql 73 | before_script: 74 | - apt-get update 75 | - apt-get -y install iputils-ping git curl mariadb-client wget curl net-tools libjson-perl 76 | - mkdir -p /opt/releem 77 | script: 78 | - echo "[client]" > ~/.my.cnf 79 | - echo "user=root" >> ~/.my.cnf 80 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 81 | - echo "host=mysql57" >> ~/.my.cnf 82 | - cat ~/.my.cnf 83 | #Creating test db 84 | - git clone https://github.com/datacharmer/test_db.git 85 | - cd test_db 86 | - mysql < employees.sql 87 | #Execute MySQLConfigurer 88 | - cd .. 89 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 90 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 91 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 92 | 93 | when: manual 94 | 95 | 96 | test_preprod_mysql80: 97 | stage: testing 98 | services: 99 | - name: mysql:8.0 100 | command: ["--default-authentication-plugin=mysql_native_password"] 101 | alias: mysql80 102 | variables: 103 | MYSQL_ROOT_PASSWORD: mysql 104 | before_script: 105 | - apt-get update 106 | - apt-get -y install iputils-ping git curl mariadb-client wget curl net-tools libjson-perl 107 | - mkdir -p /opt/releem 108 | script: 109 | - echo "[client]" > ~/.my.cnf 110 | - echo "user=root" >> ~/.my.cnf 111 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 112 | - echo "host=mysql80" >> ~/.my.cnf 113 | - cat ~/.my.cnf 114 | #Creating test db 115 | - git clone https://github.com/datacharmer/test_db.git 116 | - cd test_db 117 | - mysql < employees.sql 118 | #Execute MySQLConfigurer 119 | - cd .. 120 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 121 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 122 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 123 | 124 | when: manual 125 | 126 | 127 | test_preprod_percona55: 128 | stage: testing 129 | services: 130 | - name: percona:5.5 131 | alias: percona55 132 | variables: 133 | MYSQL_ROOT_PASSWORD: mysql 134 | before_script: 135 | - apt-get update 136 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 137 | - mkdir -p /opt/releem 138 | script: 139 | - echo "[client]" > ~/.my.cnf 140 | - echo "user=root" >> ~/.my.cnf 141 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 142 | - echo "host=percona55" >> ~/.my.cnf 143 | - cat ~/.my.cnf 144 | #Creating test db 145 | - git clone https://github.com/datacharmer/test_db.git 146 | - cd test_db 147 | - mysql < employees.sql 148 | #Execute MySQLConfigurer 149 | - cd .. 150 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 151 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 152 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 153 | 154 | when: manual 155 | 156 | 157 | test_preprod_percona56: 158 | stage: testing 159 | services: 160 | - name: percona:5.6 161 | alias: percona56 162 | variables: 163 | MYSQL_ROOT_PASSWORD: mysql 164 | before_script: 165 | - apt-get update 166 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 167 | - mkdir -p /opt/releem 168 | script: 169 | - echo "[client]" > ~/.my.cnf 170 | - echo "user=root" >> ~/.my.cnf 171 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 172 | - echo "host=percona56" >> ~/.my.cnf 173 | - cat ~/.my.cnf 174 | #Creating test db 175 | - git clone https://github.com/datacharmer/test_db.git 176 | - cd test_db 177 | - mysql < employees.sql 178 | #Execute MySQLConfigurer 179 | - cd .. 180 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 181 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 182 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 183 | 184 | when: manual 185 | 186 | 187 | test_preprod_percona57: 188 | stage: testing 189 | services: 190 | - name: percona:5.7 191 | alias: percona57 192 | variables: 193 | MYSQL_ROOT_PASSWORD: mysql 194 | before_script: 195 | - apt-get update 196 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 197 | - mkdir -p /opt/releem 198 | script: 199 | - echo "[client]" > ~/.my.cnf 200 | - echo "user=root" >> ~/.my.cnf 201 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 202 | - echo "host=percona57" >> ~/.my.cnf 203 | - cat ~/.my.cnf 204 | #Creating test db 205 | - git clone https://github.com/datacharmer/test_db.git 206 | - cd test_db 207 | - mysql < employees.sql 208 | #Execute MySQLConfigurer 209 | - cd .. 210 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 211 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 212 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 213 | 214 | when: manual 215 | 216 | 217 | 218 | test_preprod_percona80: 219 | stage: testing 220 | services: 221 | - name: percona:8.0 222 | command: ["--default-authentication-plugin=mysql_native_password"] 223 | alias: percona80 224 | variables: 225 | MYSQL_ROOT_PASSWORD: mysql 226 | before_script: 227 | - apt-get update 228 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 229 | - mkdir -p /opt/releem 230 | script: 231 | - echo "[client]" > ~/.my.cnf 232 | - echo "user=root" >> ~/.my.cnf 233 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 234 | - echo "host=percona80" >> ~/.my.cnf 235 | - cat ~/.my.cnf 236 | #Creating test db 237 | - git clone https://github.com/datacharmer/test_db.git 238 | - cd test_db 239 | - mysql < employees.sql 240 | #Execute MySQLConfigurer 241 | - cd .. 242 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 243 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 244 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 245 | 246 | when: manual 247 | 248 | 249 | test_preprod_maria101: 250 | stage: testing 251 | services: 252 | - name: mariadb:10.1 253 | alias: maria101 254 | variables: 255 | MYSQL_ROOT_PASSWORD: mysql 256 | before_script: 257 | - apt-get update 258 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 259 | - mkdir -p /opt/releem 260 | script: 261 | - echo "[client]" > ~/.my.cnf 262 | - echo "user=root" >> ~/.my.cnf 263 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 264 | - echo "host=maria101" >> ~/.my.cnf 265 | - cat ~/.my.cnf 266 | #Creating test db 267 | - git clone https://github.com/datacharmer/test_db.git 268 | - cd test_db 269 | - mysql < employees.sql 270 | #Execute MySQLConfigurer 271 | - cd .. 272 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 273 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 274 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 275 | 276 | when: manual 277 | 278 | 279 | test_preprod_maria102: 280 | stage: testing 281 | services: 282 | - name: mariadb:10.2 283 | alias: maria102 284 | variables: 285 | MYSQL_ROOT_PASSWORD: mysql 286 | before_script: 287 | - apt-get update 288 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 289 | - mkdir -p /opt/releem 290 | script: 291 | - echo "[client]" > ~/.my.cnf 292 | - echo "user=root" >> ~/.my.cnf 293 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 294 | - echo "host=maria102" >> ~/.my.cnf 295 | - cat ~/.my.cnf 296 | #Creating test db 297 | - git clone https://github.com/datacharmer/test_db.git 298 | - cd test_db 299 | - mysql < employees.sql 300 | #Execute MySQLConfigurer 301 | - cd .. 302 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 303 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 304 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 305 | 306 | when: manual 307 | 308 | 309 | test_preprod_maria103: 310 | stage: testing 311 | services: 312 | - name: mariadb:10.3 313 | alias: maria103 314 | variables: 315 | MYSQL_ROOT_PASSWORD: mysql 316 | before_script: 317 | - apt-get update 318 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 319 | - mkdir -p /opt/releem 320 | script: 321 | - echo "[client]" > ~/.my.cnf 322 | - echo "user=root" >> ~/.my.cnf 323 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 324 | - echo "host=maria103" >> ~/.my.cnf 325 | - cat ~/.my.cnf 326 | #Creating test db 327 | - git clone https://github.com/datacharmer/test_db.git 328 | - cd test_db 329 | - mysql < employees.sql 330 | #Execute MySQLConfigurer 331 | - cd .. 332 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 333 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 334 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 335 | 336 | when: manual 337 | 338 | 339 | test_preprod_maria104: 340 | stage: testing 341 | services: 342 | - name: mariadb:10.4 343 | alias: maria104 344 | variables: 345 | MYSQL_ROOT_PASSWORD: mysql 346 | before_script: 347 | - apt-get update 348 | - apt-get -y install iputils-ping git curl mariadb-client wget curl net-tools libjson-perl 349 | - mkdir -p /opt/releem 350 | script: 351 | - echo "[client]" > ~/.my.cnf 352 | - echo "user=root" >> ~/.my.cnf 353 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 354 | - echo "host=maria104" >> ~/.my.cnf 355 | - cat ~/.my.cnf 356 | #Creating test db 357 | - git clone https://github.com/datacharmer/test_db.git 358 | - cd test_db 359 | - mysql < employees.sql 360 | #Execute MySQLConfigurer 361 | - cd .. 362 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 363 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 364 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 365 | 366 | when: manual 367 | 368 | 369 | test_preprod_maria105: 370 | stage: testing 371 | services: 372 | - name: mariadb:10.5 373 | alias: maria105 374 | variables: 375 | MYSQL_ROOT_PASSWORD: mysql 376 | before_script: 377 | - apt-get update 378 | - apt-get -y install iputils-ping git curl mariadb-client wget curl net-tools libjson-perl 379 | - mkdir -p /opt/releem 380 | script: 381 | - echo "[client]" > ~/.my.cnf 382 | - echo "user=root" >> ~/.my.cnf 383 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 384 | - echo "host=maria105" >> ~/.my.cnf 385 | - cat ~/.my.cnf 386 | #Creating test db 387 | - git clone https://github.com/datacharmer/test_db.git 388 | - cd test_db 389 | - mysql < employees.sql 390 | #Execute MySQLConfigurer 391 | - cd .. 392 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 393 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 394 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 395 | 396 | when: manual 397 | 398 | 399 | test_preprod_maria106: 400 | stage: testing 401 | services: 402 | - name: mariadb:10.6 403 | alias: maria106 404 | variables: 405 | MYSQL_ROOT_PASSWORD: mysql 406 | before_script: 407 | - apt-get update 408 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 409 | - mkdir -p /opt/releem 410 | script: 411 | - echo "[client]" > ~/.my.cnf 412 | - echo "user=root" >> ~/.my.cnf 413 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 414 | - echo "host=maria106" >> ~/.my.cnf 415 | - cat ~/.my.cnf 416 | #Creating test db 417 | - git clone https://github.com/datacharmer/test_db.git 418 | - cd test_db 419 | - mysql < employees.sql 420 | #Execute MySQLConfigurer 421 | - cd .. 422 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 423 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 424 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 425 | 426 | when: manual 427 | 428 | 429 | test_preprod_maria107: 430 | stage: testing 431 | services: 432 | - name: mariadb:10.7 433 | alias: maria107 434 | variables: 435 | MYSQL_ROOT_PASSWORD: mysql 436 | before_script: 437 | - apt-get update 438 | - apt-get -y install iputils-ping git curl mysql-client wget curl net-tools libjson-perl 439 | - mkdir -p /opt/releem 440 | script: 441 | - echo "[client]" > ~/.my.cnf 442 | - echo "user=root" >> ~/.my.cnf 443 | - echo "password=$MYSQL_ROOT_PASSWORD" >> ~/.my.cnf 444 | - echo "host=maria107" >> ~/.my.cnf 445 | - cat ~/.my.cnf 446 | #Creating test db 447 | - git clone https://github.com/datacharmer/test_db.git 448 | - cd test_db 449 | - mysql < employees.sql 450 | #Execute MySQLConfigurer 451 | - cd .. 452 | - bash mysqlconfigurer.sh -k $RELEEM_API_KEY 453 | - cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 454 | - cat /tmp/.mysqlconfigurer/mysqltunerreport.json 455 | 456 | when: manual 457 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: focal 3 | 4 | services: 5 | - docker 6 | 7 | env: 8 | - DB=mariadb:5.5 9 | - DB=mariadb:10.2 10 | - DB=mariadb:10.3 11 | - DB=mariadb:10.4 12 | - DB=mariadb:10.5 13 | - DB=mariadb:10.6 14 | - DB=mysql:5.5 15 | - DB=mysql:5.7 16 | - DB=mysql:8.0 17 | - DB=percona:5.6 18 | - DB=percona:5.7 19 | - DB=percona:8.0 20 | addons: 21 | apt: 22 | update: true 23 | 24 | before_script: 25 | - sudo apt install curl libjson-perl 26 | - mysql --version 27 | - mysqladmin --version 28 | - docker run -it --name=mysqltestinstance -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} 29 | - echo -e "[client]\nuser=root\npassword=\"\"\nhost=127.0.0.1" > ~/.my.cnf 30 | - chmod 600 ~/.my.cnf 31 | - git clone https://github.com/datacharmer/test_db.git 32 | - cd test_db 33 | - count=10 34 | - while ! mysql -e 'select version()' && [ $count -gt 0 ]; do echo $count seconds to go; sleep 1; count=$(( $count - 1 )); done 35 | - if [[ $DB =~ .*:8.0 ]] ; then 36 | for file in public_key.pem ca.pem server-cert.pem client-key.pem client-cert.pem ; do 37 | docker cp mysqltestinstance:/var/lib/mysql/$file "${HOME}" ; 38 | done ; 39 | fi 40 | - "cat employees.sql | grep -v 'storage_engine' | mysql" 41 | - cd .. 42 | 43 | script: 44 | - /bin/bash ./mysqlconfigurer.sh -k $TRAVIS_RELEEM_API_KEY 45 | - sudo cat /tmp/.mysqlconfigurer/mysqltunerreport.json 46 | - sudo cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf 47 | 48 | after_script: 49 | - docker stop mysqltestinstance 50 | - echo "Standard Output: $(cat /tmp/.mysqlconfigurer/mysqltunerreport.json)" 51 | - echo "Standard Error : $(cat /tmp/.mysqlconfigurer/z_aiops_mysql.cnf)" -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Releem releases 2 | --- 3 | Information about releases of the Releem. 4 | 5 | Releem 1.20.0, 2025-03-30 ([What's New At Releem | April 2025](https://releem.com/blog/whats-new-at-releem-april-2025)) 6 | - Added new New Events Section to track key MySQL events 7 | - Added MySQL Process List to spot long-running queries. Closes #332 8 | - Added Configuration History to compare different configurations. Closes #257 9 | - Added Index Incompatibility Detection for SQL Query Optimization 10 | - Added Embedded SQL Warnings for SQL Query Optimization 11 | - Added Fragmented Table Detection to the [Schema Checks](https://releem.com/blog/mysql-database-schema-checks#rec907599082). Closes #185 12 | - Fixed innodb_flush_method settings were incompatible with Windows servers. Closes #401 13 | - Fixed index corruption issue when using innodb_change_buffering with MariaDB. Closes #393 14 | - Improved performance on servers with many databases by optimizing how Releem queries database information. Closes #398 15 | - Improved Releem Score Calculation to make performance metrics more intuitive. Added documentation. Closes #198 16 | 17 | Releem 1.19.9, 2025-01-31 ([What's New At Releem | February 2025](https://releem.com/blog/whats-new-at-releem-february-2025)) 18 | - Added automatic SQL Query Analytics collection 19 | - Added threshold for how large the changes need to be to be recommended to the user. Closes #188 20 | - Added additional periods to monitoring 21 | - Added support of arm64 for Releem Agent 22 | - Added support of Percona XtraDB Cluster 23 | - Added New Health Check Query Truncation status which identifies truncated queries in Query analytics. 24 | - Added checks for duplicated servers to prevent mixing metrics from servers with the same name. 25 | - Added export of query analytics to CSV. Closes #384 26 | - Added special screen to teach new users how to use product features 27 | - Improved query analytics and now it depends on selected period, users can see queries executed in selected period. Closes #318 28 | - Improved Releem Expert System that checks consistency of configuration before recommendation to prevent incorrect recommendations 29 | - Improved analysis for queries including functions on indexed columns 30 | - Improved detection of conditions comparing two columns of the same table 31 | - Fixed Latency calculation. Closes #391 32 | - Fixed health check"Table Definition Cache Hit Rate" drops to 0%. Closes #390 33 | - Improved open_files_limit recommendation for AWS RDS. Closes #389 34 | - Fixed Releem Agent was stopped unexpectedly. Closes #368 35 | 36 | Releem 1.19.8, 2024-12-31 ([What's New At Releem | December 2024](https://releem.com/blog/whats-new-at-releem-december-2024)) 37 | - Added automatic index detection and impact tracking. 38 | - Added one-click configuration updates for AWS RDS. Closes #320 39 | - Released Releem Agent for Windows. Closes #207 40 | - Improved latency calculation by recalculating latency every minute based on the latest queries. Closes #266 41 | - Improved InnoDB log file size recommendations. Closes #312 42 | - Fixed formatting errors in SQL query recommendations. Closes #358 43 | - Fixed `innodb_log_file_size` being too large. Closes #367 44 | - Fixed high memory usage of Releem Agent on servers with thousands of tables. Closes #376 45 | - Fixed the `query_optimization` parameter not affecting the collection of query optimization information using the "Load Data" button. Closes #379 46 | - Fixed adjusting suggestions to new memory limit faster. Closes #232 47 | 48 | Releem 1.19.5, 2024-10-31 49 | - Added [Schema Checks](https://releem.com/mysql-schema-optimization). Closes #324 50 | - Added security check for database end-of-life (EOL). 51 | - Improved SQL query optimization reports. 52 | - Fixed MySQL root user password being stored in web cache. Closes #321 53 | - Fixed MySQL restart termination. Closed #330 54 | - Fixed changed logs path. Closes #337 55 | - Fixed SQL query optimization error due to `sql_mode=only_full_group_by`. Closes #363 56 | - Fixed index creation error. Closes #362 57 | 58 | Releem 1.19.0, 2024-09-30 ([What's New At Releem | September 2024](https://releem.com/blog/whats-new-at-releem-september-2024)) 59 | - Added [SQL Query Optimization](https://releem.com/blog/introducing-automatic-sql-query-optimization) tab in the Releem Portal 60 | - Added collection of Query Examples for Query Optimization for MariaDB 61 | - Added collection of EXPLAINs for Query Optimization for MariaDB 62 | - Added support of MariaDB 11.5. Closes #349 63 | - Fixed incorrect latest version in the Update notification. Closes #346 64 | - Fixed high resource usage during metric collection on servers with more than 1000 databases. Closes #347 65 | - Fixed installation failed due incorrect PATH variable. Closes #348 66 | - Fixed recommendation of innodb_log_buffer_size variable. Closes #352 67 | - Fixed Query Analytics is showing a database name that does not exist on my db server. Closes #357 68 | - Fixed Applying recommended configuration without restart doesn't work when mysql_restart_service setting is empty. Closes #350 69 | 70 | Releem 1.18.0, 2024-08-31 71 | - Added EXPLAINs collection for SQL Query Optimization for MySQL 8.x 72 | - Added collection of InnoDB engine status 73 | - Added query_optimization parameter to Releem Agent 74 | - Added query digest monitoring 75 | - Added collection of database schema for query optimization. Closes #357 76 | - Added collection of schema information 77 | - Added automatics SQL Query Optimization reports 78 | - Fixed crontab error during installation process. Closes #341 79 | - Fixed SUM_ROWS_SENT error during loadin Query Analytics. Closes #336 80 | - Fixed table_definition_cache recommendations. Closes #335 81 | 82 | Releem 1.17.0, 2024-06-30 ([What's New At Releem | June 2024](https://releem.com/blog/whats-new-at-releem-june-2024)) 83 | - Improved Weekly Reports 84 | - Added Alerts on CPU Utilization and Disk space. Closes #147 85 | - Added applying configuration without restart to Releem Agent. 86 | - Added SSL support to Releem Agent. Closes #310 87 | - Added collection information from performance_schema.file_summary_by_instance 88 | - Fixed Automatic installation doens't work in some cases. Closes #166 89 | - Fixed buffer recommendations more that total RAM. Closes #262 90 | 91 | Releem 1.16.0, 2024-05-31 ([What's New At Releem | May 2024](https://releem.com/blog/whats-new-at-releem-may-2024)) 92 | - Added Security Checks 93 | - Added new event for partially applied configuration. 94 | - Added support of AWS RDS Aurora. Closes #308 95 | - Fixed Many events on MySQL restart. Closes #305 96 | 97 | Releem 1.15.0, 2024-04-30 98 | - Added new option to disable Query Cache manually. Feature request #289 99 | - Added Max Query Length option allows Releem to save full queries for analysis. Closes #291 100 | - Added Query Inspect popup displays the details of queries. 101 | - Fixed IP address update in the dashboard if real IP address was changed. Closes #292 102 | - Fixed Releem can't recognize that innodb_log_file_buffering is enabled. Closes #285 103 | - Fixed Latency on the Servers page displays not in ms. Closes #303 104 | - Fixed Incorrect aggregation on weekly and monthly charts. 105 | - Fixed Using innodb_log_file_size instead of innodb_redo_log_capacity for Percona 8.x 106 | - Removed transaction_prealloc_size for Percona 8.0.29 and later 107 | 108 | Releem 1.14.0, 2024-03-31 ([What's New At Releem | April 2024](https://releem.com/blog/whats-new-at-releem-april-2024)) 109 | - Added [SQL Query Analytics](https://releem.com/query-analytics) block to the Dashboard. Closes #256 110 | - Added new task collection of performance_schema.events_statements_summary_by_digest to Releem Agent 111 | - Added new metric to the Dashboard [Aborted_clients](https://releem.com/docs/mysql-aborted-clients) 112 | - Fixed incorrect permissions after installation. Closes #272 113 | - Fixed File integrity issue after the agent update. Closes #277 114 | - Fixed open_files_limit issue for AWS RDS. Closes #284 115 | - Fixed recommendations of variables that couldn't be changed ffor AWS RDS Aurora. Closes #281 116 | - Fixed Releem Agent fatal error when an agent couldn't get disk information. Closes #276 117 | - Improved Wizard to add new servers 118 | 119 | Releem 1.13.0, 2024-02-29 ([What's New At Releem | February 2024](https://releem.com/blog/whats-new-in-releem-february-2024)) 120 | - Added MySQL memory_limit and long_query_time to settings in the Releem dashboard 121 | - Added sending logs to Platform when agent crashed 122 | - Added detection that configuration was changed without MySQL restart (for AWS RDS customers). Closes #270 123 | - Added collection information about the file system. Closes #189 124 | - Added version of Releem Agent for i686. Closes #263 125 | - Improved description of MySQL Health Checks 126 | - Integrated RabbitMQ to remove dependence on AWS Lambda and process metrics asynchronously 127 | - Changed time of metrics to Releem Platform time instead of time on customers servers Closes #264 128 | - Removed transaction_prealloc_size from recommendations for MySQL 8.0.29 as deprecated. Closes #267 129 | 130 | Releem 1.12.0, 2024-01-31 ([What's New At Releem | January 2024](https://releem.com/blog/whats-new-at-releem-january-2024)) 131 | - Improved Servers page to highligt servers with unapplied recommendations 132 | - Added email reports for newly recommended MySQL configurations Closes #193 133 | - Disabled apply button for old agents 134 | - Published [list of MySQL variables](https://releem.com/docs/mysql-performance-parameters) that Releem tuned on Free plan 135 | - [Migrated database metrics](https://releem.com/blog/migrating-to-clickhouse) from MySQL to ClickHouse and improve performance of Dashboard by 25% 136 | - Fixed bugs on RAM and IOPS charts 137 | 138 | Releem 1.11.0, 2023-12-31 ([What's New At Releem | December 2023](https://releem.com/blog/whats-new-at-releem-november-2023)) 139 | - Added notification on increasing open_files_limit. Closes #171 140 | - Added server restarts to MySQL Metrics graphs. Closes #191 141 | - Added Automatic applying configuration by clicking button in the web interface. 142 | - Added automatic rollback function if any issues arise while applying a new configuration. Closes #187 143 | - Added new Health check - Table Definition Cache 144 | - Fixed unable to bring recommendations for servers with disabled InnoDB. Closes #213 145 | - Fixed InnoDB log file size Health Check calculation. Closes #234 146 | - Fixed Releem Agent installation guide for AWS RDS. Closes #223 147 | - Fixed Releem Agent tends to stop by itself from time to time. Closes #210 148 | 149 | Releem 1.9.0, 2023-10-31 ([What's New At Releem | October 2023](https://releem.com/blog/whats-new-at-releem-october-2023)) 150 | - Improved InnoDB Log File Size Health Check. Closes #202 151 | - Improved Table Cache Hit Rate Health Check. Closes #201 152 | - Added [Open Files Utilization](https://releem.com/blog/mysql-health-checks#rec667806004) 153 | - Added [Table Locking Efficiency](https://releem.com/blog/mysql-health-checks#rec667808781) 154 | - Added [InnoDB Dirty Pages Ratio](https://releem.com/blog/mysql-health-checks#rec667811185) 155 | - Added default start page for users with multiple servers. Closes #177 156 | - Added new Help windows with Frequently Asked Questions. 157 | - Fixed RDS Memory Usage. Closes #212 158 | - Fixed query_cache_type. Closes #214 159 | - Fixed query_cache_size. Closes #216 160 | - Fixed the time of applying configuration events on the day graph. Closes #220 161 | - Improved documentation. 162 | 163 | Releem 1.8.0, 2023-09-30 ([What's New At Releem | September 2023](https://releem.com/blog/whats-new-at-releem-september-2023)) 164 | - Added OS version to the Releem Score block 165 | - Fixed the issue with graphs for the America/Mexico_City timezone Closes #196. 166 | - Added a detailed description for the [Memory Limit] (https://releem.com/docs/getstarted#rec586933587). Closes #205. 167 | - Added unapplied recommendations to the server list. Closes #176. 168 | - Fixed innodb_page_cleaners wasn't changed during applying configuration. Closes #197. 169 | 170 | Releem 1.7.0, 2023-08-31 ([What's New At Releem | August 2023](https://releem.com/blog/whats-new-at-releem-august-2023)) 171 | - Added Automated Updates for Releem Agent installed in docker container. Closes #184 172 | - Improved version for mobile and Firefox compatibility. 173 | - Improved Query Cache suggestions. Closes #135 174 | - Fixed 'innodb_max_dirty_pages_pct' bug on MySQL 5.5. Closes #183 175 | - Fixed metrics collecttion issue for db servers with Sphinx engine. Closes #174 , Closes #175 176 | - Fixed bug for users with unapproved email. Closes #179 177 | - Fixed bug with saving errors in MySQL configuration when Releem Platform reply with errors. 178 | 179 | Releem 1.6.0, 2023-07-31 ([What's New At Releem | July 2023](https://releem.com/blog/whats-new-at-releem-july-2023)) 180 | - Added IOPS graph to Releem Dashboard. 181 | - Added System Information to System Metrics block. Closes #169 182 | - Improved MySQL Metrics graph and included ‘Applying Configuration’ events on the timeline. 183 | - Improved graphs and made Y-axis absolute and not relative. Closes #167 184 | - Removed innodb_flush_log_at_trx_commit automatic recommendations. Closes #170 185 | - Fixed minor issues with the 'innodb_buffer_pool_instance' and 'thread_cache' MySQL variables. 186 | - Improved Releem Agent installation for older MySQL and MariaDB versions without Performance Schema. 187 | 188 | Releem 1.5.0, 2023-06-30 ([What's New At Releem | June 2023](https://releem.com/blog/whats-new-at-releem-june-2023)) 189 | - Improved the Recommended configuration window to show users all variables that Releem tunes and details on variables. 190 | - Improved MySQL metric charts with buttons and avg metrics remove Latency and SlowLog 191 | - Improved design of Recommendation block re current applied configuration and enable Configure button. 192 | - Fixed bug in Releem Agent to work with old databases. Closes #163 193 | - Fixed bug in full data metrics collection prevented collecting minute metrics. 194 | - Added collecting configuration performance metric (Latency) in the period when configuration applied 195 | - Added support of MariaDB 11 196 | 197 | Releem 1.4.0, 2023-05-31 ([What's New At Releem | May 2023](https://releem.com/blog/whats-new-at-releem-may-2023)) 198 | - Improved “Add server” page to simplify the installation depending on environment 199 | - Added new states for Recommendation block to make clear current state of Releem. 200 | - Fixed bug in Releem Agent to collect information on database size 1 time in 12 hours to prevent performance issues. 201 | - Add change period of all metrics collection in docker. Closes #161 202 | - Added new variables 'innodb_change_buffering', 'innodb_autoextend_increment', 'innodb_change_buffer_max_size', 'thread_stack', 'innodb_adaptive_flushing_lwm', 'transaction_prealloc_size', 'innodb_max_dirty_pages_pct' 203 | 204 | Releem 1.3.0, 2023-04-30 205 | - Improved [Documentation](https://releem.com/docs/getstarted) 206 | - Fixed bug agents for AWS periodical restarts. Closes #159 207 | - Fixed bug in Releem Agents calculation of iops for cpanel with cagefs. Closes #149 208 | - Added fast detection of applying MySQL configuration 209 | - Added detection that MySQL server was restarted 210 | - Added support for arm64 architecture 211 | 212 | Releem 1.2.0, 2023-03-31 ([What's New At Releem | March 2023](https://releem.com/blog/whats-new-at-releem-march-2023)) 213 | - Added deletion servers in the Releem Customer Portal 214 | - Improved charts performance in the Releem Customer Portal 215 | - Added a start screen for users without servers in the Releem Customer Portal 216 | - Improved the installation process of Releem Agent and show users if Agent installed not properly 217 | - Added hostname for Releem Agent in docker containers 218 | - Added Events 219 | 220 | Releem 1.1.0, 2023-02-28 ([What's New At Releem | February 2023](https://releem.com/blog/whats-new-in-releem-february-2023)) 221 | - Added Display RDS instanses in the Releem Customer Portal 222 | - Added [MySQL Health Checks](https://releem.com/blog/mysql-health-checks) in the Releem Customer Portal 223 | - Redesigned Recommendation block in the Releem Customer Portal 224 | - Renamed and redesigned MySQL Performance Score block to Releem Score 225 | - Added Releem Agent Uninstallation 226 | - Fixed MySQL socket detection in mysql_host 227 | 228 | Releem 1.0.0, 2023-01-31 ([What’s New At Releem | January 2023](https://releem.com/blog/whats-new-at-releem-january-2023)) 229 | - Added new insights (QPS and Latency) to Weekly Reports 230 | - Added CPU, IOPS, Memory charts for all users in the Releem Customer Portal 231 | - Added Collecting RDS metrics from Enhanced monitoring 232 | - Added Period selector to see data on graphs for more than 1 day in the Releem Customer Portal 233 | - Added Initialize server in docker. 234 | - Added Automated deployment via Fargate in AWS account. 235 | - Fixed connection to db using hostname. 236 | - Fixed default value to timer. 237 | - Fixed Output after successfull installation. Closes #142 238 | - Fixed Set domain name in mysql_host automatically in case using RDS. Closes #138 239 | - Fixed Agent crashed when set domain name instead of IP in mysql_host. Closes #137 240 | - Fixed Failed installation when password contains "!". Closes #121 241 | 242 | Releem 0.9.9, 2022-12-31 243 | - Added system metrics collection CPU, RAM, Swap, IOPS 244 | - Added Slow Log Graph in the Releem Customer Portal 245 | - Added CPU, IOPS, and Memory gauges for all users in the Releem Customer Portal 246 | - Added Docker integration container. Closes #108 247 | - Added Connection to MySQL via socket. Closes #117 248 | - Added All Servers page in the Releem Customer Portal 249 | - Improved Best Practices and Recommendations Block in the Releem Customer Portal 250 | - Improved Documentation 251 | - Fixed Installation with custome ip address. Closes #118 252 | - Fixed Releem Agent stopped after server reboot. Closes #122 253 | - Fixed During installation /etc/mysql/my.cnf was broke. Closes #126 254 | 255 | Releem 0.9.8, 2022-11-30 256 | - Added slow log queriest collection 257 | - Added Latency Graph in the Releem Customer Portal 258 | - Added collecting metrics from Performance Scheme 259 | - Improved Releem Agent installation process just in one command 260 | - Fixed output color. Closes #109 261 | - Fixed Exclude "MySQL client" information. Closes #45 262 | - Fixed Can't open error on CloudLinux. Closes #101 263 | 264 | Releem 0.9.7, 2022-10-31 265 | - Added installation logs collection. 266 | - Improved metrics collection using new Releem Agent implemented using Go. 267 | - Improved installation. Removed cron to collect metrics. 268 | - Redesigned front page of Releem Customer Portal. 269 | - Fixed run installation with sudo user. 270 | 271 | Releem 0.9.6, 2022-09-30 272 | - Added Queries per Second metric collection 273 | - Added graph QPS in Releem Customer Portal 274 | - Redesigned Dashboard in Releem Customer Portal 275 | - Added Weekly Reelem Reports 276 | - Improved server metrics aggregation algorithm (hostnames instead IP adresses) Closes #100 277 | - Fixed warning during execution. Closes #93 278 | 279 | Releem 0.9.5, 2022-08-31 280 | - Added Apply recommended MySQL configuration and rollback to previous configuration. Closes #63 281 | - Added Automatic update. 282 | - Added innodb_page_cleaners and innodb_purge_threads calculations. 283 | - Improved performance of Releem Agent minimize workload an run on servers with hundreds databases. Closes #30. Closes #58 284 | 285 | Releem 0.9.4, 2022-07-31 286 | - Added FreeBSD support. Closes #95 287 | - Added innodb_redo_log_capacity calculation 288 | - Added query_cache_min_res_unit calculation 289 | - Improved calculation thread_cache_size. Closes #91 290 | - Fixed Error is:'int' object has no attribute 'strip' 291 | - Fixed Error KeyError: 'Virtual Machine' 292 | 293 | Releem 0.9.3, 2022-06-30 294 | - Added thread_pool_size calculation. 295 | - Improved performance of metrics page. 296 | - Improved [MySQL Performance Score](https://releem.com/docs/mysql-performance-score?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md). 297 | - Fixed [innodb_buffer_pool_size](https://releem.com/docs/mysql-performance-tuning/innodb_buffer_pool_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) calculation. 298 | - Fixed height of Recommended Configuration block. 299 | 300 | Releem 0.9.2, 2022-05-31 301 | - Added manual selection of [innodb_flush_log_at_trx_commit](https://releem.com/docs/mysql-performance-tuning/innodb_flush_log_at_trx_commit?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) in Releem Customer Portal for every server. 302 | - Added innodb_log_buffer_size calculaction. 303 | - Added optimizer_search_depth calculaction. 304 | - Improved [innodb_log_file_size](https://releem.com/docs/mysql-performance-tuning/innodb_log_file_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) variable. Closes #3 305 | - Improved [Documentation](https://releem.com/docs/getstarted). 306 | - Fixed Metrics Collection Issue. 307 | 308 | Releem Agent 0.9.1, 2022-04-30 309 | - Added display of Memory Limit in Releem Customer Portal 310 | - Improved [MySQL Performance Score](https://releem.com/docs/mysql-performance-score?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) 311 | - Fixed duplicated servers in Releem Customer Portal 312 | - Removed servers where Releem Agent is not active. 313 | 314 | Releem MySQLConfigurer 0.9.0, 2022-03-30 315 | - Added checks of the database server version 316 | - Added configuration file releem.conf 317 | - Added -u option to update Releem Agent 318 | - Added list of variable changes. Closes #75 319 | - Improved calculation of [max_heap_table_size](https://releem.com/docs/mysql-performance-tuning/max_heap_table_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) 320 | - Improved calculation of [tmp_table_size](https://releem.com/docs/mysql-performance-tuning/tmp_table_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) 321 | - Fixed MySQLTuner version 322 | - Fixed metrics collection 323 | 324 | Releem MySQLConfigurer 0.8.0, 2022-01-12 325 | - Added support of MariaDB 10.6. Closes #82 326 | - Added Automated subscriptions and credit card payments 327 | - Added hostnames to servers list. Closes #77 328 | - Improved documentation 329 | 330 | Releem MySQLConfigurer 0.7.0, 2021-11-16 331 | - Added Display timezone on server page. Closes #72 332 | - Added [Documentation](https://releem.com/docs/getstarted). 333 | - Added Automated Installation of Releem Agent 334 | - Fixed Cache values too high. Closes #73 335 | - Fixed Error when no Innodb tables only MyISAM. Closes #76 336 | - Fixed The values on the left and right are not in the same terminology. Closes #74 337 | - Removed Removed MySQLTuner Recommendations 338 | 339 | Releem MySQLConfigurer 0.6.0, 2021-06-17 340 | - Added [MySQL Performance Score](https://releem.com/docs/mysql-performance-score?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md) metric. 341 | - Added runtime information. Closes #62 342 | - Added Display Recommended Configuration. 343 | - Improved documentation Installation, Usage and Tests. 344 | - Improved calcualtion of the 'myisam_sort_buffer_size' variable. 345 | - Improved calculation of the 'read_rnd_buffer_size' variable. 346 | - Improved calculation of the 'sort_buffer_size' variable. 347 | - Removed usage of "mysqltuner.pl" domain. 348 | 349 | Releem MySQLConfigurer 0.5.0, 2021-01-30 350 | - Added simple one step installation process. Closes #23. 351 | - Improved documentation. 352 | - Improved and published tests description at [releem.com](https://releem.com/blog/how-to-improve-performance-mysql57-default-configuration). Closes #31. 353 | - Fixed problem with timeout variables. Closes #29. 354 | - Added calculation of the '[max_allowed_packet](https://releem.com/docs/mysql-performance-tuning/max_allowed_packet?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md)' variable. 355 | - Added calculation of the 'read_rnd_buffer_size' variable. 356 | - Improved calcualtion of the 'sort_buffer_size' variable. 357 | - Improved calculation of the '[innodb_buffer_pool_size](https://releem.com/docs/mysql-performance-tuning/innodb_buffer_pool_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md)' variable. 358 | - Improved calculation of the 'key_buffer_size' variable. 359 | - Improved calculation of the 'innodb_buffer_pool_instances' variable. Closes #37. 360 | 361 | Releem MySQLConfigurer 0.4.0, 2020-11-21 362 | - Improved documentation 363 | - Added option -m to set memory limit for MySQL in MBs. Closes #42. 364 | - Fixed downloading MySQLTuner every launch. Closes #46. 365 | - Added option -k - Releem API Key authorization. 366 | - Created Releem Community groups on Slack and Telegram. 367 | 368 | MySQL Configurer 0.3.2, 2020-08-24 369 | - Added MySQL 8 support. Closes #39 370 | - Fixed calculation of the 'key_buffer_size' variable for MySQL 8.0. 371 | - Tested compatibility with MySQL 5.5, MySQL 5.6, MySQL 5.7, MySQL 8.0, MariaDB 10.1, MariaDB 10.2, MariaDB 10.3, MariaDB 10.4, MariaDB 10.5. 372 | - Improved documentation with Security section. 373 | - Improved documentation with information about setting open_files_limit. 374 | - Improved documentation with installation perl-Data-Dumper module on Centos. 375 | 376 | MySQL Configurer 0.3.1, 2020-07-08 377 | - Added calculation of the '[table_open_cache](https://releem.com/docs/mysql-performance-tuning/table_open_cache)' variable. 378 | - Added calculation of the 'table_definition_cache' variable. Closes #18 379 | 380 | MySQL Configurer 0.3.0, 2020-06-24 381 | - Tested compatibility with MySQL 5.5, MySQL 5.6, MySQL 5.7, MariaDB 10.1, MariaDB 10.2, MariaDB 10.3. 382 | - Added calculation of the '[key_buffer_size](https://releem.com/docs/mysql-performance-tuning/key_buffer_size)' variable for improve performance of the MyIsam storage engine. 383 | - Added calculation of the '[innodb_buffer_pool_chunk_size](https://releem.com/docs/mysql-performance-tuning/innodb_buffer_pool_chunk_size)' variable for MySQL 5.7.5 and later, MariaDB 10.2.2 and later. 384 | - Added calculation of the '[max_connections](https://releem.com/docs/mysql-performance-tuning/max_connections)' variable based on 'Max_used_connections' MySQL status variable. 385 | - Improve calculation of the '[innodb_log_file_size](https://releem.com/docs/mysql-performance-tuning/innodb_log_file_size)' variable using 'innodb_log_files_in_group' variable. 386 | - Improve documentation with install dependencies step for Debian/Ubuntu and Centos/Redhat. 387 | - Fix documentation. Update example of the recommended configuration file. Closes #35 388 | - Fix documentation. How to safely apply the configuration file. Closes #36 389 | 390 | MySQL Configurer 0.2.2, 2020-04-25 391 | - Improve documentation. Added supported MySQL versions. Closes #22 392 | - Imrove stability. Response message for incompatible report. Closes #10 393 | 394 | MySQL Configurer 0.2.1, 2020-04-11 395 | - Fixed rename file z_aiops_mysql.conf -> z_aiops_mysql.cnf. Issue #14 was closed 396 | - Added rounding of variables. Issue #17 was closed. 397 | - Added calculation '[max_connections](https://releem.com/docs/mysql-performance-tuning/max_connections?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md)'. Issue #16 was closed. 398 | - Added calculation '[thread_cache_size](https://releem.com/docs/mysql-performance-tuning/thread_cache_size?utm_source=github&utm_medium=social&utm_campaign=changelog&utm_content=md)'. Issue #15 was closed. 399 | - Improve documentation. Issue #13 was closed. 400 | 401 | MySQL Configurer 0.1.2, 2020-01-15 402 | - Fixed "internal server error" in logging subsystem returned when the mysqltuner report contains empty parameter name. Issue #9 was closed. 403 | 404 | MySQL Configurer 0.1.1, 2020-01-11 405 | - Added check MySQLTuner exit code for prevent invalid requests to API. Issue #5 was closed 406 | - Added -s option for the curl command to hide unnecessary output. Issue #2 was closed 407 | - Fixed documentation and added check for JSON module. Issue #6 was closed 408 | - Added old values to configuration file. Issue #4 was closed 409 | - Fixed calculations of the innodb_buffer_pool_instances. Issue #1 was closed 410 | - Improve advanced output for errors 411 | 412 | MySQL Configurer 0.1.0, 2019-12-25 413 | First release 414 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian 2 | 3 | ARG DB_HOST 4 | ARG DB_PORT 5 | ARG DB_PASSWORD 6 | ARG DB_USER 7 | 8 | ARG RELEEM_API_KEY 9 | ARG MEMORY_LIMIT 10 | 11 | ARG INSTANCE_TYPE 12 | ARG AWS_REGION 13 | ARG AWS_RDS_DB 14 | ARG AWS_RDS_PARAMETER_GROUP 15 | 16 | ARG RELEEM_ENV 17 | ARG RELEEM_DEBUG 18 | ARG RELEEM_HOSTNAME 19 | ARG RELEEM_INTERVAL_COLLECT_ALL_METRICS 20 | ARG RELEEM_QUERY_OPTIMIZATION 21 | ARG RELEEM_DATABASES_QUERY_OPTIMIZATION 22 | ARG RELEEM_REGION 23 | 24 | RUN apt update \ 25 | && apt install -y \ 26 | curl \ 27 | mariadb-client 28 | 29 | RUN curl -L https://github.com/a8m/envsubst/releases/download/v1.4.2/envsubst-`uname -s`-`uname -m | sed 's/aarch64/arm64/g'` -o envsubst \ 30 | && chmod +x envsubst \ 31 | && mv envsubst /usr/local/bin 32 | 33 | WORKDIR /opt/releem 34 | RUN mkdir /opt/releem/conf 35 | 36 | COPY docker/ /docker/ 37 | 38 | RUN curl -L -o releem-agent https://releem.s3.amazonaws.com/v2/releem-agent-$(arch) \ 39 | && curl -L -o mysqlconfigurer.sh https://releem.s3.amazonaws.com/v2/mysqlconfigurer.sh \ 40 | && chmod +x releem-agent mysqlconfigurer.sh /docker/entrypoint.sh 41 | 42 | RUN mkdir -p /etc/mysql/releem.conf.d 43 | 44 | ENTRYPOINT [ "/docker/entrypoint.sh" ] 45 | CMD ["/opt/releem/releem-agent"] 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Releem Agent 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/Releem/mysqlconfigurer)](https://goreportcard.com/report/github.com/Releem/mysqlconfigurer) 4 | [![slack](https://img.shields.io/badge/slack-Releem%20Community-brightgreen.svg?logo=slack)](https://join.slack.com/t/releem-community/shared_invite/zt-1j3d0vosh-AJHbDiQrzVDvLat5eqQorQ) 5 | 6 |

7 | 8 | Releem 9 | 10 |

11 |

12 | Docs | 13 | Security | 14 | Compare MySQLTuner | 15 | SQL Query Optimization | 16 | MySQL Optimization Center | 17 | Blog 18 |

19 |

20 | 21 | 22 | 23 | The present repository contains the source code of the **Releem Agent**. 24 | 25 | [Releem](https://releem.com) is a MySQL performance monitoring tool that delivers consistent performance through continuous database profiling, configuration tuning, and SQL query optimization. 26 | 27 | With Releem we are trying to bring top-notch experience in database performance management and save thousands of software engineers hours. 28 | 29 |

30 | 31 |

32 | 33 | ## Why Releem? 34 | - **Clutter Free**: Releem provides simple dashboard and it cuts through the noise. No layers of menus, no need for custom reports. Get all the important metrics on one single page. No training necessary. 35 | - **Hassle free**: Simple one-step Installation on most popular Linux platforms and Support of all MySQL/MariaDB/Percona versions. 36 | - **Performance Booster**: Recommended configuration delivers up to [290% boost](#Tests) to MySQL performance compare to the default configuration. 37 | - **Simplified Monitoring**: [MySQL Health Checks](https://releem.com/blog/mysql-health-checks?utm_source=github&utm_medium=social&utm_campaign=mysql-health-checks&utm_content=post) greatly simplifies the process of monitoring and maintaining a healthy database by focusing on key aspects that describe the efficiency and "best practices" of using Memory, Connections, Logs, Cache, Disk, Indexes, and Threads. Releem Score metric calculates by summarizing Health Checks statuses. 38 | - **Automatic SQL Query Optimization and Index suggestions**: Releem automatically identifies inefficient queries and offers missed indexes, enabling database administrators to boost query performance without extensive manual analysis. 39 | - **Security**: Releem Agent is open-source and does not collect your database data. [Learn more](#security) 40 | - **Email report**: Keep an eye on your servers with weekly email reports. 41 | - **Simple Applying**: Releem Agent allows simply apply recommended MySQL configuration just in one click or in one command. 42 |

43 | 44 |

45 | 46 |

47 | 48 |

49 | 50 | ## How it works 51 | 52 | **Releem Agent** - Has been installed on servers, collects MySQL metrics, sends them to Cloud Platforms, and applies MySQL configurations. Open Source daemon built on Go. 53 | 54 | **Releem Cloud Platform** - Analyzes collected metrics, detects performance issues, and recommends MySQL configurations. 55 | 56 | **Releem Customer Portal** - Web interface displays recommended configurations and current information about all MySQL servers with installed Releem Agent. It looks like this on the screenshot. 57 | 58 | ## Getting started with Releem 59 | The easiest way to get started with Releem is with [our managed service in the cloud](https://releem.com) and one step installation command. It takes up to 5 minutes to start monitoring your MySQL servers and get recommendations to improve performance. 60 | 61 | To start using Releem just sign up at [https://releem.com](https://releem.com/?utm_source=github&utm_medium=link&utm_campaign=signup#) and install Releem Agent on your server. 62 | 63 | ## Security 64 | 65 | Releem does not collect any user data. 66 | 67 | The Releem agent is open-source and does not require opening ports. 68 | 69 | The Releem agent collects the following data: 70 | - Memory, CPU, and disk usage statistics 71 | - MySQL system variables & status information 72 | - Data size statistics from information_schema 73 | - Table & schema names (but not actual table content) 74 | - Table structure details, indexes, and usage statistics for schema optimization 75 | - Query execution statistics with placeholders from performance_schema, including execution counts, average execution time, query example, and EXPLAIN plans for the top queries 76 | 77 | ## Support 78 | Join the Releem Community on [Slack](https://join.slack.com/t/releem-community/shared_invite/zt-1j3d0vosh-AJHbDiQrzVDvLat5eqQorQ). 79 | 80 | ## Compatibility 81 | - MySQL 8.0, MySQL 5.7, MySQL 5.6, MySQL 5.5 82 | - MariaDB 10.1, MariaDB 10.2, MariaDB 10.3, MariaDB 10.4, MariaDB 10.5, MariaDB 10.6, MariaDB 10.7, MariaDB 10.8, MariaDB 10.9, MariaDB 10.10, MariaDB 10.11, MariaDB 11.0 83 | - Percona 8.0, Percona 5.7, Percona 5.6, Percona 5.5 84 | - Centos, CloudLinux, Debian, Ubuntu, RockyLinux 85 | - Windows Server 2012, Windows Server 2016, Windows Server 2019, Windows Server 2022, Windows Server 2025 86 | - Amazon RDS MySQL, Amazon RDS Aurora, Amazon RDS MariaDB 87 | 88 | *** MINIMAL REQUIREMENTS *** 89 | - Unix/Linux based operating system (tested on Linux, BSD variants, and Solaris variants) 90 | - Unrestricted read access to the MySQL server 91 | 92 | ## Tests 93 | We tested the results with Sysbench on a virtual server running Debian 9 (2 CPU, 2GB Ram) the table contained 10 million entries. 94 | Two configurations were tested, the MySQL default configuration and the configuration recommended by the **Releem** service. The tests were two-step: read (test1) only and read/write (test2). 95 | 96 | Recommended configuration delivered a 30% boost to MySQL performance compared to the default configuration. 97 | 98 | Follow this links to see results: 99 | - [MySQL 5.7 Benchmark](https://releem.com/blog/how-to-improve-performance-mysql57-default-configuration) 100 | - [MySQL 8 Benchmark](https://releem.com/blog/mysql-8-performance-benchmark) 101 | - [How MySQL Configuration Impacts the Performance of Web Applications](https://releem.com/blog/web-applications-performance) 102 | 103 | ## Feedback 104 | We welcome feedback from our community. Take a look at our [feedback board](https://releem.com/wall-of-love). Please let us know if you have any requests and vote on open issues so we can better prioritize. 105 | 106 | To stay up to date with all the latest news and product updates, make sure to follow us on [Twitter](https://twitter.com/releemhq), [LinkedIn](https://www.linkedin.com/company/releem). 107 | 108 | ## Contribute 109 | 110 | You can help us by reporting problems, suggestions or contributing to the code. 111 | 112 | ### Report a problem or suggestion 113 | 114 | Go to our [issue tracker](https://github.com/releem/mysqlconfigurer/issues) and check if your problem is already reported. If not, create a new issue with a descriptive title and detail your suggestion or steps to reproduce the problem. 115 | 116 | If you have suggestions or want to discuss potential improvements, please visit our [Discussions](https://github.com/releem/mysqlconfigurer/discussions) page. We value your input and look forward to engaging with the community to enhance our product. 117 | 118 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o releem-agent-x86_64 2 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o releem-agent-amd64 3 | CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o releem-agent-aarch64 4 | CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -o releem-agent-i686 5 | CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -o releem-agent-freebsd-amd64 6 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o releem-agent.exe 7 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | logging "github.com/google/logger" 8 | "github.com/hashicorp/hcl" 9 | ) 10 | 11 | const ( 12 | ReleemAgentVersion = "1.21.3.1" 13 | ) 14 | 15 | type Config struct { 16 | Debug bool `hcl:"debug"` 17 | Env string `hcl:"env"` 18 | Hostname string `hcl:"hostname"` 19 | ApiKey string `hcl:"apikey"` 20 | MetricsPeriod time.Duration `hcl:"interval_seconds"` 21 | ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"` 22 | GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"` 23 | QueryOptimizationPeriod time.Duration `hcl:"interval_query_optimization_seconds"` 24 | QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"` 25 | MysqlPassword string `hcl:"mysql_password" json:"-"` 26 | MysqlUser string `hcl:"mysql_user"` 27 | MysqlHost string `hcl:"mysql_host"` 28 | MysqlPort string `hcl:"mysql_port"` 29 | MysqlSslMode bool `hcl:"mysql_ssl_mode"` 30 | CommandRestartService string `hcl:"mysql_restart_service"` 31 | MysqlConfDir string `hcl:"mysql_cnf_dir"` 32 | ReleemConfDir string `hcl:"releem_cnf_dir"` 33 | ReleemDir string `hcl:"releem_dir"` 34 | MemoryLimit int `hcl:"memory_limit"` 35 | InstanceType string `hcl:"instance_type"` 36 | AwsRegion string `hcl:"aws_region"` 37 | AwsRDSDB string `hcl:"aws_rds_db"` 38 | AwsRDSParameterGroup string `hcl:"aws_rds_parameter_group"` 39 | QueryOptimization bool `hcl:"query_optimization"` 40 | DatabasesQueryOptimization string `hcl:"databases_query_optimization"` 41 | ReleemRegion string `hcl:"releem_region"` 42 | } 43 | 44 | func LoadConfig(filename string, logger logging.Logger) (*Config, error) { 45 | logger.Infof("Loading config %s", filename) 46 | configBytes, err := os.ReadFile(filename) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return LoadConfigFromString(string(configBytes), logger) 51 | } 52 | 53 | func LoadConfigFromString(data string, logger logging.Logger) (*Config, error) { 54 | config := &Config{} 55 | err := hcl.Decode(&config, data) 56 | if err != nil { 57 | return nil, err 58 | } 59 | if config.MetricsPeriod == 0 { 60 | config.MetricsPeriod = 60 61 | } 62 | if config.ReadConfigPeriod == 0 { 63 | config.ReadConfigPeriod = 3600 64 | } 65 | if config.GenerateConfigPeriod == 0 { 66 | config.GenerateConfigPeriod = 43200 67 | } 68 | if config.QueryOptimizationPeriod == 0 { 69 | config.QueryOptimizationPeriod = 3600 70 | } 71 | if config.QueryOptimizationCollectSqlTextPeriod == 0 { 72 | config.QueryOptimizationCollectSqlTextPeriod = 1 73 | } 74 | if config.MysqlHost == "" { 75 | config.MysqlHost = "127.0.0.1" 76 | } 77 | if config.MysqlPort == "" { 78 | config.MysqlPort = "3306" 79 | } 80 | if config.ReleemDir == "" { 81 | config.ReleemDir = "/opt/releem" 82 | } 83 | return config, nil 84 | } 85 | 86 | func (config *Config) GetApiKey() string { 87 | return config.ApiKey 88 | } 89 | func (config *Config) GetEnv() string { 90 | return config.Env 91 | } 92 | 93 | func (config *Config) GetMemoryLimit() int { 94 | return config.MemoryLimit 95 | } 96 | func (config *Config) GetReleemConfDir() string { 97 | return config.ReleemConfDir 98 | } 99 | -------------------------------------------------------------------------------- /current_version_agent: -------------------------------------------------------------------------------- 1 | 1.21.3.1 -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | # # Substitute environment variables in Prosody configs 4 | envsubst < /docker/releem.conf.tpl > /opt/releem/releem.conf 5 | 6 | echo -e "### This configuration was recommended by Releem. https://releem.com\n[mysqld]\nperformance_schema = 1\nslow_query_log = 1" > "/etc/mysql/releem.conf.d/collect_metrics.cnf" 7 | if [ -n "$RELEEM_QUERY_OPTIMIZATION" -a "$RELEEM_QUERY_OPTIMIZATION" = true ]; then 8 | echo "performance-schema-consumer-events-statements-history = ON" | tee -a "/etc/mysql/releem.conf.d/collect_metrics.cnf" >/dev/null 9 | echo "performance-schema-consumer-events-statements-current = ON" | tee -a "/etc/mysql/releem.conf.d/collect_metrics.cnf" >/dev/null 10 | echo "performance_schema_events_statements_history_size = 500" | tee -a "/etc/mysql/releem.conf.d/collect_metrics.cnf" >/dev/null 11 | fi 12 | 13 | /opt/releem/releem-agent -f 14 | 15 | exec "$@" 16 | -------------------------------------------------------------------------------- /docker/releem.conf.tpl: -------------------------------------------------------------------------------- 1 | # ApiKey string `hcl:"apikey"` 2 | # Defaults to 3600 seconds, api key for Releem Platform. 3 | apikey="${RELEEM_API_KEY}" 4 | 5 | hostname="${RELEEM_HOSTNAME}" 6 | 7 | # MemoryLimit int `hcl:"memory_limit"` 8 | # Defaults to 0, Mysql memory usage limit. 9 | memory_limit=${MEMORY_LIMIT:-0} 10 | 11 | # MetricsPeriod time.Duration `hcl:"interval_seconds"` 12 | # Defaults to 30 seconds, how often metrics are collected. 13 | interval_seconds=60 14 | 15 | # ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"` 16 | # Defaults to 3600 seconds, how often to update the values from the config. 17 | interval_read_config_seconds=3600 18 | 19 | # GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"` 20 | # Defaults to 43200 seconds, how often to generate recommend the config. 21 | interval_generate_config_seconds=${RELEEM_INTERVAL_COLLECT_ALL_METRICS:-43200} 22 | 23 | # QueryOptimization time.Duration `hcl:"interval_query_optimization_seconds"` 24 | # Defaults to 3600 seconds, how often query metrics are collected. 25 | interval_query_optimization_seconds=3600 26 | 27 | # MysqlUser string`hcl:"mysql_user"` 28 | # Mysql user name for collection metrics. 29 | mysql_user="${DB_USER:-releem}" 30 | 31 | # MysqlPassword string `hcl:"mysql_password"` 32 | # Mysql user password for collection metrics. 33 | mysql_password="${DB_PASSWORD:-releem}" 34 | 35 | # MysqlHost string `hcl:"mysql_host"` 36 | # Mysql host for collection metrics. 37 | mysql_host="${DB_HOST:-127.0.0.1}" 38 | 39 | # MysqlPort string `hcl:"mysql_port"` 40 | # Mysql port for collection metrics. 41 | mysql_port="${DB_PORT:-3306}" 42 | 43 | # CommandRestartService string `hcl:"mysql_restart_service"` 44 | # Defaults to 3600 seconds, command to restart service mysql. 45 | mysql_restart_service=" /bin/systemctl restart mysql" 46 | 47 | # MysqlConfDir string `hcl:"mysql_cnf_dir"` 48 | # The path to copy the recommended config. 49 | mysql_cnf_dir="/etc/mysql/releem.conf.d" 50 | 51 | # ReleemConfDir string `hcl:"releem_cnf_dir"` 52 | # Releem Agent configuration path. 53 | releem_cnf_dir="/opt/releem/conf" 54 | 55 | # InstanceType string `hcl:"instance_type"` 56 | # Defaults to local, type of instance "local" or "aws/rds" 57 | instance_type="${INSTANCE_TYPE:-local}" 58 | 59 | # AwsRegion string `hcl:"aws_region"` 60 | # Defaults to us-east-1, AWS region for RDS 61 | aws_region="${AWS_REGION:-us-east-1}" 62 | 63 | # AwsRDSDB string `hcl:"aws_rds_db"` 64 | # RDS database name. 65 | aws_rds_db="${AWS_RDS_DB}" 66 | 67 | # AWS_RDS_PARAMETER_GROUP string `hcl:"aws_rds_parameter_group"` 68 | # RDS database parameter group name. 69 | aws_rds_parameter_group="${AWS_RDS_PARAMETER_GROUP}" 70 | 71 | # Env string `hcl:"env"` 72 | # Releem Environment. 73 | env="${RELEEM_ENV:-prod}" 74 | 75 | # Debug string `hcl:"debug"` 76 | # Releem Debug messages 77 | debug=${RELEEM_DEBUG:-false} 78 | 79 | # Collect Explain string `hcl:"query_optimization"` 80 | # Releem collect explain for query 81 | query_optimization=${RELEEM_QUERY_OPTIMIZATION:-false} 82 | 83 | # databases_query_optimization string `hcl:"databases_query_optimization"` 84 | # List of databases for query optimization 85 | databases_query_optimization="${RELEEM_DATABASES_QUERY_OPTIMIZATION}" 86 | 87 | # releem_region string `hcl:"releem_region"` 88 | # Server data storage region - EU or empty. 89 | releem_region="${RELEEM_REGION}" 90 | 91 | -------------------------------------------------------------------------------- /errors/releemErrors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/Releem/mysqlconfigurer/config" 8 | logging "github.com/google/logger" 9 | 10 | "time" 11 | ) 12 | 13 | type ReleemErrorsRepeater struct { 14 | logger logging.Logger 15 | configuration *config.Config 16 | } 17 | 18 | func (repeater ReleemErrorsRepeater) ProcessErrors(message string) interface{} { 19 | var env string 20 | bodyReader := strings.NewReader(message) 21 | 22 | repeater.logger.V(5).Info("Result Send data: ", message) 23 | var api_domain, domain string 24 | if repeater.configuration != nil { 25 | env = repeater.configuration.Env 26 | } else { 27 | env = "prod" 28 | } 29 | if repeater.configuration.ReleemRegion == "EU" { 30 | domain = "eu.releem.com" 31 | } else { 32 | domain = "releem.com" 33 | } 34 | if env == "dev2" { 35 | api_domain = "https://api.dev2." + domain + "/v2/events/agent_errors_log" 36 | } else if env == "dev" { 37 | api_domain = "https://api.dev." + domain + "/v2/events/agent_errors_log" 38 | } else if env == "stage" { 39 | api_domain = "https://api.stage." + domain + "/v2/events/agent_errors_log" 40 | } else { 41 | api_domain = "https://api." + domain + "/v2/events/agent_errors_log" 42 | } 43 | req, err := http.NewRequest(http.MethodPost, api_domain, bodyReader) 44 | if err != nil { 45 | repeater.logger.Error("Request: could not create request: ", err) 46 | return nil 47 | } 48 | if repeater.configuration != nil { 49 | req.Header.Set("x-releem-api-key", repeater.configuration.ApiKey) 50 | } 51 | 52 | client := http.Client{ 53 | Timeout: 30 * time.Second, 54 | } 55 | 56 | res, err := client.Do(req) 57 | if err != nil { 58 | repeater.logger.Error("Request: error making http request: ", err) 59 | return nil 60 | } 61 | repeater.logger.V(5).Info("Response: status code: ", res.StatusCode) 62 | return res 63 | } 64 | 65 | func NewReleemErrorsRepeater(configuration *config.Config, logger logging.Logger) ReleemErrorsRepeater { 66 | return ReleemErrorsRepeater{logger, configuration} 67 | } 68 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Releem/mysqlconfigurer 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/Releem/daemon v0.0.0-20241028135502-b7f24658ba58 7 | github.com/aws/aws-sdk-go v1.53.7 8 | github.com/aws/aws-sdk-go-v2 v1.27.0 9 | github.com/aws/aws-sdk-go-v2/config v1.27.15 10 | github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.3 11 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.4 12 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.3 13 | github.com/aws/aws-sdk-go-v2/service/rds v1.79.1 14 | github.com/go-sql-driver/mysql v1.8.1 15 | github.com/google/logger v1.1.1 16 | github.com/hashicorp/hcl v1.0.0 17 | github.com/pkg/errors v0.9.1 18 | github.com/shirou/gopsutil/v4 v4.24.10 19 | ) 20 | 21 | require ( 22 | filippo.io/edwards25519 v1.1.0 // indirect 23 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect 24 | github.com/aws/aws-sdk-go-v2/credentials v1.17.15 // indirect 25 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect 26 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.8 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.2 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.9 // indirect 34 | github.com/aws/smithy-go v1.20.2 // indirect 35 | github.com/ebitengine/purego v0.8.1 // indirect 36 | github.com/go-ole/go-ole v1.2.6 // indirect 37 | github.com/jmespath/go-jmespath v0.4.0 // indirect 38 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 39 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 40 | github.com/tklauser/go-sysconf v0.3.12 // indirect 41 | github.com/tklauser/numcpus v0.6.1 // indirect 42 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 43 | golang.org/x/sys v0.26.0 // indirect 44 | ) 45 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= 2 | filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= 3 | github.com/Releem/daemon v0.0.0-20241028135502-b7f24658ba58 h1:ISdzv2VqJ4GVjtvT0WAWsg9n2iKOjYMcMpSL3VJSoqg= 4 | github.com/Releem/daemon v0.0.0-20241028135502-b7f24658ba58/go.mod h1:GxsvZP6BRrh+rSHnXaXzuShSiiiMEfDxuutowpgCG8w= 5 | github.com/aws/aws-sdk-go v1.53.7 h1:ZSsRYHLRxsbO2rJR2oPMz0SUkJLnBkN+1meT95B6Ixs= 6 | github.com/aws/aws-sdk-go v1.53.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= 7 | github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo= 8 | github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= 9 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= 10 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= 11 | github.com/aws/aws-sdk-go-v2/config v1.27.15 h1:uNnGLZ+DutuNEkuPh6fwqK7LpEiPmzb7MIMA1mNWEUc= 12 | github.com/aws/aws-sdk-go-v2/config v1.27.15/go.mod h1:7j7Kxx9/7kTmL7z4LlhwQe63MYEE5vkVV6nWg4ZAI8M= 13 | github.com/aws/aws-sdk-go-v2/credentials v1.17.15 h1:YDexlvDRCA8ems2T5IP1xkMtOZ1uLJOCJdTr0igs5zo= 14 | github.com/aws/aws-sdk-go-v2/credentials v1.17.15/go.mod h1:vxHggqW6hFNaeNC0WyXS3VdyjcV0a4KMUY4dKJ96buU= 15 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE= 16 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw= 17 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 h1:lf/8VTF2cM+N4SLzaYJERKEWAXq8MOMpZfU6wEPWsPk= 18 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7/go.mod h1:4SjkU7QiqK2M9oozyMzfZ/23LmUY+h3oFqhdeP5OMiI= 19 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 h1:4OYVp0705xu8yjdyoWix0r9wPIRXnIzzOoUpQVHIJ/g= 20 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7/go.mod h1:vd7ESTEvI76T2Na050gODNmNU7+OyKrIKroYTu4ABiI= 21 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= 22 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= 23 | github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.3 h1:pGlGfpL+Su4SFonqREs/0u+Uq0iYv+FMhg2NmFHGBYo= 24 | github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.3/go.mod h1:ECX6i01ws5YQ8L58dwwoexhCmDR6hAV/sv+Q8IQ+jj4= 25 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.4 h1:QSIpvF/tE8Uoy+RNkbMpTahLZHLA1c6vi9tbSE7PZUY= 26 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.4/go.mod h1:OfO65DNsDX+wgWmjljN55I+Dzo4nbhWNlNFuco5AAgw= 27 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.3 h1:l0mvKOGm25yo/Fy+Y/08Cm4aTA4XmnIuq4ppy+shfMI= 28 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.3/go.mod h1:iJ2sQeUTkjNp3nL7kE/Bav0xXYhtiRCRP5ZXk4jFhCQ= 29 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= 30 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= 31 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY= 32 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y= 33 | github.com/aws/aws-sdk-go-v2/service/rds v1.79.1 h1:OzwXMImfUSYfmAxtZB1LC0ZM5PayF7llq3I7SSPDxcY= 34 | github.com/aws/aws-sdk-go-v2/service/rds v1.79.1/go.mod h1:/SU1vNf8MsUyfRkEkv3Hcz9y5uSTyBS+ohATQOj6ioQ= 35 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.8 h1:Kv1hwNG6jHC/sxMTe5saMjH6t6ZLkgfvVxyEjfWL1ks= 36 | github.com/aws/aws-sdk-go-v2/service/sso v1.20.8/go.mod h1:c1qtZUWtygI6ZdvKppzCSXsDOq5I4luJPZ0Ud3juFCA= 37 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.2 h1:nWBZ1xHCF+A7vv9sDzJOq4NWIdzFYm0kH7Pr4OjHYsQ= 38 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.2/go.mod h1:9lmoVDVLz/yUZwLaQ676TK02fhCu4+PgRSmMaKR1ozk= 39 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.9 h1:Qp6Boy0cGDloOE3zI6XhNLNZgjNS8YmiFQFHe71SaW0= 40 | github.com/aws/aws-sdk-go-v2/service/sts v1.28.9/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws= 41 | github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= 42 | github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= 43 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 45 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE= 47 | github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= 48 | github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= 49 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 50 | github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= 51 | github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= 52 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 53 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 54 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 55 | github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= 56 | github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= 57 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 58 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 59 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 60 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 61 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 62 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 63 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= 64 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= 65 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 66 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 67 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 68 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 69 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= 70 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 71 | github.com/shirou/gopsutil/v4 v4.24.10 h1:7VOzPtfw/5YDU+jLEoBwXwxJbQetULywoSV4RYY7HkM= 72 | github.com/shirou/gopsutil/v4 v4.24.10/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8= 73 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 74 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 75 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 76 | github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= 77 | github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= 78 | github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= 79 | github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= 80 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 81 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 82 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 83 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 86 | golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 87 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= 88 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 89 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 90 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 91 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 92 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 93 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 94 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 95 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Example of a daemon with echo service 2 | package main 3 | 4 | import ( 5 | "context" 6 | "flag" 7 | "io" 8 | "log" 9 | "os" 10 | "runtime" 11 | 12 | "github.com/Releem/daemon" 13 | "github.com/Releem/mysqlconfigurer/config" 14 | "github.com/Releem/mysqlconfigurer/metrics" 15 | "github.com/Releem/mysqlconfigurer/models" 16 | r "github.com/Releem/mysqlconfigurer/repeater" 17 | "github.com/Releem/mysqlconfigurer/utils" 18 | 19 | awsconfig "github.com/aws/aws-sdk-go-v2/config" 20 | "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" 21 | "github.com/aws/aws-sdk-go-v2/service/rds" 22 | "github.com/aws/aws-sdk-go/aws/awserr" 23 | _ "github.com/go-sql-driver/mysql" 24 | logging "github.com/google/logger" 25 | ) 26 | 27 | const ( 28 | // name of the service 29 | serviceName = "releem-agent" 30 | serviceDescription = "Releem Agent" 31 | ) 32 | 33 | var logger logging.Logger 34 | var SetConfigRun, GetConfigRun *bool 35 | var ConfigFile, AgentEvent, AgentTask *string 36 | 37 | // Service has embedded daemon 38 | type Service struct { 39 | daemon.Daemon 40 | } 41 | type Programm struct{} 42 | 43 | func (programm *Programm) Stop() { 44 | // Stop should not block. Return with a few seconds. 45 | } 46 | 47 | func (programm *Programm) Start() { 48 | // Start should not block. Do the actual work async. 49 | go programm.Run() 50 | } 51 | 52 | func (programm *Programm) Run() { 53 | 54 | var TypeConfiguration string 55 | var gatherers, gatherers_configuration, gatherers_query_optimization []models.MetricsGatherer 56 | var Mode models.ModeType 57 | 58 | if *SetConfigRun { 59 | TypeConfiguration = "set" 60 | } else if *GetConfigRun { 61 | TypeConfiguration = "get" 62 | } else { 63 | TypeConfiguration = "default" 64 | } 65 | 66 | // Do something, call your goroutines, etc 67 | logger.Info("Starting releem-agent of version is ", config.ReleemAgentVersion) 68 | configuration, err := config.LoadConfig(*ConfigFile, logger) 69 | if err != nil { 70 | logger.Error("Config load failed", err) 71 | return 72 | } 73 | defer utils.HandlePanic(configuration, logger) 74 | 75 | if configuration.Debug { 76 | logger.SetLevel(10) 77 | } else { 78 | logger.SetLevel(1) 79 | } 80 | 81 | if len(*AgentEvent) > 0 { 82 | Mode.Name = "Event" 83 | Mode.Type = *AgentEvent 84 | } else if len(*AgentTask) > 0 { 85 | Mode.Name = "TaskSet" 86 | Mode.Type = *AgentTask 87 | } else { 88 | Mode.Name = "Configurations" 89 | Mode.Type = TypeConfiguration 90 | } 91 | // if Mode.Name != "Event" { 92 | // Select how we collect instance metrics depending on InstanceType 93 | switch configuration.InstanceType { 94 | case "aws/rds": 95 | logger.Info("InstanceType is aws/rds") 96 | logger.Info("Loading AWS configuration") 97 | 98 | awscfg, err := awsconfig.LoadDefaultConfig(context.TODO(), awsconfig.WithRegion(configuration.AwsRegion)) 99 | if err != nil { 100 | logger.Error("Load AWS configuration FAILED", err) 101 | return 102 | } else { 103 | logger.Info("AWS configuration loaded SUCCESS") 104 | } 105 | 106 | cwlogsclient := cloudwatchlogs.NewFromConfig(awscfg) 107 | // cwclient := cloudwatch.NewFromConfig(awscfg) 108 | rdsclient := rds.NewFromConfig(awscfg) 109 | // ec2client := ec2.NewFromConfig(awscfg) 110 | 111 | // Prepare request to RDS 112 | input := &rds.DescribeDBInstancesInput{ 113 | DBInstanceIdentifier: &configuration.AwsRDSDB, 114 | } 115 | 116 | // Request to RDS 117 | result, err := rdsclient.DescribeDBInstances(context.TODO(), input) 118 | 119 | if err != nil { 120 | if aerr, ok := err.(awserr.Error); ok { 121 | logger.Error(aerr.Error()) 122 | return 123 | } else { 124 | // Print the error, cast err to awserr.Error to get the Code and 125 | // Message from an error. 126 | logger.Error(err.Error()) 127 | return 128 | } 129 | } 130 | 131 | logger.Info("RDS.DescribeDBInstances SUCCESS") 132 | 133 | // Request detailed instance info 134 | if result != nil && len(result.DBInstances) == 1 { 135 | // gatherers = append(gatherers, models.NewAWSRDSMetricsGatherer(logger, cwclient, configuration)) 136 | // gatherers = append(gatherers, models.NewAWSRDSInstanceGatherer(logger, rdsclient, ec2client, configuration)) 137 | configuration.Hostname = configuration.AwsRDSDB 138 | configuration.MysqlHost = *result.DBInstances[0].Endpoint.Address 139 | gatherers = append(gatherers, metrics.NewAWSRDSEnhancedMetricsGatherer(logger, result.DBInstances[0], cwlogsclient, configuration)) 140 | } else if result != nil && len(result.DBInstances) > 1 { 141 | logger.Infof("RDS.DescribeDBInstances: Database has %d instances. Clusters are not supported", len(result.DBInstances)) 142 | return 143 | } else { 144 | logger.Info("RDS.DescribeDBInstances: No instances") 145 | return 146 | } 147 | default: 148 | logger.Info("InstanceType is Local") 149 | gatherers = append(gatherers, metrics.NewOSMetricsGatherer(logger, configuration)) 150 | 151 | } 152 | 153 | models.DB = utils.ConnectionDatabase(configuration, logger, "mysql") 154 | defer models.DB.Close() 155 | 156 | //Init repeaters 157 | // repeaters := make(map[string]models.MetricsRepeater) 158 | // repeaters["Metrics"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "Metrics", Type: ""})) 159 | // repeaters["Configurations"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode)) 160 | // repeaters["Event"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode)) 161 | // repeaters["TaskGet"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "TaskGet", Type: ""})) 162 | // repeaters["TaskStatus"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "TaskStatus", Type: ""})) 163 | // repeaters["TaskSet"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, Mode)) 164 | // repeaters["GetConfigurationJson"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "Configurations", Type: "get-json"})) 165 | // repeaters["QueryOptimization"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "Metrics", Type: "QuerysOptimization"})) 166 | // repeaters["QueriesOptimization"] = models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, models.Mode{Name: "TaskSet", Type: "queries_optimization"})) 167 | //var repeaters models.MetricsRepeater 168 | repeaters := models.MetricsRepeater(r.NewReleemConfigurationsRepeater(configuration, logger)) 169 | 170 | //Init gatherers 171 | gatherers = append(gatherers, 172 | metrics.NewDbConfGatherer(logger, configuration), 173 | metrics.NewDbInfoBaseGatherer(logger, configuration), 174 | metrics.NewDbMetricsBaseGatherer(logger, configuration), 175 | metrics.NewAgentMetricsGatherer(logger, configuration)) 176 | gatherers_configuration = append(gatherers_configuration, metrics.NewDbMetricsGatherer(logger, configuration), metrics.NewDbInfoGatherer(logger, configuration)) 177 | gatherers_query_optimization = append(gatherers_query_optimization, metrics.NewDbCollectQueriesOptimization(logger, configuration)) 178 | 179 | metrics.RunWorker(gatherers, gatherers_configuration, gatherers_query_optimization, repeaters, logger, configuration, Mode) 180 | 181 | } 182 | 183 | // Manage by daemon commands or run the daemon 184 | func (service *Service) Manage(command []string) (string, error) { 185 | usage := "Usage: myservice install | remove | start | stop | status" 186 | // if received any kind of command, do it 187 | if len(command) >= 1 { 188 | switch command[0] { 189 | case "install": 190 | return service.Install() 191 | case "remove": 192 | return service.Remove() 193 | case "start": 194 | return service.Start() 195 | case "stop": 196 | return service.Stop() 197 | case "status": 198 | return service.Status() 199 | default: 200 | return usage, nil 201 | } 202 | } 203 | 204 | return service.Run(&Programm{}) 205 | 206 | // never happen, but need to complete code 207 | } 208 | func defaultConfigPath() string { 209 | switch runtime.GOOS { 210 | case "windows": 211 | return "C:\\ProgramData\\ReleemAgent\\releem.conf" 212 | default: // для Linux и других UNIX-подобных систем 213 | return "/opt/releem/releem.conf" 214 | } 215 | } 216 | func defaultDependencies() []string { 217 | switch runtime.GOOS { 218 | case "windows": 219 | return []string{} 220 | default: // для Linux и других UNIX-подобных систем 221 | return []string{"network.target"} 222 | } 223 | } 224 | 225 | func defaultSystemLogFlag() bool { 226 | switch runtime.GOOS { 227 | case "windows": 228 | return true 229 | default: // для Linux и других UNIX-подобных систем 230 | return false 231 | } 232 | } 233 | 234 | func main() { 235 | logger = *logging.Init("releem-agent", true, defaultSystemLogFlag(), io.Discard) 236 | defer logger.Close() 237 | logging.SetFlags(log.LstdFlags | log.Lshortfile) 238 | 239 | defaultPath := defaultConfigPath() 240 | SetConfigRun = flag.Bool("f", false, "Releem agent generate config") 241 | GetConfigRun = flag.Bool("c", false, "Releem agent get config") 242 | ConfigFile = flag.String("config", defaultPath, "Releem agent config") 243 | AgentEvent = flag.String("event", "", "Releem agent type event") 244 | AgentTask = flag.String("task", "", "Releem agent task name") 245 | flag.Parse() 246 | command := flag.Args() 247 | 248 | dependencies := defaultDependencies() 249 | srv, err := daemon.New(serviceName, serviceDescription, daemon.SystemDaemon, dependencies...) 250 | if err != nil { 251 | logger.Error("Error: ", err) 252 | os.Exit(1) 253 | } 254 | service := &Service{srv} 255 | status, err := service.Manage(command) 256 | 257 | if err != nil { 258 | logger.Info(status, "\nError: ", err) 259 | os.Exit(1) 260 | } 261 | logger.Info(status) 262 | } 263 | -------------------------------------------------------------------------------- /metrics/agent.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/Releem/mysqlconfigurer/config" 7 | "github.com/Releem/mysqlconfigurer/models" 8 | "github.com/Releem/mysqlconfigurer/utils" 9 | logging "github.com/google/logger" 10 | ) 11 | 12 | type AgentMetricsGatherer struct { 13 | logger logging.Logger 14 | configuration *config.Config 15 | } 16 | 17 | func NewAgentMetricsGatherer(logger logging.Logger, configuration *config.Config) *AgentMetricsGatherer { 18 | return &AgentMetricsGatherer{ 19 | logger: logger, 20 | configuration: configuration, 21 | } 22 | } 23 | 24 | func (Agent *AgentMetricsGatherer) GetMetrics(metrics *models.Metrics) error { 25 | defer utils.HandlePanic(Agent.configuration, Agent.logger) 26 | 27 | output := make(map[string]interface{}) 28 | output["Version"] = config.ReleemAgentVersion 29 | if len(Agent.configuration.Hostname) > 0 { 30 | output["Hostname"] = Agent.configuration.Hostname 31 | } 32 | output["QueryOptimization"] = Agent.configuration.QueryOptimization 33 | models.SqlTextMutex.RLock() 34 | output["QueryOptimizationSqlTextCount"] = len(models.SqlText) 35 | models.SqlTextMutex.RUnlock() 36 | 37 | var m runtime.MemStats 38 | runtime.ReadMemStats(&m) 39 | output["AllocMemory"] = m.Alloc 40 | output["TotalAllocMemory"] = m.TotalAlloc 41 | 42 | metrics.ReleemAgent.Info = output 43 | metrics.ReleemAgent.Conf = *Agent.configuration 44 | 45 | Agent.logger.V(5).Info("CollectMetrics Agent ", output) 46 | return nil 47 | 48 | } 49 | -------------------------------------------------------------------------------- /metrics/awsrdsdiscovery.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/Releem/mysqlconfigurer/config" 8 | "github.com/Releem/mysqlconfigurer/models" 9 | "github.com/Releem/mysqlconfigurer/utils" 10 | logging "github.com/google/logger" 11 | 12 | "github.com/aws/aws-sdk-go-v2/service/ec2" 13 | "github.com/aws/aws-sdk-go-v2/service/ec2/types" 14 | "github.com/aws/aws-sdk-go-v2/service/rds" 15 | "github.com/aws/aws-sdk-go/aws/awserr" 16 | ) 17 | 18 | type AWSRDSInstanceGatherer struct { 19 | logger logging.Logger 20 | debug bool 21 | rdsclient *rds.Client 22 | ec2client *ec2.Client 23 | configuration *config.Config 24 | } 25 | 26 | func NewAWSRDSInstanceGatherer(logger logging.Logger, rdsclient *rds.Client, ec2client *ec2.Client, configuration *config.Config) *AWSRDSInstanceGatherer { 27 | return &AWSRDSInstanceGatherer{ 28 | logger: logger, 29 | debug: configuration.Debug, 30 | rdsclient: rdsclient, 31 | ec2client: ec2client, 32 | configuration: configuration, 33 | } 34 | } 35 | 36 | func (awsrdsinstance *AWSRDSInstanceGatherer) GetMetrics(metrics *models.Metrics) error { 37 | defer utils.HandlePanic(awsrdsinstance.configuration, awsrdsinstance.logger) 38 | 39 | //output := make(models.MetricGroupValue) 40 | info := make(models.MetricGroupValue) 41 | 42 | // Prepare request to RDS 43 | input := &rds.DescribeDBInstancesInput{ 44 | DBInstanceIdentifier: &awsrdsinstance.configuration.AwsRDSDB, 45 | } 46 | 47 | // Request to RDS 48 | result, err := awsrdsinstance.rdsclient.DescribeDBInstances(context.TODO(), input) 49 | 50 | if err != nil { 51 | if aerr, ok := err.(awserr.Error); ok { 52 | awsrdsinstance.logger.Error(aerr.Error()) 53 | 54 | } else { 55 | // Print the error, cast err to awserr.Error to get the Code and 56 | // Message from an error. 57 | awsrdsinstance.logger.Error(err.Error()) 58 | } 59 | } else { 60 | awsrdsinstance.logger.Info("RDS.DescribeDBInstances SUCCESS") 61 | 62 | // Request detailed instance info 63 | if len(result.DBInstances) == 1 { 64 | 65 | r := result.DBInstances[0] 66 | 67 | awsrdsinstance.logger.V(5).Info("DBInstance ", r.DBInstanceIdentifier) 68 | awsrdsinstance.logger.V(5).Info("DBInstanceClass ", r.DBInstanceClass) 69 | awsrdsinstance.logger.V(5).Info("ProcessorFeatures ", r.ProcessorFeatures) 70 | 71 | // Prepare request to Ec2 to determine CPU count and Ram for InstanceClass 72 | instanceName := strings.TrimPrefix(*r.DBInstanceClass, "db.") 73 | ec2input := &ec2.DescribeInstanceTypesInput{ 74 | InstanceTypes: []types.InstanceType{types.InstanceType(instanceName)}, 75 | } 76 | 77 | // Request to EC2 to get Type info 78 | typeinfo, infoerr := awsrdsinstance.ec2client.DescribeInstanceTypes(context.TODO(), ec2input) 79 | 80 | if infoerr != nil { 81 | if aerr, ok := infoerr.(awserr.Error); ok { 82 | awsrdsinstance.logger.Error(aerr.Error()) 83 | 84 | } else { 85 | // Print the error, cast err to awserr.Error to get the Code and 86 | // Message from an error. 87 | awsrdsinstance.logger.Error(infoerr.Error()) 88 | } 89 | } else { 90 | awsrdsinstance.logger.V(5).Infof("EC2.DescribeInstanceType SUCCESS") 91 | awsrdsinstance.logger.V(5).Infof("EC2.DescribeInstanceType %v", typeinfo) 92 | } 93 | 94 | if len(typeinfo.InstanceTypes) > 0 { 95 | info["CPU"] = models.MetricGroupValue{"Counts": typeinfo.InstanceTypes[0].VCpuInfo.DefaultVCpus} 96 | info["PhysicalMemory"] = models.MetricGroupValue{"total": *typeinfo.InstanceTypes[0].MemoryInfo.SizeInMiB * 1024 * 1024} 97 | } 98 | 99 | info["Host"] = models.MetricGroupValue{"InstanceType": "aws/rds"} 100 | 101 | } else if len(result.DBInstances) > 1 { 102 | awsrdsinstance.logger.Infof("RDS.DescribeDBInstances: Database has %d instances. Clusters are not supported", len(result.DBInstances)) 103 | } else { 104 | awsrdsinstance.logger.Info("RDS.DescribeDBInstances: No instances") 105 | } 106 | 107 | } 108 | 109 | metrics.System.Info = info 110 | awsrdsinstance.logger.V(5).Info("CollectMetrics awsrdsinstance", info) 111 | return nil 112 | 113 | } 114 | -------------------------------------------------------------------------------- /metrics/awsrdsenhancedmetrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "time" 9 | 10 | "github.com/Releem/mysqlconfigurer/config" 11 | "github.com/Releem/mysqlconfigurer/models" 12 | "github.com/Releem/mysqlconfigurer/utils" 13 | logging "github.com/google/logger" 14 | 15 | "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" 16 | "github.com/aws/aws-sdk-go-v2/service/rds/types" 17 | "github.com/aws/aws-sdk-go/aws" 18 | ) 19 | 20 | const rdsMetricsLogGroupName = "RDSOSMetrics" 21 | 22 | type AWSRDSEnhancedMetricsGatherer struct { 23 | logger logging.Logger 24 | debug bool 25 | dbinstance types.DBInstance 26 | cwlogsclient *cloudwatchlogs.Client 27 | configuration *config.Config 28 | } 29 | 30 | type osMetrics struct { 31 | Engine string `json:"engine" help:"The database engine for the DB instance."` 32 | InstanceID string `json:"instanceID" help:"The DB instance identifier."` 33 | InstanceResourceID string `json:"instanceResourceID" help:"A region-unique, immutable identifier for the DB instance, also used as the log stream identifier."` 34 | NumVCPUs int `json:"numVCPUs" help:"The number of virtual CPUs for the DB instance."` 35 | Timestamp time.Time `json:"timestamp" help:"The time at which the metrics were taken."` 36 | Uptime string `json:"uptime" help:"The amount of time that the DB instance has been active."` 37 | Version float64 `json:"version" help:"The version of the OS metrics' stream JSON format."` 38 | 39 | CPUUtilization cpuUtilization `json:"cpuUtilization"` 40 | DiskIO []diskIO `json:"diskIO"` 41 | FileSys []fileSys `json:"fileSys"` 42 | LoadAverageMinute loadAverageMinute `json:"loadAverageMinute"` 43 | Memory memory `json:"memory"` 44 | Network []network `json:"network"` 45 | ProcessList []processList `json:"processList"` 46 | Swap swap `json:"swap"` 47 | Tasks RdsTasks `json:"tasks"` 48 | 49 | // TODO Handle this: https://jira.percona.com/browse/PMM-3835 50 | PhysicalDeviceIO []diskIO `json:"physicalDeviceIO"` 51 | } 52 | 53 | type cpuUtilization struct { 54 | Guest float64 `json:"guest" help:"The percentage of CPU in use by guest programs."` 55 | Idle float64 `json:"idle" help:"The percentage of CPU that is idle."` 56 | Irq float64 `json:"irq" help:"The percentage of CPU in use by software interrupts."` 57 | Nice float64 `json:"nice" help:"The percentage of CPU in use by programs running at lowest priority."` 58 | Steal float64 `json:"steal" help:"The percentage of CPU in use by other virtual machines."` 59 | System float64 `json:"system" help:"The percentage of CPU in use by the kernel."` 60 | Total float64 `json:"total" help:"The total percentage of the CPU in use. This value includes the nice value."` 61 | User float64 `json:"user" help:"The percentage of CPU in use by user programs."` 62 | Wait float64 `json:"wait" help:"The percentage of CPU unused while waiting for I/O access."` 63 | } 64 | 65 | //nolint:lll 66 | type diskIO struct { 67 | // common 68 | ReadIOsPS float64 `json:"readIOsPS" help:"The number of read operations per second."` 69 | WriteIOsPS float64 `json:"writeIOsPS" help:"The number of write operations per second."` 70 | Device string `json:"device" help:"The identifier of the disk device in use."` 71 | 72 | // non-Aurora 73 | AvgQueueLen *float64 `json:"avgQueueLen" help:"The number of requests waiting in the I/O device's queue."` 74 | AvgReqSz *float64 `json:"avgReqSz" help:"The average request size, in kilobytes."` 75 | Await *float64 `json:"await" help:"The number of milliseconds required to respond to requests, including queue time and service time."` 76 | ReadKb *int `json:"readKb" help:"The total number of kilobytes read."` 77 | ReadKbPS *float64 `json:"readKbPS" help:"The number of kilobytes read per second."` 78 | RrqmPS *float64 `json:"rrqmPS" help:"The number of merged read requests queued per second."` 79 | TPS *float64 `json:"tps" help:"The number of I/O transactions per second."` 80 | Util *float64 `json:"util" help:"The percentage of CPU time during which requests were issued."` 81 | WriteKb *int `json:"writeKb" help:"The total number of kilobytes written."` 82 | WriteKbPS *float64 `json:"writeKbPS" help:"The number of kilobytes written per second."` 83 | WrqmPS *float64 `json:"wrqmPS" help:"The number of merged write requests queued per second."` 84 | 85 | // Aurora 86 | DiskQueueDepth *float64 `json:"diskQueueDepth" help:"The number of outstanding IOs (read/write requests) waiting to access the disk."` 87 | ReadLatency *float64 `json:"readLatency" help:"The average amount of time taken per disk I/O operation."` 88 | ReadThroughput *float64 `json:"readThroughput" help:"The average number of bytes read from disk per second."` 89 | WriteLatency *float64 `json:"writeLatency" help:"The average amount of time taken per disk I/O operation."` 90 | WriteThroughput *float64 `json:"writeThroughput" help:"The average number of bytes written to disk per second."` 91 | } 92 | 93 | //nolint:lll 94 | type fileSys struct { 95 | MaxFiles int `json:"maxFiles" help:"The maximum number of files that can be created for the file system."` 96 | MountPoint string `json:"mountPoint" help:"The path to the file system."` 97 | Name string `json:"name" help:"The name of the file system."` 98 | Total int `json:"total" help:"The total number of disk space available for the file system, in kilobytes."` 99 | Used int `json:"used" help:"The amount of disk space used by files in the file system, in kilobytes."` 100 | UsedFilePercent float64 `json:"usedFilePercent" help:"The percentage of available files in use."` 101 | UsedFiles int `json:"usedFiles" help:"The number of files in the file system."` 102 | UsedPercent float64 `json:"usedPercent" help:"The percentage of the file-system disk space in use."` 103 | } 104 | 105 | type loadAverageMinute struct { 106 | Fifteen float64 `json:"fifteen" help:"The number of processes requesting CPU time over the last 15 minutes."` 107 | Five float64 `json:"five" help:"The number of processes requesting CPU time over the last 5 minutes."` 108 | One float64 `json:"one" help:"The number of processes requesting CPU time over the last minute."` 109 | } 110 | 111 | //nolint:lll 112 | type memory struct { 113 | Active int `json:"active" node:"Active_bytes" m:"1024" help:"The amount of assigned memory, in kilobytes."` 114 | Buffers int `json:"buffers" node:"Buffers_bytes" m:"1024" help:"The amount of memory used for buffering I/O requests prior to writing to the storage device, in kilobytes."` 115 | Cached int `json:"cached" node:"Cached_bytes" m:"1024" help:"The amount of memory used for caching file system–based I/O."` 116 | Dirty int `json:"dirty" node:"Dirty_bytes" m:"1024" help:"The amount of memory pages in RAM that have been modified but not written to their related data block in storage, in kilobytes."` 117 | Free int `json:"free" node:"MemFree_bytes" m:"1024" help:"The amount of unassigned memory, in kilobytes."` 118 | HugePagesFree int `json:"hugePagesFree" node:"HugePages_Free" m:"1" help:"The number of free huge pages. Huge pages are a feature of the Linux kernel."` 119 | HugePagesRsvd int `json:"hugePagesRsvd" node:"HugePages_Rsvd" m:"1" help:"The number of committed huge pages."` 120 | HugePagesSize int `json:"hugePagesSize" node:"Hugepagesize_bytes" m:"1024" help:"The size for each huge pages unit, in kilobytes."` 121 | HugePagesSurp int `json:"hugePagesSurp" node:"HugePages_Surp" m:"1" help:"The number of available surplus huge pages over the total."` 122 | HugePagesTotal int `json:"hugePagesTotal" node:"HugePages_Total" m:"1" help:"The total number of huge pages for the system."` 123 | Inactive int `json:"inactive" node:"Inactive_bytes" m:"1024" help:"The amount of least-frequently used memory pages, in kilobytes."` 124 | Mapped int `json:"mapped" node:"Mapped_bytes" m:"1024" help:"The total amount of file-system contents that is memory mapped inside a process address space, in kilobytes."` 125 | PageTables int `json:"pageTables" node:"PageTables_bytes" m:"1024" help:"The amount of memory used by page tables, in kilobytes."` 126 | Slab int `json:"slab" node:"Slab_bytes" m:"1024" help:"The amount of reusable kernel data structures, in kilobytes."` 127 | Total int `json:"total" node:"MemTotal_bytes" m:"1024" help:"The total amount of memory, in kilobytes."` 128 | Writeback int `json:"writeback" node:"Writeback_bytes" m:"1024" help:"The amount of dirty pages in RAM that are still being written to the backing storage, in kilobytes."` 129 | } 130 | 131 | type network struct { 132 | Interface string `json:"interface" help:"The identifier for the network interface being used for the DB instance."` 133 | Rx float64 `json:"rx" help:"The number of bytes received per second."` 134 | Tx float64 `json:"tx" help:"The number of bytes uploaded per second."` 135 | } 136 | 137 | //nolint:lll 138 | type processList struct { 139 | CPUUsedPC float64 `json:"cpuUsedPc" help:"The percentage of CPU used by the process."` 140 | ID int `json:"id" help:"The identifier of the process."` 141 | MemoryUsedPC float64 `json:"memoryUsedPc" help:"The amount of memory used by the process, in kilobytes."` 142 | Name string `json:"name" help:"The name of the process."` 143 | ParentID int `json:"parentID" help:"The process identifier for the parent process of the process."` 144 | RSS int `json:"rss" help:"The amount of RAM allocated to the process, in kilobytes."` 145 | TGID int `json:"tgid" help:"The thread group identifier, which is a number representing the process ID to which a thread belongs. This identifier is used to group threads from the same process."` 146 | VSS int `json:"vss" help:"The amount of virtual memory allocated to the process, in kilobytes."` 147 | 148 | // TODO Handle this: https://jira.percona.com/browse/PMM-5150 149 | VMLimit interface{} `json:"vmlimit" help:"-"` 150 | } 151 | 152 | //nolint:lll 153 | type swap struct { 154 | Cached float64 `json:"cached" node:"node_memory_SwapCached_bytes" m:"1024" help:"The amount of swap memory, in kilobytes, used as cache memory." nodehelp:"Memory information field SwapCached."` 155 | Free float64 `json:"free" node:"node_memory_SwapFree_bytes" m:"1024" help:"The total amount of swap memory free, in kilobytes." nodehelp:"Memory information field SwapFree."` 156 | Total float64 `json:"total" node:"node_memory_SwapTotal_bytes" m:"1024" help:"The total amount of swap memory available, in kilobytes." nodehelp:"Memory information field SwapTotal."` 157 | 158 | // we use multiplier 0.25 to convert a number of kilobytes to a number of 4k pages (what our dashboards assume) 159 | In float64 `json:"in" node:"node_vmstat_pswpin" m:"0.25" help:"The total amount of memory, in kilobytes, swapped in from disk." nodehelp:"/proc/vmstat information field pswpin"` 160 | Out float64 `json:"out" node:"node_vmstat_pswpout" m:"0.25" help:"The total amount of memory, in kilobytes, swapped out to disk." nodehelp:"/proc/vmstat information field pswpout"` 161 | } 162 | 163 | type RdsTasks struct { 164 | Blocked int `json:"blocked" help:"The number of tasks that are blocked."` 165 | Running int `json:"running" help:"The number of tasks that are running."` 166 | Sleeping int `json:"sleeping" help:"The number of tasks that are sleeping."` 167 | Stopped int `json:"stopped" help:"The number of tasks that are stopped."` 168 | Total int `json:"total" help:"The total number of tasks."` 169 | Zombie int `json:"zombie" help:"The number of child tasks that are inactive with an active parent task."` 170 | } 171 | 172 | // parseOSMetrics parses OS metrics from given JSON data. 173 | func parseOSMetrics(b []byte, disallowUnknownFields bool) (*osMetrics, error) { 174 | d := json.NewDecoder(bytes.NewReader(b)) 175 | if disallowUnknownFields { 176 | d.DisallowUnknownFields() 177 | } 178 | 179 | var m osMetrics 180 | if err := d.Decode(&m); err != nil { 181 | return nil, err 182 | } 183 | return &m, nil 184 | } 185 | 186 | func NewAWSRDSEnhancedMetricsGatherer(logger logging.Logger, dbinstance types.DBInstance, cwlogsclient *cloudwatchlogs.Client, configuration *config.Config) *AWSRDSEnhancedMetricsGatherer { 187 | return &AWSRDSEnhancedMetricsGatherer{ 188 | logger: logger, 189 | debug: configuration.Debug, 190 | cwlogsclient: cwlogsclient, 191 | dbinstance: dbinstance, 192 | configuration: configuration, 193 | } 194 | } 195 | 196 | func (awsrdsenhancedmetrics *AWSRDSEnhancedMetricsGatherer) GetMetrics(metrics *models.Metrics) error { 197 | defer utils.HandlePanic(awsrdsenhancedmetrics.configuration, awsrdsenhancedmetrics.logger) 198 | 199 | info := make(models.MetricGroupValue) 200 | metricsMap := make(models.MetricGroupValue) 201 | 202 | input := cloudwatchlogs.GetLogEventsInput{ 203 | Limit: aws.Int32(1), 204 | StartFromHead: aws.Bool(false), 205 | LogGroupName: aws.String(rdsMetricsLogGroupName), 206 | LogStreamName: awsrdsenhancedmetrics.dbinstance.DbiResourceId, 207 | } 208 | 209 | result, err := awsrdsenhancedmetrics.cwlogsclient.GetLogEvents(context.TODO(), &input) 210 | 211 | if err != nil { 212 | awsrdsenhancedmetrics.logger.Fatalf("failed to read log stream %s:%s: %s", rdsMetricsLogGroupName, aws.StringValue(awsrdsenhancedmetrics.dbinstance.DbiResourceId), err) 213 | return err 214 | } 215 | 216 | awsrdsenhancedmetrics.logger.V(5).Info("CloudWatchLogs.GetLogEvents SUCCESS") 217 | 218 | if len(result.Events) < 1 { 219 | awsrdsenhancedmetrics.logger.Warning("CloudWatchLogs.GetLogEvents No data") 220 | return errors.New("CloudWatchLogs.GetLogEvents No data") 221 | } 222 | 223 | osMetrics, err := parseOSMetrics([]byte(*result.Events[0].Message), true) 224 | 225 | if err != nil { 226 | awsrdsenhancedmetrics.logger.Errorf("Failed to parse metrics: %s.", err) 227 | return err 228 | } 229 | 230 | // Set IOPS 231 | var readCount, writeCount float64 232 | 233 | for _, diskio := range osMetrics.DiskIO { 234 | readCount = readCount + diskio.ReadIOsPS 235 | writeCount = writeCount + diskio.WriteIOsPS 236 | } 237 | 238 | metricsMap["IOP"] = models.MetricGroupValue{"IOPRead": readCount, "IOPWrite": writeCount} 239 | 240 | // Set FileSystem 241 | metricsMap["FileSystem"] = osMetrics.FileSys 242 | 243 | // OS RAM 244 | metricsMap["PhysicalMemory"] = osMetrics.Memory 245 | info["PhysicalMemory"] = models.MetricGroupValue{"total": osMetrics.Memory.Total} 246 | info["PhysicalMemory"] = utils.MapJoin(info["PhysicalMemory"].(models.MetricGroupValue), models.MetricGroupValue{"swapTotal": osMetrics.Swap.Total}) 247 | 248 | // Swap 249 | metricsMap["Swap"] = osMetrics.Swap 250 | awsrdsenhancedmetrics.logger.V(5).Info("Swap ", osMetrics.Swap) 251 | 252 | //CPU Counts 253 | info["CPU"] = models.MetricGroupValue{"Counts": osMetrics.NumVCPUs} 254 | 255 | // FileSys 256 | metricsMap["FileSystem"] = osMetrics.FileSys 257 | awsrdsenhancedmetrics.logger.V(5).Info("FileSystem ", osMetrics.FileSys) 258 | 259 | //DiskIO 260 | metricsMap["DiskIO"] = osMetrics.DiskIO 261 | awsrdsenhancedmetrics.logger.V(5).Info("DiskIO ", osMetrics.DiskIO) 262 | 263 | // CPU load avarage 264 | metricsMap["CPU"] = osMetrics.LoadAverageMinute //StructToMap(Avg.String()) 265 | awsrdsenhancedmetrics.logger.V(5).Info("CPU ", osMetrics.LoadAverageMinute) 266 | 267 | info["Host"] = models.MetricGroupValue{ 268 | "InstanceType": "aws/rds", 269 | "Timestamp": osMetrics.Timestamp, 270 | "Uptime": osMetrics.Uptime, 271 | "Engine": osMetrics.Engine, 272 | "Version": osMetrics.Version, 273 | } 274 | 275 | metrics.System.Info = info 276 | metrics.System.Metrics = metricsMap 277 | awsrdsenhancedmetrics.logger.V(5).Info("CollectMetrics awsrdsenhancedmetrics ", metrics.System) 278 | 279 | return nil 280 | } 281 | -------------------------------------------------------------------------------- /metrics/awsrdsmetrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/Releem/mysqlconfigurer/config" 9 | "github.com/Releem/mysqlconfigurer/models" 10 | "github.com/Releem/mysqlconfigurer/utils" 11 | logging "github.com/google/logger" 12 | 13 | "github.com/aws/aws-sdk-go-v2/aws" 14 | "github.com/aws/aws-sdk-go-v2/service/cloudwatch" 15 | "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" 16 | "github.com/aws/aws-sdk-go/aws/awserr" 17 | ) 18 | 19 | type AWSRDSMetricsGatherer struct { 20 | logger logging.Logger 21 | debug bool 22 | cwclient *cloudwatch.Client 23 | configuration *config.Config 24 | } 25 | 26 | type rdsMetric struct { 27 | name string 28 | } 29 | 30 | var rdsMetrics = []rdsMetric{ 31 | {name: "BinLogDiskUsage"}, 32 | {name: "BurstBalance"}, 33 | {name: "CPUUtilization"}, 34 | {name: "CPUCreditUsage"}, 35 | {name: "CPUCreditBalance"}, 36 | {name: "CPUSurplusCreditBalance"}, 37 | {name: "CPUSurplusCreditsCharged"}, 38 | {name: "DatabaseConnections"}, 39 | {name: "DiskQueueDepth"}, 40 | {name: "FreeableMemory"}, 41 | {name: "FreeStorageSpace"}, 42 | {name: "LVMReadIOPS"}, 43 | {name: "LVMWriteIOPS"}, 44 | {name: "NetworkReceiveThroughput"}, 45 | {name: "NetworkTransmitThroughput"}, 46 | {name: "ReadIOPS"}, 47 | {name: "ReadLatency"}, 48 | {name: "ReadThroughput"}, 49 | {name: "ReplicaLag"}, 50 | {name: "SwapUsage"}, 51 | {name: "WriteIOPS"}, 52 | {name: "WriteLatency"}, 53 | {name: "WriteThroughput"}, 54 | {name: "NumVCPUs"}, 55 | } 56 | 57 | func NewAWSRDSMetricsGatherer(logger logging.Logger, cwclient *cloudwatch.Client, configuration *config.Config) *AWSRDSMetricsGatherer { 58 | return &AWSRDSMetricsGatherer{ 59 | logger: logger, 60 | debug: configuration.Debug, 61 | cwclient: cwclient, 62 | configuration: configuration, 63 | } 64 | } 65 | 66 | func (awsrdsmetrics *AWSRDSMetricsGatherer) GetMetrics(metrics *models.Metrics) error { 67 | defer utils.HandlePanic(awsrdsmetrics.configuration, awsrdsmetrics.logger) 68 | 69 | MetricDataQueries := []types.MetricDataQuery{} 70 | output := make(models.MetricGroupValue) 71 | 72 | // Prepare request to CloudWatch 73 | for _, metric := range rdsMetrics { 74 | MetricDataQueries = append(MetricDataQueries, 75 | types.MetricDataQuery{ 76 | Id: aws.String("id" + metric.name), 77 | MetricStat: &types.MetricStat{ 78 | Metric: &types.Metric{ 79 | Namespace: aws.String("AWS/RDS"), 80 | MetricName: aws.String(metric.name), 81 | Dimensions: []types.Dimension{ 82 | { 83 | Name: aws.String("DBInstanceIdentifier"), 84 | Value: aws.String(awsrdsmetrics.configuration.AwsRDSDB), 85 | }, 86 | }, 87 | }, 88 | Period: aws.Int32(60), 89 | Stat: aws.String("Average"), 90 | }, 91 | }) 92 | } 93 | 94 | input := &cloudwatch.GetMetricDataInput{ 95 | EndTime: aws.Time(time.Unix(time.Now().Unix(), 0)), 96 | StartTime: aws.Time(time.Unix(time.Now().Add(time.Duration(-2)*time.Minute).Unix(), 0)), 97 | MetricDataQueries: MetricDataQueries, 98 | } 99 | 100 | // Request to CloudWatch 101 | result, err := awsrdsmetrics.cwclient.GetMetricData(context.TODO(), input) 102 | 103 | if err != nil { 104 | if aerr, ok := err.(awserr.Error); ok { 105 | awsrdsmetrics.logger.Error(aerr.Error()) 106 | 107 | } else { 108 | // Print the error, cast err to awserr.Error to get the Code and 109 | // Message from an error. 110 | awsrdsmetrics.logger.Error(err.Error()) 111 | } 112 | } else { 113 | awsrdsmetrics.logger.Info("CloudWatch.GetMetricData SUCCESS") 114 | } 115 | 116 | // Prepare results 117 | for _, r := range result.MetricDataResults { 118 | awsrdsmetrics.logger.V(5).Info("Metric ID ", *r.Id) 119 | awsrdsmetrics.logger.V(5).Info("Metric Label ", *r.Label) 120 | 121 | if len(r.Values) > 0 { 122 | output[*r.Label] = fmt.Sprintf("%f", r.Values[0]) 123 | awsrdsmetrics.logger.V(5).Info("Metric Timestamp ", r.Timestamps[0]) 124 | } else { 125 | awsrdsmetrics.logger.V(5).Info("CloudWatch.GetMetricData no Values for ", *r.Label) 126 | } 127 | } 128 | // temperary 129 | metrics.System.Metrics = output 130 | awsrdsmetrics.logger.V(5).Info("CollectMetrics awsrdsmetrics", output) 131 | return nil 132 | 133 | } 134 | -------------------------------------------------------------------------------- /metrics/dbConf.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/Releem/mysqlconfigurer/config" 5 | "github.com/Releem/mysqlconfigurer/models" 6 | "github.com/Releem/mysqlconfigurer/utils" 7 | logging "github.com/google/logger" 8 | ) 9 | 10 | type DbConfGatherer struct { 11 | logger logging.Logger 12 | configuration *config.Config 13 | } 14 | 15 | func NewDbConfGatherer(logger logging.Logger, configuration *config.Config) *DbConfGatherer { 16 | return &DbConfGatherer{ 17 | logger: logger, 18 | configuration: configuration, 19 | } 20 | } 21 | 22 | func (DbConf *DbConfGatherer) GetMetrics(metrics *models.Metrics) error { 23 | defer utils.HandlePanic(DbConf.configuration, DbConf.logger) 24 | 25 | output := make(models.MetricGroupValue) 26 | 27 | rows, err := models.DB.Query("SHOW VARIABLES") 28 | if err != nil { 29 | DbConf.logger.Error(err) 30 | return nil 31 | } 32 | defer rows.Close() 33 | 34 | for rows.Next() { 35 | var row models.MetricValue 36 | 37 | if err := rows.Scan(&row.Name, &row.Value); err != nil { 38 | DbConf.logger.Error(err) 39 | } 40 | output[row.Name] = row.Value 41 | } 42 | rows.Close() 43 | 44 | rows, err = models.DB.Query("SHOW GLOBAL VARIABLES") 45 | if err != nil { 46 | DbConf.logger.Error(err) 47 | return nil 48 | } 49 | defer rows.Close() 50 | 51 | for rows.Next() { 52 | var row models.MetricValue 53 | if err := rows.Scan(&row.Name, &row.Value); err != nil { 54 | DbConf.logger.Error(err) 55 | } 56 | output[row.Name] = row.Value 57 | } 58 | metrics.DB.Conf.Variables = output 59 | DbConf.logger.V(5).Info("CollectMetrics DbConf ", output) 60 | 61 | return nil 62 | 63 | } 64 | -------------------------------------------------------------------------------- /metrics/dbInfo.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/Releem/mysqlconfigurer/config" 7 | "github.com/Releem/mysqlconfigurer/models" 8 | "github.com/Releem/mysqlconfigurer/utils" 9 | logging "github.com/google/logger" 10 | ) 11 | 12 | type DbInfoGatherer struct { 13 | logger logging.Logger 14 | configuration *config.Config 15 | } 16 | 17 | func NewDbInfoGatherer(logger logging.Logger, configuration *config.Config) *DbInfoGatherer { 18 | return &DbInfoGatherer{ 19 | logger: logger, 20 | configuration: configuration, 21 | } 22 | } 23 | 24 | func (DbInfo *DbInfoGatherer) GetMetrics(metrics *models.Metrics) error { 25 | defer utils.HandlePanic(DbInfo.configuration, DbInfo.logger) 26 | var row models.MetricValue 27 | 28 | var output []string 29 | rows, err := models.DB.Query("SHOW GRANTS") 30 | if err != nil { 31 | DbInfo.logger.Error(err) 32 | return err 33 | } 34 | for rows.Next() { 35 | err := rows.Scan(&row.Value) 36 | if err != nil { 37 | DbInfo.logger.Error(err) 38 | return err 39 | } 40 | output = append(output, row.Value) 41 | } 42 | rows.Close() 43 | metrics.DB.Info["Grants"] = output 44 | 45 | metrics.DB.Info["Users"] = security_recommendations(DbInfo) 46 | 47 | DbInfo.logger.V(5).Info("CollectMetrics DbInfo ", metrics.DB.Info) 48 | return nil 49 | 50 | } 51 | 52 | func security_recommendations(DbInfo *DbInfoGatherer) []models.MetricGroupValue { 53 | var output_users []models.MetricGroupValue 54 | 55 | var password_column_exists, authstring_column_exists int 56 | 57 | // New table schema available since mysql-5.7 and mariadb-10.2 58 | // But need to be checked 59 | models.DB.QueryRow("SELECT 1 FROM information_schema.columns WHERE TABLE_SCHEMA = 'mysql' AND TABLE_NAME = 'user' AND COLUMN_NAME = 'password'").Scan(&password_column_exists) 60 | models.DB.QueryRow("SELECT 1 FROM information_schema.columns WHERE TABLE_SCHEMA = 'mysql' AND TABLE_NAME = 'user' AND COLUMN_NAME = 'authentication_string'").Scan(&authstring_column_exists) 61 | PASS_COLUMN_NAME := "password" 62 | if password_column_exists == 1 && authstring_column_exists == 1 { 63 | PASS_COLUMN_NAME = "IF(plugin='mysql_native_password', authentication_string, password)" 64 | } else if authstring_column_exists == 1 { 65 | PASS_COLUMN_NAME = "authentication_string" 66 | } else if password_column_exists != 1 { 67 | DbInfo.logger.Info("Skipped due to none of known auth columns exists") 68 | return output_users 69 | } 70 | DbInfo.logger.Info("Password column = ", PASS_COLUMN_NAME) 71 | 72 | var Username, User, Host, Password_As_User string 73 | rows_users, err := models.DB.Query("SELECT CONCAT(QUOTE(user), '@', QUOTE(host)), user, host, (CAST(" + PASS_COLUMN_NAME + " as Binary) = PASSWORD(user) OR CAST(" + PASS_COLUMN_NAME + " as Binary) = PASSWORD(UPPER(user)) ) as Password_As_User FROM mysql.user") 74 | if err != nil || !rows_users.Next() { 75 | if err != nil { 76 | if strings.Contains(err.Error(), "Error 1064 (42000): You have an error in your SQL syntax") { 77 | DbInfo.logger.Info("PASSWORD() function is not supported. Try another query...") 78 | } else { 79 | DbInfo.logger.Error(err) 80 | } 81 | } else { 82 | DbInfo.logger.Info("Plugin validate_password is activated. Try another query...") 83 | } 84 | rows_users, err = models.DB.Query("SELECT CONCAT(QUOTE(user), '@', QUOTE(host)), user, host, (CAST(" + PASS_COLUMN_NAME + " as Binary) = CONCAT('*',UPPER(SHA1(UNHEX(SHA1(user))))) OR CAST(" + PASS_COLUMN_NAME + " as Binary) = CONCAT('*',UPPER(SHA1(UNHEX(SHA1(UPPER(user)))))) ) as Password_As_User FROM mysql.user") 85 | if err != nil { 86 | DbInfo.logger.Error(err) 87 | } 88 | defer rows_users.Close() 89 | for rows_users.Next() { 90 | err := rows_users.Scan(&Username, &User, &Host, &Password_As_User) 91 | if err != nil { 92 | DbInfo.logger.Error(err) 93 | } else { 94 | output_users = append(output_users, models.MetricGroupValue{"Username": Username, "User": User, "Host": Host, "Password_As_User": Password_As_User}) 95 | } 96 | } 97 | } else { 98 | defer rows_users.Close() 99 | err := rows_users.Scan(&Username, &User, &Host, &Password_As_User) 100 | if err != nil { 101 | DbInfo.logger.Error(err) 102 | } else { 103 | output_users = append(output_users, models.MetricGroupValue{"Username": Username, "User": User, "Host": Host, "Password_As_User": Password_As_User}) 104 | } 105 | for rows_users.Next() { 106 | err := rows_users.Scan(&Username, &User, &Host, &Password_As_User) 107 | if err != nil { 108 | DbInfo.logger.Error(err) 109 | } else { 110 | output_users = append(output_users, models.MetricGroupValue{"Username": Username, "User": User, "Host": Host, "Password_As_User": Password_As_User}) 111 | } 112 | } 113 | } 114 | 115 | output_user_blank_password := make(models.MetricGroupValue) 116 | rows_users, err = models.DB.Query("SELECT CONCAT(QUOTE(user), '@', QUOTE(host)) FROM mysql.global_priv WHERE ( user != '' AND JSON_CONTAINS(Priv, '\"mysql_native_password\"', '$.plugin') AND JSON_CONTAINS(Priv, '\"\"', '$.authentication_string') AND NOT JSON_CONTAINS(Priv, 'true', '$.account_locked'))") 117 | if err != nil { 118 | if strings.Contains(err.Error(), "Error 1146 (42S02): Table 'mysql.global_priv' doesn't exist") { 119 | DbInfo.logger.Info("Not MariaDB, try another query...") 120 | } else { 121 | DbInfo.logger.Error(err) 122 | } 123 | rows_users, err = models.DB.Query("SELECT CONCAT(QUOTE(user), '@', QUOTE(host)) FROM mysql.user WHERE (" + PASS_COLUMN_NAME + " = '' OR " + PASS_COLUMN_NAME + " IS NULL) AND user != '' /*!50501 AND plugin NOT IN ('auth_socket', 'unix_socket', 'win_socket', 'auth_pam_compat') */ /*!80000 AND account_locked = 'N' AND password_expired = 'N' */") 124 | if err != nil { 125 | DbInfo.logger.Error(err) 126 | } 127 | defer rows_users.Close() 128 | for rows_users.Next() { 129 | err := rows_users.Scan(&Username) 130 | if err != nil { 131 | DbInfo.logger.Error(err) 132 | } else { 133 | output_user_blank_password[Username] = 1 134 | } 135 | } 136 | } else { 137 | defer rows_users.Close() 138 | for rows_users.Next() { 139 | err := rows_users.Scan(&Username) 140 | if err != nil { 141 | DbInfo.logger.Error(err) 142 | } else { 143 | output_user_blank_password[Username] = 1 144 | } 145 | } 146 | } 147 | 148 | for i, user := range output_users { 149 | if _, ok := output_user_blank_password[user["Username"].(string)]; ok { 150 | output_users[i]["Blank_Password"] = 1 151 | } else { 152 | output_users[i]["Blank_Password"] = 0 153 | } 154 | } 155 | 156 | return output_users 157 | } 158 | -------------------------------------------------------------------------------- /metrics/dbInfoBase.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "runtime" 7 | 8 | "github.com/Releem/mysqlconfigurer/config" 9 | "github.com/Releem/mysqlconfigurer/models" 10 | "github.com/Releem/mysqlconfigurer/utils" 11 | logging "github.com/google/logger" 12 | ) 13 | 14 | type DbInfoBaseGatherer struct { 15 | logger logging.Logger 16 | configuration *config.Config 17 | } 18 | 19 | func NewDbInfoBaseGatherer(logger logging.Logger, configuration *config.Config) *DbInfoBaseGatherer { 20 | return &DbInfoBaseGatherer{ 21 | logger: logger, 22 | configuration: configuration, 23 | } 24 | } 25 | 26 | func (DbInfoBase *DbInfoBaseGatherer) GetMetrics(metrics *models.Metrics) error { 27 | defer utils.HandlePanic(DbInfoBase.configuration, DbInfoBase.logger) 28 | 29 | var row models.MetricValue 30 | var mysql_version string 31 | 32 | info := make(models.MetricGroupValue) 33 | // Mysql version 34 | err := models.DB.QueryRow("select VERSION()").Scan(&row.Value) 35 | if err != nil { 36 | DbInfoBase.logger.Error(err) 37 | return nil 38 | } 39 | re := regexp.MustCompile(`(.*?)\-.*`) 40 | version := re.FindStringSubmatch(row.Value) 41 | if len(version) > 0 { 42 | mysql_version = version[1] 43 | } else { 44 | mysql_version = row.Value 45 | } 46 | info["Version"] = mysql_version 47 | err = os.WriteFile(DbInfoBase.configuration.ReleemConfDir+MysqlVersionFile(), []byte(mysql_version), 0644) 48 | if err != nil { 49 | DbInfoBase.logger.Error("WriteFile: Error write to file: ", err) 50 | } 51 | // Mysql force memory limit 52 | info["MemoryLimit"] = DbInfoBase.configuration.GetMemoryLimit() 53 | 54 | metrics.DB.Info = info 55 | DbInfoBase.logger.V(5).Info("CollectMetrics DbInfoBase ", info) 56 | return nil 57 | 58 | } 59 | func MysqlVersionFile() string { 60 | switch runtime.GOOS { 61 | case "windows": 62 | return "\\MysqlVersion.txt" 63 | default: // для Linux и других UNIX-подобных систем 64 | return "/mysql_version" 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /metrics/dbMetrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Releem/mysqlconfigurer/config" 7 | "github.com/Releem/mysqlconfigurer/models" 8 | "github.com/Releem/mysqlconfigurer/utils" 9 | logging "github.com/google/logger" 10 | ) 11 | 12 | type DbMetricsGatherer struct { 13 | logger logging.Logger 14 | configuration *config.Config 15 | } 16 | 17 | func NewDbMetricsGatherer(logger logging.Logger, configuration *config.Config) *DbMetricsGatherer { 18 | return &DbMetricsGatherer{ 19 | logger: logger, 20 | configuration: configuration, 21 | } 22 | } 23 | 24 | func (DbMetrics *DbMetricsGatherer) GetMetrics(metrics *models.Metrics) error { 25 | defer utils.HandlePanic(DbMetrics.configuration, DbMetrics.logger) 26 | //list of databases 27 | { 28 | var database string 29 | var output []string 30 | rows, err := models.DB.Query("SELECT table_schema FROM INFORMATION_SCHEMA.tables group BY table_schema") 31 | if err != nil { 32 | DbMetrics.logger.Error(err) 33 | return err 34 | } 35 | for rows.Next() { 36 | err := rows.Scan(&database) 37 | if err != nil { 38 | DbMetrics.logger.Error(err) 39 | return err 40 | } 41 | output = append(output, database) 42 | } 43 | rows.Close() 44 | metrics.DB.Metrics.Databases = output 45 | } 46 | //Total table 47 | { 48 | var row int 49 | err := models.DB.QueryRow("SELECT COUNT(*) as count FROM information_schema.tables").Scan(&row) 50 | if err != nil { 51 | DbMetrics.logger.Error(err) 52 | return err 53 | } 54 | metrics.DB.Metrics.TotalTables = row 55 | } 56 | //Stat mysql Engine 57 | { 58 | var engine_db, engineenabled string 59 | var size, count, dsize, isize int 60 | output := make(map[string]models.MetricGroupValue) 61 | engine_elem := make(map[string]models.MetricGroupValue) 62 | 63 | rows, err := models.DB.Query("SELECT ENGINE,SUPPORT FROM information_schema.ENGINES ORDER BY ENGINE ASC") 64 | if err != nil { 65 | DbMetrics.logger.Error(err) 66 | return err 67 | } 68 | for rows.Next() { 69 | err := rows.Scan(&engine_db, &engineenabled) 70 | if err != nil { 71 | DbMetrics.logger.Error(err) 72 | return err 73 | } 74 | output[engine_db] = models.MetricGroupValue{"Enabled": engineenabled} 75 | engine_elem[engine_db] = models.MetricGroupValue{"Table Number": 0, "Total Size": 0, "Data Size": 0, "Index Size": 0} 76 | } 77 | rows.Close() 78 | i := 0 79 | for _, database := range metrics.DB.Metrics.Databases { 80 | rows, err = models.DB.Query(`SELECT ENGINE, IFNULL(SUM(DATA_LENGTH+INDEX_LENGTH), 0), IFNULL(COUNT(ENGINE), 0), IFNULL(SUM(DATA_LENGTH), 0), IFNULL(SUM(INDEX_LENGTH), 0) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND ENGINE IS NOT NULL GROUP BY ENGINE ORDER BY ENGINE ASC`, database) 81 | if err != nil { 82 | DbMetrics.logger.Error(err) 83 | return err 84 | } 85 | for rows.Next() { 86 | err := rows.Scan(&engine_db, &size, &count, &dsize, &isize) 87 | if err != nil { 88 | DbMetrics.logger.Error(err) 89 | return err 90 | } 91 | if engine_elem[engine_db]["Table Number"] == nil { 92 | engine_elem[engine_db] = models.MetricGroupValue{"Table Number": 0, "Total Size": 0, "Data Size": 0, "Index Size": 0} 93 | } 94 | engine_elem[engine_db]["Table Number"] = engine_elem[engine_db]["Table Number"].(int) + count 95 | engine_elem[engine_db]["Total Size"] = engine_elem[engine_db]["Total Size"].(int) + size 96 | engine_elem[engine_db]["Data Size"] = engine_elem[engine_db]["Data Size"].(int) + dsize 97 | engine_elem[engine_db]["Index Size"] = engine_elem[engine_db]["Index Size"].(int) + isize 98 | } 99 | rows.Close() 100 | i += 1 101 | if i%25 == 0 { 102 | time.Sleep(3 * time.Second) 103 | } 104 | } 105 | for k := range output { 106 | output[k] = utils.MapJoin(output[k], engine_elem[k]) 107 | } 108 | 109 | metrics.DB.Metrics.Engine = output 110 | if metrics.DB.Metrics.Engine["MyISAM"] == nil { 111 | metrics.DB.Metrics.TotalMyisamIndexes = 0 112 | } else { 113 | metrics.DB.Metrics.TotalMyisamIndexes = metrics.DB.Metrics.Engine["MyISAM"]["Index Size"].(int) 114 | } 115 | } 116 | DbMetrics.logger.V(5).Info("CollectMetrics DbMetrics ", metrics.DB.Metrics) 117 | return nil 118 | 119 | } 120 | -------------------------------------------------------------------------------- /metrics/dbMetricsBase.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "database/sql" 5 | "sort" 6 | "strconv" 7 | 8 | "github.com/Releem/mysqlconfigurer/config" 9 | "github.com/Releem/mysqlconfigurer/models" 10 | "github.com/Releem/mysqlconfigurer/utils" 11 | logging "github.com/google/logger" 12 | ) 13 | 14 | type DbMetricsBaseGatherer struct { 15 | logger logging.Logger 16 | configuration *config.Config 17 | } 18 | 19 | func NewDbMetricsBaseGatherer(logger logging.Logger, configuration *config.Config) *DbMetricsBaseGatherer { 20 | return &DbMetricsBaseGatherer{ 21 | logger: logger, 22 | configuration: configuration, 23 | } 24 | } 25 | 26 | func (DbMetricsBase *DbMetricsBaseGatherer) GetMetrics(metrics *models.Metrics) error { 27 | defer utils.HandlePanic(DbMetricsBase.configuration, DbMetricsBase.logger) 28 | // Mysql Status 29 | output := make(models.MetricGroupValue) 30 | { 31 | var row models.MetricValue 32 | rows, err := models.DB.Query("SHOW STATUS") 33 | 34 | if err != nil { 35 | DbMetricsBase.logger.Error(err) 36 | return err 37 | } 38 | for rows.Next() { 39 | err := rows.Scan(&row.Name, &row.Value) 40 | if err != nil { 41 | DbMetricsBase.logger.Error(err) 42 | return err 43 | } 44 | output[row.Name] = row.Value 45 | } 46 | rows.Close() 47 | 48 | rows, err = models.DB.Query("SHOW GLOBAL STATUS") 49 | if err != nil { 50 | DbMetricsBase.logger.Error(err) 51 | return err 52 | } 53 | for rows.Next() { 54 | err := rows.Scan(&row.Name, &row.Value) 55 | if err != nil { 56 | DbMetricsBase.logger.Error(err) 57 | return err 58 | } 59 | output[row.Name] = row.Value 60 | } 61 | metrics.DB.Metrics.Status = output 62 | rows.Close() 63 | } 64 | // Latency 65 | { 66 | var output []models.MetricGroupValue 67 | var schema_name, query_id string 68 | var calls, avg_time_us, sum_time_us int 69 | 70 | rows, err := models.DB.Query("SELECT IFNULL(schema_name, 'NULL') as schema_name, IFNULL(digest, 'NULL') as query_id, count_star as calls, round(avg_timer_wait/1000000, 0) as avg_time_us, round(SUM_TIMER_WAIT/1000000, 0) as sum_time_us FROM performance_schema.events_statements_summary_by_digest") 71 | if err != nil { 72 | if err != sql.ErrNoRows { 73 | DbMetricsBase.logger.Error(err) 74 | } 75 | } else { 76 | for rows.Next() { 77 | err := rows.Scan(&schema_name, &query_id, &calls, &avg_time_us, &sum_time_us) 78 | if err != nil { 79 | DbMetricsBase.logger.Error(err) 80 | return err 81 | } 82 | output = append(output, models.MetricGroupValue{"schema_name": schema_name, "query_id": query_id, "calls": calls, "avg_time_us": avg_time_us, "sum_time_us": sum_time_us}) 83 | } 84 | } 85 | metrics.DB.Metrics.QueriesLatency = output 86 | 87 | if len(output) != 0 { 88 | totalQueryCount := len(output) 89 | dictQueryCount := make(map[int]int) 90 | listAvgTimeDistinct := make([]int, 0) 91 | listAvgTime := make([]int, 0) 92 | 93 | for _, query := range output { 94 | avgTime := query["avg_time_us"].(int) 95 | if !contains(listAvgTimeDistinct, avgTime) { 96 | listAvgTimeDistinct = append(listAvgTimeDistinct, avgTime) 97 | } 98 | listAvgTime = append(listAvgTime, avgTime) 99 | } 100 | sort.Sort(sort.Reverse(sort.IntSlice(listAvgTimeDistinct))) 101 | 102 | for _, avgTime1 := range listAvgTime { 103 | for _, avgTime2 := range listAvgTimeDistinct { 104 | if avgTime2 >= avgTime1 { 105 | if _, ok := dictQueryCount[avgTime2]; !ok { 106 | dictQueryCount[avgTime2] = 1 107 | } else { 108 | dictQueryCount[avgTime2]++ 109 | } 110 | } else { 111 | break 112 | } 113 | } 114 | } 115 | 116 | latency := 0 117 | for _, avgTime := range listAvgTimeDistinct { 118 | if float64(dictQueryCount[avgTime])/float64(totalQueryCount) <= 0.95 { 119 | break 120 | } 121 | latency = avgTime 122 | } 123 | 124 | metrics.DB.Metrics.Latency = strconv.Itoa(latency) 125 | } else { 126 | metrics.DB.Metrics.Latency = "" 127 | } 128 | } 129 | //status innodb engine 130 | { 131 | var engine, name, status string 132 | err := models.DB.QueryRow("show engine innodb status").Scan(&engine, &name, &status) 133 | if err != nil { 134 | DbMetricsBase.logger.Error(err) 135 | } else { 136 | metrics.DB.Metrics.InnoDBEngineStatus = status 137 | } 138 | } 139 | // ProcessList 140 | { 141 | 142 | type information_schema_processlist_type struct { 143 | ID string 144 | USER string 145 | HOST string 146 | DB string 147 | COMMAND string 148 | TIME string 149 | STATE string 150 | INFO string 151 | } 152 | var information_schema_processlist information_schema_processlist_type 153 | 154 | rows, err := models.DB.Query("SELECT IFNULL(ID, 'NULL') as ID, IFNULL(USER, 'NULL') as USER, IFNULL(HOST, 'NULL') as HOST, IFNULL(DB, 'NULL') as DB, IFNULL(COMMAND, 'NULL') as COMMAND, IFNULL(TIME, 'NULL') as TIME, IFNULL(STATE, 'NULL') as STATE, IFNULL(INFO, 'NULL') as INFO FROM information_schema.PROCESSLIST ORDER BY ID") 155 | if err != nil { 156 | DbMetricsBase.logger.Error(err) 157 | } else { 158 | for rows.Next() { 159 | err := rows.Scan(&information_schema_processlist.ID, &information_schema_processlist.USER, &information_schema_processlist.HOST, &information_schema_processlist.DB, &information_schema_processlist.COMMAND, &information_schema_processlist.TIME, &information_schema_processlist.STATE, &information_schema_processlist.INFO) 160 | if err != nil { 161 | DbMetricsBase.logger.Error(err) 162 | return err 163 | } 164 | metrics.DB.Metrics.ProcessList = append(metrics.DB.Metrics.ProcessList, models.MetricGroupValue{"ID": information_schema_processlist.ID, "USER": information_schema_processlist.USER, "HOST": information_schema_processlist.HOST, "DB": information_schema_processlist.DB, "COMMAND": information_schema_processlist.COMMAND, "TIME": information_schema_processlist.TIME, "STATE": information_schema_processlist.STATE, "INFO": information_schema_processlist.INFO}) 165 | } 166 | rows.Close() 167 | } 168 | } 169 | 170 | DbMetricsBase.logger.V(5).Info("CollectMetrics DbMetricsBase ", metrics.DB.Metrics) 171 | return nil 172 | } 173 | 174 | func contains(arr []int, num int) bool { 175 | for _, n := range arr { 176 | if n == num { 177 | return true 178 | } 179 | } 180 | return false 181 | } 182 | -------------------------------------------------------------------------------- /metrics/os.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/Releem/mysqlconfigurer/config" 8 | "github.com/Releem/mysqlconfigurer/models" 9 | "github.com/Releem/mysqlconfigurer/utils" 10 | logging "github.com/google/logger" 11 | "github.com/shirou/gopsutil/v4/cpu" 12 | "github.com/shirou/gopsutil/v4/disk" 13 | "github.com/shirou/gopsutil/v4/host" 14 | "github.com/shirou/gopsutil/v4/load" 15 | "github.com/shirou/gopsutil/v4/mem" 16 | ) 17 | 18 | type OSMetricsGatherer struct { 19 | logger logging.Logger 20 | debug bool 21 | configuration *config.Config 22 | } 23 | 24 | func NewOSMetricsGatherer(logger logging.Logger, configuration *config.Config) *OSMetricsGatherer { 25 | return &OSMetricsGatherer{ 26 | logger: logger, 27 | debug: configuration.Debug, 28 | configuration: configuration, 29 | } 30 | } 31 | 32 | // func cpu_cores(os_type string) string { 33 | // if os_type == "Linux" { 34 | // cntCPU, _ := exec.Command("awk -F: '/^core id/ && !P[$2] { CORES++; P[$2]=1 }; /^physical id/ && !N[$2] { CPUs++; N[$2]=1 }; END { print CPUs*CORES }' /proc/cpuinfo").Output() 35 | // cntCPU_1 := strings.Trim(string(cntCPU), "\n") 36 | // if cntCPU_1 == "0" { 37 | // out, _ := exec.Command("nproc").Output() 38 | // // string to int 39 | // i, err := strconv.Atoi(string(out)) 40 | // if err != nil { 41 | // return strconv.Itoa(0) 42 | // } 43 | // return strconv.Itoa(i) 44 | // } else { 45 | // i, err := strconv.Atoi(string(cntCPU_1)) 46 | // if err != nil { 47 | // return strconv.Itoa(0) 48 | // } 49 | // return strconv.Itoa(i) 50 | // } 51 | // } 52 | // if os_type == "FreeBSD" { 53 | // cntCPU, _ := exec.Command("sysctl -n kern.smp.cores").Output() 54 | // cntCPU_1 := strings.Trim(string(cntCPU), "\n") 55 | // i, err := strconv.Atoi(string(cntCPU_1)) 56 | // if err != nil { 57 | // return strconv.Itoa(0) 58 | // } 59 | // return strconv.Itoa(i + 1) 60 | // } 61 | 62 | // return strconv.Itoa(0) 63 | // } 64 | 65 | // func is_virtual_machine(os_type string) int { 66 | // if os_type == "Linux" { 67 | // isVm, _ := exec.Command("grep -Ec '^flags.* hypervisor ' /proc/cpuinfo").Output() 68 | // if string(isVm) == "0" { 69 | // return 0 70 | // } else { 71 | // return 1 72 | // } 73 | // } 74 | // if os_type == "FreeBSD" { 75 | // isVm, _ := exec.Command("sysctl -n kern.vm_guest").Output() 76 | // isVm_1 := strings.Trim(string(isVm), "\n") 77 | // if isVm_1 == "none" { 78 | // return 0 79 | // } else { 80 | // return 1 81 | // } 82 | // } 83 | 84 | // return 0 85 | // } 86 | 87 | func StructToMap(valueStruct string) models.MetricGroupValue { 88 | var value_map models.MetricGroupValue 89 | 90 | _ = json.Unmarshal([]byte(valueStruct), &value_map) 91 | return value_map 92 | } 93 | func (OS *OSMetricsGatherer) GetMetrics(metrics *models.Metrics) error { 94 | defer utils.HandlePanic(OS.configuration, OS.logger) 95 | info := make(models.MetricGroupValue) 96 | metricsMap := make(models.MetricGroupValue) 97 | 98 | // if out, err := exec.Command("uname").Output(); err != nil { 99 | // return err 100 | // } else { 101 | // info["OS Type"] = strings.Trim(string(out), "\n") 102 | // } 103 | //output["Physical Memory"] = make(map[string]string) 104 | // if forcemem := OS.configuration.GetMemoryLimit(); forcemem == 0 { 105 | // virtualMemory, _ := mem.VirtualMemory() 106 | // output["Physical Memory"] = map[string]uint64{"bytes": uint64(virtualMemory.Total)} 107 | // } else { 108 | // output["Physical Memory"] = map[string]uint64{"bytes": uint64(forcemem * 1048576)} 109 | // } 110 | 111 | // OS RAM 112 | VirtualMemory, _ := mem.VirtualMemory() 113 | metricsMap["PhysicalMemory"] = StructToMap(VirtualMemory.String()) 114 | info["PhysicalMemory"] = models.MetricGroupValue{"total": VirtualMemory.Total} 115 | 116 | // OS SwapMemory 117 | SwapMemory, _ := mem.SwapMemory() 118 | metricsMap["SwapMemory"] = StructToMap(SwapMemory.String()) 119 | info["PhysicalMemory"] = utils.MapJoin(info["PhysicalMemory"].(models.MetricGroupValue), models.MetricGroupValue{"swapTotal": SwapMemory.Total}) 120 | 121 | //CPU Counts 122 | CpuCounts, _ := cpu.Counts(true) 123 | info["CPU"] = models.MetricGroupValue{"Counts": CpuCounts} 124 | 125 | //OS host info 126 | hostInfo, _ := host.Info() 127 | hostInfoMap := utils.MapJoin(StructToMap(hostInfo.String()), models.MetricGroupValue{"InstanceType": "local"}) 128 | info["Host"] = hostInfoMap 129 | 130 | //Get partitions, for each pert get usage and io stat 131 | var UsageArray, PartitionsArray, IOCountersArray []models.MetricGroupValue 132 | var readCount, writeCount uint64 133 | //:= make(models.MetricGroupValue) 134 | PartitionCheck := make(map[string]int) 135 | Partitions, err := disk.Partitions(false) 136 | if err != nil { 137 | OS.logger.Error(err) 138 | } 139 | for _, part := range Partitions { 140 | Usage, err := disk.Usage(part.Mountpoint) 141 | if err != nil { 142 | OS.logger.Error(err) 143 | } else { 144 | UsageArray = append(UsageArray, StructToMap(Usage.String())) 145 | } 146 | PartitionsArray = append(PartitionsArray, StructToMap(part.String())) 147 | PartName := part.Device[strings.LastIndex(part.Device, "/")+1:] 148 | IOCounters, err := disk.IOCounters(PartName) 149 | if err != nil { 150 | OS.logger.Error(err) 151 | } else { 152 | if _, exists := PartitionCheck[part.Device]; !exists { 153 | 154 | readCount = readCount + IOCounters[PartName].ReadCount 155 | writeCount = writeCount + IOCounters[PartName].WriteCount 156 | PartitionCheck[part.Device] = 1 157 | } 158 | OS.logger.V(5).Info("IOCounters ", IOCounters) 159 | IOCountersArray = append(IOCountersArray, models.MetricGroupValue{PartName: StructToMap(IOCounters[PartName].String())}) 160 | } 161 | } 162 | OS.logger.V(5).Info("PartitionCheck ", PartitionCheck) 163 | info["Partitions"] = PartitionsArray 164 | OS.logger.V(5).Info("Partitions ", PartitionsArray) 165 | 166 | // info["Usage"] = UsageArray 167 | metricsMap["FileSystem"] = UsageArray 168 | OS.logger.V(5).Info("Usage ", UsageArray) 169 | 170 | metricsMap["DiskIO"] = IOCountersArray 171 | OS.logger.V(5).Info("IOCountersArray ", IOCountersArray) 172 | 173 | // CPU load avarage 174 | Avg, _ := load.Avg() 175 | metricsMap["CPU"] = StructToMap(Avg.String()) 176 | OS.logger.V(5).Info("Avg ", Avg) 177 | 178 | // CpuUtilisation := float64(metrics.System.Metrics.CPU["load1"].(float64) / float64(info["CPU"].(models.MetricGroupValue)["Counts"].(int))) 179 | // metrics.System.Metrics.CPU["CpuUtilisation"] = CpuUtilisation 180 | // info["Cpu"] = models.MetricGroupValue{"CpuUtilisation": (info["Avg"].(models.MetricGroupValue)["load1"].(float64) / float64(info["Cpu"].(models.MetricGroupValue)["Counts"].(int)))} 181 | 182 | //Calc iops read and write as io count / uptime 183 | metricsMap["IOP"] = models.MetricGroupValue{"IOPRead": float64(readCount), "IOPWrite": float64(writeCount)} 184 | 185 | metrics.System.Info = info 186 | metrics.System.Metrics = metricsMap 187 | OS.logger.V(5).Info("CollectMetrics OS ", metrics.System) 188 | return nil 189 | 190 | } 191 | -------------------------------------------------------------------------------- /metrics/runner.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "sync" 7 | "syscall" 8 | "time" 9 | 10 | "github.com/Releem/mysqlconfigurer/config" 11 | "github.com/Releem/mysqlconfigurer/models" 12 | "github.com/Releem/mysqlconfigurer/tasks" 13 | "github.com/Releem/mysqlconfigurer/utils" 14 | logging "github.com/google/logger" 15 | ) 16 | 17 | var Ready bool 18 | 19 | // Set up channel on which to send signal notifications. 20 | // We must use a buffered channel or risk missing the signal 21 | // if we're not ready to receive when the signal is sent. 22 | func makeTerminateChannel() <-chan os.Signal { 23 | ch := make(chan os.Signal, 1) 24 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) 25 | return ch 26 | } 27 | 28 | func RunWorker(gatherers []models.MetricsGatherer, gatherers_configuration []models.MetricsGatherer, gatherers_query_optimization []models.MetricsGatherer, repeaters models.MetricsRepeater, logger logging.Logger, 29 | configuration *config.Config, Mode models.ModeType) { 30 | var GenerateTimer, timer, QueryOptimizationTimer *time.Timer 31 | defer utils.HandlePanic(configuration, logger) 32 | 33 | if (Mode.Name == "Configurations" && Mode.Type != "default") || Mode.Name == "Event" || Mode.Name == "TaskSet" { 34 | GenerateTimer = time.NewTimer(0 * time.Second) 35 | timer = time.NewTimer(3600 * time.Second) 36 | } else { 37 | GenerateTimer = time.NewTimer(configuration.GenerateConfigPeriod * time.Second) 38 | timer = time.NewTimer(1 * time.Second) 39 | } 40 | QueryOptimizationTimer = time.NewTimer(1 * time.Minute) 41 | QueryOptimizationCollectSqlText := time.NewTimer(1 * time.Second) 42 | models.SqlText = make(map[string]map[string]string) 43 | models.SqlTextMutex = sync.RWMutex{} 44 | 45 | if !configuration.QueryOptimization { 46 | QueryOptimizationCollectSqlText.Stop() 47 | } 48 | terminator := makeTerminateChannel() 49 | 50 | loop: 51 | for { 52 | select { 53 | case <-terminator: 54 | logger.Info("Exiting") 55 | break loop 56 | case <-timer.C: 57 | logger.Info(" * Starting collection of data for saving a metrics...") 58 | timer.Reset(configuration.MetricsPeriod * time.Second) 59 | go func() { 60 | defer utils.HandlePanic(configuration, logger) 61 | metrics := utils.CollectMetrics(gatherers, logger, configuration) 62 | if metrics != nil { 63 | metrics.DB.Metrics.CountEnableEventsStatementsConsumers = utils.EnableEventsStatementsConsumers(configuration, logger, metrics.DB.Metrics.Status["Uptime"].(string)) 64 | task := utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "Metrics", Type: ""}) 65 | if task == "Task" { 66 | logger.Info(" * A task has been found for the agent...") 67 | f := tasks.ProcessTaskFunc(metrics, repeaters, gatherers, logger, configuration) 68 | time.AfterFunc(5*time.Second, f) 69 | } 70 | } 71 | logger.Info("Saved a metrics...") 72 | }() 73 | case <-GenerateTimer.C: 74 | logger.Info(" * Starting collection of data for generating a config...") 75 | GenerateTimer.Reset(configuration.GenerateConfigPeriod * time.Second) 76 | go func() { 77 | var metrics *models.Metrics 78 | logger.Info(" * Collecting metrics to recommend a config...") 79 | defer utils.HandlePanic(configuration, logger) 80 | if Mode.Name == "TaskSet" && Mode.Type == "queries_optimization" { 81 | metrics = utils.CollectMetrics(append(gatherers, gatherers_query_optimization...), logger, configuration) 82 | } else { 83 | metrics = utils.CollectMetrics(append(gatherers, gatherers_configuration...), logger, configuration) 84 | } 85 | if metrics != nil { 86 | metrics.DB.Metrics.CountEnableEventsStatementsConsumers = utils.EnableEventsStatementsConsumers(configuration, logger, "0") 87 | logger.Info(" * Sending metrics to Releem Cloud Platform...") 88 | utils.ProcessRepeaters(metrics, repeaters, configuration, logger, Mode) 89 | if Mode.Name == "Configurations" { 90 | logger.Info("Recommended MySQL configuration downloaded to ", configuration.GetReleemConfDir()) 91 | } 92 | } 93 | if (Mode.Name == "Configurations" && Mode.Type != "default") || Mode.Name == "Event" || Mode.Name == "TaskSet" { 94 | logger.Info("Exiting") 95 | os.Exit(0) 96 | } 97 | logger.Info("Saved a config...") 98 | }() 99 | case <-QueryOptimizationTimer.C: 100 | logger.Info("Starting collection of data for queries optimization...") 101 | QueryOptimizationTimer.Reset(configuration.QueryOptimizationPeriod * time.Second) 102 | go func() { 103 | defer utils.HandlePanic(configuration, logger) 104 | logger.Info("QueryOptimization") 105 | metrics := utils.CollectMetrics(append(gatherers, gatherers_query_optimization...), logger, configuration) 106 | if metrics != nil { 107 | utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "Metrics", Type: "QueryOptimization"}) 108 | } 109 | logger.Info("Saved a queries...") 110 | }() 111 | case <-QueryOptimizationCollectSqlText.C: 112 | QueryOptimizationCollectSqlText.Reset(configuration.QueryOptimizationCollectSqlTextPeriod * time.Second) 113 | go func() { 114 | defer utils.HandlePanic(configuration, logger) 115 | Ready = false 116 | var SqlText_elem models.SqlTextType 117 | rows, err := models.DB.Query("SELECT CURRENT_SCHEMA, DIGEST, SQL_TEXT FROM performance_schema.events_statements_history WHERE DIGEST IS NOT NULL AND CURRENT_SCHEMA IS NOT NULL GROUP BY CURRENT_SCHEMA, DIGEST, SQL_TEXT") 118 | if err != nil { 119 | logger.Error(err) 120 | } else { 121 | for rows.Next() { 122 | err := rows.Scan(&SqlText_elem.CURRENT_SCHEMA, &SqlText_elem.DIGEST, &SqlText_elem.SQL_TEXT) 123 | if err != nil { 124 | logger.Error(err) 125 | } else { 126 | models.SqlTextMutex.Lock() 127 | if models.SqlText[SqlText_elem.CURRENT_SCHEMA] == nil { 128 | models.SqlText[SqlText_elem.CURRENT_SCHEMA] = make(map[string]string) 129 | } 130 | models.SqlText[SqlText_elem.CURRENT_SCHEMA][SqlText_elem.DIGEST] = SqlText_elem.SQL_TEXT 131 | models.SqlTextMutex.Unlock() 132 | } 133 | } 134 | } 135 | 136 | }() 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "database/sql" 5 | "sync" 6 | 7 | "github.com/Releem/mysqlconfigurer/config" 8 | ) 9 | 10 | type MetricType byte 11 | type MetricIntervalType byte 12 | 13 | type MetricValue struct { 14 | Name string 15 | Value string 16 | } 17 | type MetricGroupValue map[string]interface{} 18 | 19 | type ModeType struct { 20 | Name string 21 | Type string 22 | } 23 | type Metrics struct { 24 | System struct { 25 | Info MetricGroupValue 26 | Conf MetricGroupValue 27 | Metrics MetricGroupValue 28 | } 29 | DB struct { 30 | Metrics struct { 31 | Status MetricGroupValue 32 | TotalTables int 33 | TotalMyisamIndexes int 34 | Engine map[string]MetricGroupValue 35 | Latency string 36 | QueriesLatency []MetricGroupValue 37 | Databases []string 38 | InnoDBEngineStatus string 39 | CountEnableEventsStatementsConsumers int 40 | ProcessList []MetricGroupValue 41 | } 42 | Conf struct { 43 | Variables MetricGroupValue 44 | } 45 | Info MetricGroupValue 46 | Queries []MetricGroupValue 47 | QueriesOptimization map[string][]MetricGroupValue 48 | } 49 | ReleemAgent struct { 50 | Info MetricGroupValue 51 | Tasks MetricGroupValue 52 | Conf config.Config 53 | } 54 | } 55 | 56 | type Metric map[string]MetricGroupValue 57 | 58 | // type Metric interface { 59 | // // GetProvider() string 60 | // // GetType() MetricType 61 | // // GetValue() string 62 | // // GetName() string 63 | // } 64 | 65 | type Task struct { 66 | TaskID *int `json:"task_id"` 67 | TaskTypeID *int `json:"task_type_id"` 68 | IsExist *string `json:"is_exist"` 69 | } 70 | 71 | type MetricContext interface { 72 | GetApiKey() string 73 | GetEnv() string 74 | GetMemoryLimit() int 75 | GetReleemConfDir() string 76 | } 77 | 78 | type MetricsGatherer interface { 79 | GetMetrics(metrics *Metrics) error 80 | } 81 | 82 | type MetricsRepeater interface { 83 | ProcessMetrics(context MetricContext, metrics Metrics, Mode ModeType) (interface{}, error) 84 | } 85 | 86 | type SqlTextType struct { 87 | CURRENT_SCHEMA string 88 | DIGEST string 89 | SQL_TEXT string 90 | } 91 | 92 | var ( 93 | DB *sql.DB 94 | SqlText map[string]map[string]string 95 | SqlTextMutex sync.RWMutex 96 | ) 97 | -------------------------------------------------------------------------------- /releem-agent/install_on_primise.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # install.sh - Version 1.19.3 3 | # (C) Releem, Inc 2022 4 | # All rights reserved 5 | 6 | # Releem installation script: install and set up the Releem Agent on supported Linux distributions 7 | # using the package manager. 8 | 9 | set -e 10 | install_script_version=1.19.3 11 | logfile="releem-install.log" 12 | 13 | WORKDIR="/opt/releem" 14 | CONF="$WORKDIR/releem.conf" 15 | MYSQL_CONF_DIR="/etc/mysql/releem.conf.d" 16 | RELEEM_COMMAND="/bin/bash $WORKDIR/mysqlconfigurer.sh" 17 | 18 | # Read configuration 19 | 20 | 21 | # Set up a named pipe for logging 22 | npipe=/tmp/$$.install.tmp 23 | mknod $npipe p 24 | 25 | # Log all output to a log for error checking 26 | tee <$npipe $logfile & 27 | exec 1>&- 28 | exec 1>$npipe 2>&1 29 | 30 | function on_exit() { 31 | rm -f $npipe 32 | } 33 | 34 | function on_error() { 35 | printf "\033[31m$ERROR_MESSAGE 36 | It looks like you hit an issue when trying to install the Releem. 37 | 38 | If you're still having problems, please send an email to hello@releem.com 39 | with the contents of $logfile and we'll do our very best to help you 40 | solve your problem.\n\033[0m\n" 41 | } 42 | trap on_error ERR 43 | trap on_exit EXIT 44 | 45 | function releem_set_cron() { 46 | ($sudo_cmd crontab -l 2>/dev/null | grep -v "$WORKDIR/mysqlconfigurer.sh" || true; echo "$RELEEM_CRON") | $sudo_cmd crontab - 47 | } 48 | 49 | function releem_update() { 50 | printf "\033[37m\n * Downloading latest version of Releem Agent...\033[0m\n" 51 | $sudo_cmd curl -w "%{http_code}" -L -o $WORKDIR/releem-agent.new https://releem.s3.amazonaws.com/v2/releem-agent-$(arch) 52 | $sudo_cmd curl -w "%{http_code}" -L -o $WORKDIR/mysqlconfigurer.sh.new https://releem.s3.amazonaws.com/v2/mysqlconfigurer.sh 53 | $sudo_cmd $WORKDIR/releem-agent stop || true 54 | $sudo_cmd mv $WORKDIR/releem-agent.new $WORKDIR/releem-agent 55 | $sudo_cmd mv $WORKDIR/mysqlconfigurer.sh.new $WORKDIR/mysqlconfigurer.sh 56 | $sudo_cmd chmod 755 $WORKDIR/mysqlconfigurer.sh $WORKDIR/releem-agent 57 | $sudo_cmd $WORKDIR/releem-agent start || true 58 | $sudo_cmd $WORKDIR/releem-agent -f 59 | 60 | echo 61 | echo 62 | echo -e "Releem Agent updated successfully." 63 | echo 64 | echo -e "To check MySQL Performance Score please visit https://app.releem.com/dashboard?menu=metrics" 65 | echo 66 | 67 | exit 0 68 | } 69 | 70 | if [ "$0" == "uninstall" ]; 71 | then 72 | trap - EXIT 73 | $WORKDIR/releem-agent --event=agent_uninstall > /dev/null 74 | printf "\033[37m\n * Configure crontab\033[0m\n" 75 | ($sudo_cmd crontab -l 2>/dev/null | grep -v "$WORKDIR/mysqlconfigurer.sh" || true) | $sudo_cmd crontab - 76 | printf "\033[37m\n * Stoping Releem Agent service...\033[0m\n" 77 | releem_agent_stop=$($sudo_cmd $WORKDIR/releem-agent stop) 78 | if [ $? -eq 0 ]; then 79 | printf "\033[32m\n Stop Releem Agent successfuly\033[0m\n" 80 | else 81 | echo $releem_agent_stop 82 | printf "\033[31m\n Restart Releem Agent failed\033[0m\n" 83 | fi 84 | printf "\033[37m\n * Uninstalling Releem Agent service...\033[0m\n" 85 | releem_agent_remove=$($sudo_cmd $WORKDIR/releem-agent remove) 86 | if [ $? -eq 0 ]; then 87 | printf "\033[32m\n Uninstall Releem Agent successfuly\033[0m\n" 88 | else 89 | echo $releem_agent_remove 90 | printf "\033[31m\n Reinstall Releem Agent failed\033[0m\n" 91 | fi 92 | printf "\033[37m\n * Remove files Releem Agent\033[0m\n" 93 | $sudo_cmd rm -rf $WORKDIR 94 | exit 0 95 | fi 96 | 97 | apikey= 98 | if [ -n "$RELEEM_API_KEY" ]; then 99 | apikey=$RELEEM_API_KEY 100 | fi 101 | 102 | if [ ! "$apikey" ]; then 103 | printf "\033[31mReleem API key is not available in RELEEM_API_KEY environment variable. Please sigh up at https://releem.com\033[0m\n" 104 | exit 1; 105 | fi 106 | 107 | connection_string="" 108 | root_connection_string="" 109 | if [ -n "$RELEEM_MYSQL_HOST" ]; then 110 | if [ -S "$RELEEM_MYSQL_HOST" ]; then 111 | mysql_user_host="localhost" 112 | connection_string="${connection_string} --socket=${RELEEM_MYSQL_HOST}" 113 | root_connection_string="${root_connection_string} --socket=${RELEEM_MYSQL_HOST}" 114 | else 115 | if [ "$RELEEM_MYSQL_HOST" == "127.0.0.1" ]; then 116 | mysql_user_host="127.0.0.1" 117 | else 118 | mysql_user_host="%" 119 | fi 120 | connection_string="${connection_string} --host=${RELEEM_MYSQL_HOST}" 121 | fi 122 | else 123 | mysql_user_host="127.0.0.1" 124 | connection_string="${connection_string} --host=127.0.0.1" 125 | fi 126 | 127 | if [ -n "$RELEEM_MYSQL_PORT" ]; then 128 | connection_string="${connection_string} --port=${RELEEM_MYSQL_PORT}" 129 | else 130 | connection_string="${connection_string} --port=3306" 131 | fi 132 | 133 | 134 | 135 | # Root user detection 136 | if [ "$(echo "$UID")" = "0" ]; then 137 | sudo_cmd='' 138 | else 139 | sudo_cmd='sudo' 140 | fi 141 | 142 | # Parse parameters 143 | while getopts "u" option 144 | do 145 | case "${option}" 146 | in 147 | u) releem_update;; 148 | esac 149 | done 150 | 151 | # OS/Distro Detection 152 | # Try lsb_release, fallback with /etc/issue then uname command 153 | KNOWN_DISTRIBUTION="(Debian|Ubuntu|RedHat|CentOS|Amazon)" 154 | DISTRIBUTION=$(lsb_release -d 2>/dev/null | grep -Eo $KNOWN_DISTRIBUTION || grep -Eo $KNOWN_DISTRIBUTION /etc/issue 2>/dev/null || grep -Eo $KNOWN_DISTRIBUTION /etc/Eos-release 2>/dev/null || grep -m1 -Eo $KNOWN_DISTRIBUTION /etc/os-release 2>/dev/null || uname -s) 155 | 156 | if [ -f /etc/debian_version ] || [ "$DISTRIBUTION" == "Debian" ] || [ "$DISTRIBUTION" == "Ubuntu" ]; then 157 | OS="Debian" 158 | elif [ -f /etc/redhat-release ] || [ "$DISTRIBUTION" == "RedHat" ] || [ "$DISTRIBUTION" == "CentOS" ] || [ "$DISTRIBUTION" == "Amazon" ]; then 159 | OS="RedHat" 160 | # Some newer distros like Amazon may not have a redhat-release file 161 | elif [ -f /etc/system-release ] || [ "$DISTRIBUTION" == "Amazon" ]; then 162 | OS="RedHat" 163 | # Arista is based off of Fedora14/18 but do not have /etc/redhat-release 164 | elif [ -f /etc/Eos-release ] || [ "$DISTRIBUTION" == "Arista" ]; then 165 | OS="RedHat" 166 | fi 167 | 168 | # Install the necessary package sources 169 | # if [ "$OS" = "RedHat" ]; then 170 | # echo -e "\033[37m\n * Installing dependencies...\n\033[0m" 171 | # if [ -x "/usr/bin/dnf" ]; then 172 | # package_manager='dnf' 173 | # else 174 | # package_manager='yum' 175 | # fi 176 | # which curl &> /dev/null || $sudo_cmd $package_manager -y install curl 177 | # elif [ "$OS" = "Debian" ]; then 178 | # printf "\033[37m\n * Installing dependences...\n\033[0m\n" 179 | # which curl &> /dev/null || ($sudo_cmd apt-get update ; $sudo_cmd apt-get install -y --force-yes curl) 180 | # else 181 | # printf "\033[31mYour OS or distribution are not supported by this install script.\033[0m\n" 182 | # exit; 183 | # fi 184 | 185 | $sudo_cmd rm -rf $WORKDIR 186 | # Create work directory 187 | if [ ! -e $CONF ]; then 188 | $sudo_cmd mkdir -p $WORKDIR 189 | $sudo_cmd mkdir -p $WORKDIR/conf 190 | fi 191 | 192 | printf "\033[37m\n * Downloading Releem Agent, architecture $(arch)...\033[0m\n" 193 | $sudo_cmd cp -f mysqlconfigurer.sh $WORKDIR/mysqlconfigurer.sh 194 | $sudo_cmd cp -f releem-agent-$(arch) $WORKDIR/releem-agent 195 | 196 | 197 | $sudo_cmd chmod 755 $WORKDIR/mysqlconfigurer.sh $WORKDIR/releem-agent 198 | 199 | 200 | printf "\033[37m\n * Configure the application...\033[0m\n" 201 | printf "\033[37m\n * Detected service name for appling config\033[0m\n" 202 | systemctl_cmd=$(which systemctl || true) 203 | if [ -n "$systemctl_cmd" ];then 204 | # Check if MySQL is running 205 | if $sudo_cmd $systemctl_cmd status mysql >/dev/null 2>&1; then 206 | service_name_cmd="$sudo_cmd $systemctl_cmd restart mysql" 207 | elif $sudo_cmd $systemctl_cmd status mysqld >/dev/null 2>&1; then 208 | service_name_cmd="$sudo_cmd $systemctl_cmd restart mysqld" 209 | elif $sudo_cmd $systemctl_cmd status mariadb >/dev/null 2>&1; then 210 | service_name_cmd="$sudo_cmd $systemctl_cmd restart mariadb" 211 | else 212 | printf "\033[31m\n * Failed to determine service to restart. The automatic applying configuration will not work. \n\033[0m" 213 | fi 214 | else 215 | # Check if MySQL is running 216 | if [ -f /etc/init.d/mysql ]; then 217 | service_name_cmd="$sudo_cmd /etc/init.d/mysql restart" 218 | elif [ -f /etc/init.d/mysqld ]; then 219 | service_name_cmd="$sudo_cmd /etc/init.d/mysqld restart" 220 | elif [ -f /etc/init.d/mariadb ]; then 221 | service_name_cmd="$sudo_cmd /etc/init.d/mariadb restart" 222 | else 223 | printf "\033[31m\n * Failed to determine service to restart. The automatic applying configuration will not work. \n\033[0m" 224 | fi 225 | fi 226 | 227 | printf "\033[37m\n * Configure catalog for copy recommend config\033[0m\n" 228 | if [[ -n $RELEEM_MYSQL_MY_CNF_PATH ]]; 229 | then 230 | MYSQL_MY_CNF_PATH=$RELEEM_MYSQL_MY_CNF_PATH 231 | else 232 | if [ -f "/etc/my.cnf" ]; then 233 | MYSQL_MY_CNF_PATH="/etc/my.cnf" 234 | elif [ -f "/etc/mysql/my.cnf" ]; then 235 | MYSQL_MY_CNF_PATH="/etc/mysql/my.cnf" 236 | else 237 | read -p "File my.cnf not found in default path. Please set the current location of the configuration file: " -r 238 | echo # move to a new line 239 | MYSQL_MY_CNF_PATH=$REPLY 240 | fi 241 | fi 242 | if [ ! -f "$MYSQL_MY_CNF_PATH" ]; then 243 | printf "\033[31m * File $MYSQL_MY_CNF_PATH not found. The automatic applying configuration is disabled. Please, reinstall the Releem Agent.\033[0m\n" 244 | else 245 | # FLAG_APPLY_CHANGE=0 246 | # if [[ -z $RELEEM_MYSQL_MY_CNF_PATH ]]; 247 | # then 248 | # read -p "Please confirm MySQL configuration location $MYSQL_MY_CNF_PATH? (Y/N) " -n 1 -r 249 | # echo # move to a new line 250 | # if [[ $REPLY =~ ^[Yy]$ ]] 251 | # then 252 | # FLAG_APPLY_CHANGE=1 253 | # else 254 | # FLAG_APPLY_CHANGE=0 255 | # printf "\033[31m\n * A confirmation has not been received. The automatic applying configuration is disabled. Please, reinstall the Releem Agent.\033[0m\n" 256 | # fi 257 | # else 258 | # FLAG_APPLY_CHANGE=1 259 | # fi 260 | # if [ $FLAG_APPLY_CHANGE -eq 1 ]; 261 | # then 262 | 263 | printf "\033[37m\n * The $MYSQL_MY_CNF_PATH file is used for automatic Releem settings. \n\033[0m" 264 | printf "\033[37m\n * Adding directive includedir to the MySQL configuration $MYSQL_MY_CNF_PATH.\n\033[0m" 265 | $sudo_cmd mkdir -p $MYSQL_CONF_DIR 266 | $sudo_cmd chmod 755 $MYSQL_CONF_DIR 267 | #Исключить дублирование 268 | if [ `$sudo_cmd grep -cE "!includedir $MYSQL_CONF_DIR" $MYSQL_MY_CNF_PATH` -eq 0 ]; 269 | then 270 | echo -e "\n!includedir $MYSQL_CONF_DIR" | $sudo_cmd tee -a $MYSQL_MY_CNF_PATH >/dev/null 271 | fi 272 | # fi 273 | fi 274 | 275 | 276 | printf "\033[37m\n * Configure MySQL user for collect data\033[0m\n" 277 | FLAG_SUCCESS=0 278 | if [ -n "$RELEEM_MYSQL_PASSWORD" ] && [ -n "$RELEEM_MYSQL_LOGIN" ]; then 279 | printf "\033[37m\n * Using MySQL login and password from environment variables\033[0m\n" 280 | FLAG_SUCCESS=1 281 | #elif [ -n "$RELEEM_MYSQL_ROOT_PASSWORD" ]; then 282 | else 283 | printf "\033[37m\n * Using MySQL root user\033[0m\n" 284 | if [[ $(mysqladmin ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} ping 2>/dev/null || true) == "mysqld is alive" ]]; 285 | then 286 | printf "\033[37m\n Connect to MySQL - successful \033[0m\n" 287 | RELEEM_MYSQL_LOGIN="releem" 288 | RELEEM_MYSQL_PASSWORD=$(cat /dev/urandom | tr -cd '%*)?@#~' | head -c2 ; cat /dev/urandom | tr -cd '%*)?@#~A-Za-z0-9%*)?@#~' | head -c16 ; cat /dev/urandom | tr -cd '%*)?@#~' | head -c2 ) 289 | mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "DROP USER '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}' ;" 2>/dev/null || true 290 | mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "CREATE USER '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}' identified by '${RELEEM_MYSQL_PASSWORD}';" 291 | mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT PROCESS ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';" 292 | mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT REPLICATION CLIENT ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';" 293 | mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SHOW VIEW ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';" 294 | if mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SELECT ON performance_schema.events_statements_summary_by_digest TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';" 295 | then 296 | echo "Successfully GRANT" > /dev/null 297 | else 298 | printf "\033[31m\n This database version is too old, and it doesn’t collect SQL Queries Latency metrics. You couldn’t see Latency in the Dashboard.\033[0m\n" 299 | fi 300 | #mysql ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} -Be "GRANT SELECT, PROCESS,EXECUTE, REPLICATION CLIENT,SHOW DATABASES,SHOW VIEW ON *.* TO '${RELEEM_MYSQL_LOGIN}'@'${mysql_user_host}';" 301 | printf "\033[32m\n Created new user \`${RELEEM_MYSQL_LOGIN}\`\033[0m\n" 302 | FLAG_SUCCESS=1 303 | else 304 | printf "\033[31m\n MySQL connection failed with user root with error:\033[0m\n" 305 | mysqladmin ${root_connection_string} --user=root --password=${RELEEM_MYSQL_ROOT_PASSWORD} ping || true 306 | printf "\033[31m\n%s\033[0m\n" "Check that the password is correct, the execution of the command \`mysqladmin ${root_connection_string} --user=root --password= ping\` and reinstall the agent." 307 | exit 1 308 | fi 309 | #else 310 | # printf "\033[31m\n Variable RELEEM_MYSQL_ROOT_PASSWORD not found.\n Please, reinstall the agent by setting the \"RELEEM_MYSQL_ROOT_PASSWORD\" variable\033[0m\n" 311 | # exit 1 312 | fi 313 | 314 | if [ "$FLAG_SUCCESS" == "1" ]; then 315 | if [[ $(mysqladmin ${connection_string} --user=${RELEEM_MYSQL_LOGIN} --password=${RELEEM_MYSQL_PASSWORD} ping 2>/dev/null || true) == "mysqld is alive" ]]; 316 | then 317 | printf "\033[32m\n Connecting to MySQL with user \`${RELEEM_MYSQL_LOGIN}\` - successfull \033[0m\n" 318 | MYSQL_LOGIN=$RELEEM_MYSQL_LOGIN 319 | MYSQL_PASSWORD=$RELEEM_MYSQL_PASSWORD 320 | else 321 | printf "\033[31m\n Connect to mysql failed with user \`${RELEEM_MYSQL_LOGIN}\` with error:\033[0m\n" 322 | mysqladmin ${connection_string} --user=${RELEEM_MYSQL_LOGIN} --password=${RELEEM_MYSQL_PASSWORD} ping || true 323 | printf "\033[31m\n%s\033[0m\n" "Check that the user and password is correct, the execution of the command \`mysqladmin ${connection_string} --user=${RELEEM_MYSQL_LOGIN} --password=${RELEEM_MYSQL_PASSWORD} ping\` and reinstall the agent." 324 | exit 1 325 | fi 326 | fi 327 | 328 | 329 | 330 | # printf "\033[37m\n * Checking ~/.my.cnf...\033[0m\n" 331 | # if [ ! -e ~/.my.cnf ]; then 332 | # printf "\033[37m\n * Please create ~/.my.cnf file with the following content:\033[0m\n" 333 | # echo -e "" 334 | # echo -e "[client]" 335 | # echo -e "user=root" 336 | # echo -e "password=[your MySQL root password]" 337 | # echo -e "" 338 | # read -p "Are you ready to proceed? (Y/N) " -n 1 -r 339 | # echo # move to a new line 340 | # if [[ $REPLY =~ ^[Nn]$ ]] 341 | # then 342 | # exit 1 343 | # fi 344 | # fi 345 | 346 | 347 | printf "\033[37m\n * Configure mysql memory limit\033[0m\n" 348 | if [ -n "$RELEEM_MYSQL_MEMORY_LIMIT" ]; then 349 | 350 | if [ "$RELEEM_MYSQL_MEMORY_LIMIT" -gt 0 ]; then 351 | MYSQL_LIMIT=$RELEEM_MYSQL_MEMORY_LIMIT 352 | fi 353 | else 354 | echo 355 | printf "\033[37m\n In case you are using MySQL in Docker or it isn't dedicated server for MySQL.\033[0m\n" 356 | read -p "Should we limit MySQL memory? (Y/N) " -n 1 -r 357 | echo # move to a new line 358 | if [[ $REPLY =~ ^[Yy]$ ]] 359 | then 360 | read -p "Please set MySQL Memory Limit (megabytes):" -r 361 | echo # move to a new line 362 | MYSQL_LIMIT=$REPLY 363 | fi 364 | fi 365 | 366 | printf "\033[37m\n * Saving variables to Releem Agent configuration\033[0m\n" 367 | 368 | printf "\033[37m\n - Adding API key to the Releem Agent configuration: $CONF\n\033[0m" 369 | echo "apikey=\"$apikey\"" | $sudo_cmd tee -a $CONF >/dev/null 370 | 371 | printf "\033[37m - Adding Releem Configuration Directory $WORKDIR/conf to Releem Agent configuration: $CONF\n\033[0m" 372 | echo "releem_cnf_dir=\"$WORKDIR/conf\"" | $sudo_cmd tee -a $CONF >/dev/null 373 | 374 | if [ -n "$MYSQL_LOGIN" ] && [ -n "$MYSQL_PASSWORD" ]; then 375 | printf "\033[37m - Adding user and password mysql to the Releem Agent configuration: $CONF\n\033[0m" 376 | echo "mysql_user=\"$MYSQL_LOGIN\"" | $sudo_cmd tee -a $CONF >/dev/null 377 | echo "mysql_password=\"$MYSQL_PASSWORD\"" | $sudo_cmd tee -a $CONF >/dev/null 378 | fi 379 | if [ -n "$RELEEM_MYSQL_HOST" ]; then 380 | printf "\033[37m - Adding MySQL host to the Releem Agent configuration: $CONF\n\033[0m" 381 | echo "mysql_host=\"$RELEEM_MYSQL_HOST\"" | $sudo_cmd tee -a $CONF >/dev/null 382 | fi 383 | if [ -n "$RELEEM_MYSQL_PORT" ]; then 384 | printf "\033[37m - Adding MySQL port to the Releem Agent configuration: $CONF\n\033[0m" 385 | echo "mysql_port=\"$RELEEM_MYSQL_PORT\"" | $sudo_cmd tee -a $CONF >/dev/null 386 | fi 387 | if [ -n "$MYSQL_LIMIT" ]; then 388 | printf "\033[37m - Adding Memory Limit to the Releem Agent configuration: $CONF\n\033[0m" 389 | echo "memory_limit=\"$MYSQL_LIMIT\"" | $sudo_cmd tee -a $CONF >/dev/null 390 | fi 391 | if [ -n "$service_name_cmd" ]; then 392 | printf "\033[37m - Adding MySQL restart command to the Releem Agent configuration: $CONF\n\033[0m" 393 | echo "mysql_restart_service=\"$service_name_cmd\"" | $sudo_cmd tee -a $CONF >/dev/null 394 | fi 395 | if [ -d "$MYSQL_CONF_DIR" ]; then 396 | printf "\033[37m - Adding MySQL include directory to the Releem Agent configuration $CONF.\n\033[0m" 397 | echo "mysql_cnf_dir=\"$MYSQL_CONF_DIR\"" | $sudo_cmd tee -a $CONF >/dev/null 398 | fi 399 | if [ -n "$RELEEM_HOSTNAME" ]; then 400 | printf "\033[37m - Adding hostname to the Releem Agent configuration: $CONF\n\033[0m" 401 | echo "hostname=\"$RELEEM_HOSTNAME\"" | $sudo_cmd tee -a $CONF >/dev/null 402 | else 403 | RELEEM_HOSTNAME=$(hostname 2>&1) 404 | if [ $? -eq 0 ]; 405 | then 406 | printf "\033[37m - Adding autodetected hostname to the Releem Agent configuration: $CONF\n\033[0m" 407 | echo "hostname=\"$RELEEM_HOSTNAME\"" | $sudo_cmd tee -a $CONF >/dev/null 408 | else 409 | printf "\033[31m The variable RELEEM_HOSTNAME is not defined and the hostname could not be determined automatically with error\033[0m\n $RELEEM_HOSTNAME.\n\033[0m" 410 | fi 411 | fi 412 | if [ -n "$RELEEM_ENV" ]; then 413 | echo "env=\"$RELEEM_ENV\"" | $sudo_cmd tee -a $CONF >/dev/null 414 | fi 415 | if [ -n "$RELEEM_DEBUG" ]; then 416 | echo "debug=$RELEEM_DEBUG" | $sudo_cmd tee -a $CONF >/dev/null 417 | fi 418 | echo "interval_seconds=60" | $sudo_cmd tee -a $CONF >/dev/null 419 | echo "interval_read_config_seconds=3600" | $sudo_cmd tee -a $CONF >/dev/null 420 | 421 | # Secure the configuration file 422 | $sudo_cmd chmod 640 $CONF 423 | 424 | 425 | printf "\033[37m\n * Configure crontab...\033[0m\n" 426 | RELEEM_CRON="00 00 * * * PATH=/bin:/sbin:/usr/bin:/usr/sbin $RELEEM_COMMAND -u" 427 | if [ -z "$RELEEM_CRON_ENABLE" ]; then 428 | printf "\033[37m Please add the following string in crontab to get recommendations:\033[0m\n" 429 | printf "\033[32m$RELEEM_CRON\033[0m\n\n" 430 | read -p "Can we do it automatically? (Y/N) " -n 1 -r 431 | echo # move to a new line 432 | if [[ $REPLY =~ ^[Yy]$ ]] 433 | then 434 | releem_set_cron 435 | fi 436 | elif [ "$RELEEM_CRON_ENABLE" -gt 0 ]; then 437 | releem_set_cron 438 | fi 439 | 440 | set +e 441 | trap - ERR 442 | if [ -z "$RELEEM_AGENT_DISABLE" ]; then 443 | # First run of Releem Agent to check MySQL Performance Score 444 | printf "\033[37m\n * Executing Releem Agent for first time...\033[0m\n" 445 | $sudo_cmd $WORKDIR/releem-agent -f 446 | $sudo_cmd timeout 3 $WORKDIR/releem-agent 447 | fi 448 | printf "\033[37m\n * Installing and starting Releem Agent service to collect metrics..\033[0m\n" 449 | releem_agent_remove=$($sudo_cmd $WORKDIR/releem-agent remove) 450 | releem_agent_install=$($sudo_cmd $WORKDIR/releem-agent install) 451 | if [ $? -eq 0 ]; then 452 | printf "\033[32m\n Installing Releem Agent - successful\033[0m\n" 453 | else 454 | echo $releem_agent_remove 455 | echo $releem_agent_install 456 | printf "\033[31m\n Installing Releem Agent - failed\033[0m\n" 457 | fi 458 | releem_agent_stop=$($sudo_cmd $WORKDIR/releem-agent stop) 459 | releem_agent_start=$($sudo_cmd $WORKDIR/releem-agent start) 460 | if [ $? -eq 0 ]; then 461 | printf "\033[32m\n Restarting Releem Agent - successful\033[0m\n" 462 | else 463 | echo $releem_agent_stop 464 | echo $releem_agent_start 465 | printf "\033[31m\n Restarting Releem Agent - failed\033[0m\n" 466 | fi 467 | # $sudo_cmd $WORKDIR/releem-agent status 468 | # if [ $? -eq 0 ]; then 469 | # echo "Status successfull" 470 | # else 471 | # echo "remove failes" 472 | # fi 473 | trap on_error ERR 474 | set -e 475 | sleep 3 476 | releem_agent_pid=$(pgrep releem-agent || true) 477 | if [ -z "$releem_agent_pid" ]; then 478 | printf "\033[31m\n The releem-agent process was not found! Check the system log for an error.\033[0m\n" 479 | on_error 480 | exit 1; 481 | fi 482 | # Enable perfomance schema 483 | $sudo_cmd $RELEEM_COMMAND -p 484 | 485 | printf "\033[37m\n\033[0m" 486 | printf "\033[37m * Releem Agent is successfully installed.\033[0m\n" 487 | printf "\033[37m\n\033[0m" 488 | printf "\033[37m * To view Releem recommendations and MySQL metrics, visit https://app.releem.com/dashboard\033[0m" 489 | printf "\033[37m\n\033[0m" 490 | -------------------------------------------------------------------------------- /releem-agent/mysqlconfigurer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # mysqlconfigurer.sh - Version 1.19.3 3 | # (C) Releem, Inc 2022 4 | # All rights reserved 5 | 6 | # Variables 7 | MYSQLCONFIGURER_PATH="/opt/releem/conf/" 8 | RELEEM_CONF_FILE="/opt/releem/releem.conf" 9 | MYSQLCONFIGURER_FILE_NAME="z_aiops_mysql.cnf" 10 | MYSQLTUNER_FILENAME=$MYSQLCONFIGURER_PATH"mysqltuner.pl" 11 | MYSQLTUNER_REPORT=$MYSQLCONFIGURER_PATH"mysqltunerreport.json" 12 | RELEEM_MYSQL_VERSION=$MYSQLCONFIGURER_PATH"mysql_version" 13 | MYSQLCONFIGURER_CONFIGFILE="${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}" 14 | MYSQL_MEMORY_LIMIT=0 15 | VERSION="1.19.3" 16 | RELEEM_INSTALL_PATH=$MYSQLCONFIGURER_PATH"install.sh" 17 | logfile="releem-mysqlconfigurer.log" 18 | 19 | # Set up a named pipe for logging 20 | npipe=/tmp/$$.mysqlconfigurer.tmp 21 | mknod $npipe p 22 | 23 | # Log all output to a log for error checking 24 | tee <$npipe $logfile & 25 | exec 1>&- 26 | exec 1>$npipe 2>&1 27 | 28 | function on_exit() { 29 | curl -s -L -d @$logfile -H "x-releem-api-key: $RELEEM_API_KEY" -H "Content-Type: application/json" -X POST https://api.releem.com/v2/events/configurer_log 30 | rm -f $npipe 31 | } 32 | 33 | trap on_exit EXIT 34 | 35 | function update_agent() { 36 | trap - EXIT 37 | /opt/releem/releem-agent start > /dev/null || true 38 | NEW_VER=$(curl -s -L https://releem.s3.amazonaws.com/v2/current_version_agent) 39 | if [ "$NEW_VER" != "$VERSION" ]; then 40 | if [ "$(printf '%s\n' "$NEW_VER" "$VERSION" | sort -V | head -n1)" = "$VERSION" ]; 41 | then 42 | printf "\033[37m\n * Updating script \e[31;1m%s\e[0m -> \e[32;1m%s\e[0m\n" "$VERSION" "$NEW_VER" 43 | curl -s -L https://releem.s3.amazonaws.com/v2/install.sh > "$RELEEM_INSTALL_PATH" 44 | RELEEM_API_KEY=$RELEEM_API_KEY exec bash "$RELEEM_INSTALL_PATH" -u 45 | /opt/releem/releem-agent --event=agent_updated > /dev/null 46 | fi 47 | fi 48 | } 49 | 50 | function non_blocking_wait() { 51 | PID=$1 52 | if [ ! -d "/proc/$PID" ]; then 53 | wait $PID 54 | CODE=$? 55 | else 56 | CODE=150 57 | fi 58 | return $CODE 59 | } 60 | 61 | 62 | function wait_restart() { 63 | sleep 1 64 | flag=0 65 | spin[0]="-" 66 | spin[1]="\\" 67 | spin[2]="|" 68 | spin[3]="/" 69 | printf "\033[37m\n Waiting for mysql service to start 1200 seconds ${spin[0]}" 70 | while /bin/true; do 71 | PID=$1 72 | non_blocking_wait $PID 73 | CODE=$? 74 | if [ $CODE -ne 150 ]; then 75 | printf "\033[0m\n PID $PID terminated with exit code $CODE" 76 | if [ $CODE -eq 0 ]; then 77 | RETURN_CODE=0 78 | else 79 | RETURN_CODE=7 80 | fi 81 | break 82 | fi 83 | flag=$(($flag + 1)) 84 | if [ $flag == 1200 ]; then 85 | RETURN_CODE=6 86 | break 87 | fi 88 | i=`expr $flag % 4` 89 | printf "\b${spin[$i]}" 90 | sleep 1 91 | done 92 | printf "\033[0m\n" 93 | return $RETURN_CODE 94 | } 95 | 96 | 97 | function check_mysql_version() { 98 | 99 | if [ -f $MYSQLTUNER_REPORT ]; then 100 | mysql_version=$(grep -o '"Version":"[^"]*' $MYSQLTUNER_REPORT | grep -o '[^"]*$') 101 | elif [ -f "$RELEEM_MYSQL_VERSION" ]; then 102 | mysql_version=$(cat $RELEEM_MYSQL_VERSION) 103 | else 104 | printf "\033[37m\n * Please try again later or run Releem Agent manually:\033[0m" 105 | printf "\033[32m\n /opt/releem/releem-agent -f \033[0m\n\n" 106 | exit 1; 107 | fi 108 | if [ -z $mysql_version ]; then 109 | printf "\033[37m\n * Please try again later or run Releem Agent manually:\033[0m" 110 | printf "\033[32m\n /opt/releem/releem-agent -f \033[0m\n\n" 111 | exit 1; 112 | fi 113 | requiredver="5.6.8" 114 | if [ "$(printf '%s\n' "$mysql_version" "$requiredver" | sort -V | head -n1)" = "$requiredver" ]; then 115 | return 0 116 | else 117 | return 1 118 | fi 119 | } 120 | 121 | 122 | function releem_rollback_config() { 123 | printf "\033[31m\n * Rolling back MySQL configuration!\033[0m\n" 124 | if ! check_mysql_version; then 125 | printf "\033[31m\n * MySQL version is lower than 5.6.7. Check the documentation https://github.com/Releem/mysqlconfigurer#how-to-apply-the-recommended-configuration for applying the configuration. \033[0m\n" 126 | exit 2 127 | fi 128 | if [ -z "$RELEEM_MYSQL_CONFIG_DIR" -o ! -d "$RELEEM_MYSQL_CONFIG_DIR" ]; then 129 | printf "\033[37m\n * MySQL configuration directory is not found.\033[0m" 130 | printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m" 131 | exit 3; 132 | fi 133 | if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then 134 | printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" 135 | exit 4; 136 | fi 137 | 138 | FLAG_RESTART_SERVICE=1 139 | if [ -z "$RELEEM_RESTART_SERVICE" ]; then 140 | read -p "Please confirm restart MySQL service? (Y/N) " -n 1 -r 141 | echo # move to a new line 142 | if [[ ! $REPLY =~ ^[Yy]$ ]] 143 | then 144 | printf "\033[37m\n * A confirmation to restart the service has not been received. Releem recommended configuration has not been roll back.\033[0m\n" 145 | FLAG_RESTART_SERVICE=0 146 | fi 147 | elif [ "$RELEEM_RESTART_SERVICE" -eq 0 ]; then 148 | FLAG_RESTART_SERVICE=0 149 | fi 150 | if [ "$FLAG_RESTART_SERVICE" -eq 0 ]; then 151 | exit 5 152 | fi 153 | 154 | printf "\033[31m\n * Deleting a configuration file... \033[0m\n" 155 | rm -rf $RELEEM_MYSQL_CONFIG_DIR/$MYSQLCONFIGURER_FILE_NAME 156 | #echo "----Test config-------" 157 | if [ -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" ]; then 158 | printf "\033[31m\n * Restoring a backup copy of the configuration file ${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp... \033[0m\n" 159 | cp -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" 160 | fi 161 | 162 | printf "\033[31m\n * Restarting with command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" 163 | eval "$RELEEM_MYSQL_RESTART_SERVICE" & 164 | wait_restart $! 165 | RESTART_CODE=$? 166 | 167 | #if [[ $(mysqladmin ${connection_string} --user=${MYSQL_LOGIN} --password=${MYSQL_PASSWORD} ping 2>/dev/null || true) == "mysqld is alive" ]]; 168 | if [ $RESTART_CODE -eq 0 ]; 169 | then 170 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m The MySQL service restarted successfully!\033[0m\n" 171 | rm -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" 172 | elif [ $RESTART_CODE -eq 6 ]; 173 | then 174 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m The MySQL service failed to restart in 1200 seconds! Check the MySQL error log! \033[0m\n" 175 | elif [ $RESTART_CODE -eq 7 ]; 176 | then 177 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m The MySQL service failed to restart with error! Check the MySQL error log! \033[0m\n" 178 | fi 179 | /opt/releem/releem-agent --event=config_rollback > /dev/null 180 | exit "${RESTART_CODE}" 181 | } 182 | 183 | 184 | 185 | function releem_ps_mysql() { 186 | FLAG_CONFIGURE=1 187 | status_ps=$(mysql ${connection_string} --user=${MYSQL_LOGIN} --password=${MYSQL_PASSWORD} -BNe "show global variables like 'performance_schema'" 2>/dev/null | awk '{print $2}') 188 | if [ "$status_ps" != "ON" ]; then 189 | FLAG_CONFIGURE=0 190 | fi 191 | 192 | status_slowlog=$(mysql ${connection_string} --user=${MYSQL_LOGIN} --password=${MYSQL_PASSWORD} -BNe "show global variables like 'slow_query_log'" 2>/dev/null | awk '{print $2}') 193 | if [ "$status_slowlog" != "ON" ]; then 194 | FLAG_CONFIGURE=0 195 | fi 196 | 197 | if [ -n "$RELEEM_MYSQL_CONFIG_DIR" -a -d "$RELEEM_MYSQL_CONFIG_DIR" ]; then 198 | printf "\033[37m\n * Enabling Performance schema and SlowLog to collect metrics...\n\033[0m\n" 199 | echo -e "### This configuration was recommended by Releem. https://releem.com\n[mysqld]\nperformance_schema = 1\nslow_query_log = 1" > "$RELEEM_MYSQL_CONFIG_DIR/collect_metrics.cnf" 200 | chmod 644 $RELEEM_MYSQL_CONFIG_DIR/collect_metrics.cnf 201 | else 202 | printf "\033[31m\n MySQL configuration directory is not found.\033[0m" 203 | printf "\033[31m\n Try to reinstall Releem Agent.\033[0m" 204 | exit 3; 205 | fi 206 | if [ "$FLAG_CONFIGURE" -eq 1 ]; then 207 | printf "\033[37m\n * Performance schema and SlowLog are enabled for metrics collection.\033[0m\n" 208 | exit 0 209 | fi 210 | printf "\033[37m To apply changes to the mysql configuration, you need to restart the service\n\033[0m\n" 211 | FLAG_RESTART_SERVICE=1 212 | if [ -z "$RELEEM_RESTART_SERVICE" ]; then 213 | read -p "Please confirm restart MySQL service? (Y/N) " -n 1 -r 214 | echo # move to a new line 215 | if [[ ! $REPLY =~ ^[Yy]$ ]] 216 | then 217 | printf "\033[31m A confirmation to restart the service has not been received. \033[0m\n" 218 | FLAG_RESTART_SERVICE=0 219 | fi 220 | elif [ "$RELEEM_RESTART_SERVICE" -eq 0 ]; then 221 | FLAG_RESTART_SERVICE=0 222 | fi 223 | if [ "$FLAG_RESTART_SERVICE" -eq 0 ]; then 224 | printf "\033[31m\n * For appling change in configuration mysql need restart service.\n\033[0m" 225 | printf "\033[31m Run the command \`bash /opt/releem/mysqlconfigurer.sh -p\` when it is possible to restart the service.\033[0m\n" 226 | exit 0 227 | fi 228 | #echo "-------Test config-------" 229 | printf "\033[37m Restarting service with command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" 230 | eval "$RELEEM_MYSQL_RESTART_SERVICE" & 231 | wait_restart $! 232 | RESTART_CODE=$? 233 | 234 | #if [[ $(mysqladmin ${connection_string} --user=${MYSQL_LOGIN} --password=${MYSQL_PASSWORD} ping 2>/dev/null || true) == "mysqld is alive" ]]; 235 | if [ $RESTART_CODE -eq 0 ]; 236 | then 237 | printf "\033[32m\n The MySQL service restarted successfully!\033[0m\n" 238 | printf "\033[32m\n Performance schema and Slow Log are enabled.\033[0m\n" 239 | elif [ $RESTART_CODE -eq 6 ]; 240 | then 241 | printf "\033[31m\n The MySQL service failed to restart in 1200 seconds! Check the MySQL error log!\033[0m\n" 242 | elif [ $RESTART_CODE -eq 7 ]; 243 | then 244 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m The MySQL service failed to restart with error! Check the MySQL error log! \033[0m\n" 245 | fi 246 | exit "${RESTART_CODE}" 247 | } 248 | 249 | 250 | function releem_apply_config() { 251 | if [ "$1" == "auto" ]; 252 | then 253 | releem_apply_auto 254 | else 255 | releem_apply_manual 256 | fi 257 | } 258 | 259 | function releem_apply_auto() { 260 | /opt/releem/releem-agent --task=apply_config > /dev/null 261 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m Request to create a task to apply the configuration was sended!\033[0m\n" 262 | exit 0 263 | } 264 | 265 | function releem_apply_manual() { 266 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Applying the recommended MySQL configuration...\033[0m\n" 267 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Getting the latest up-to-date configuration...\033[0m\n" 268 | /opt/releem/releem-agent -c >/dev/null 2>&1 || true 269 | 270 | if [ ! -f $MYSQLCONFIGURER_CONFIGFILE ]; then 271 | printf "\033[37m\n * Recommended MySQL configuration is not found.\033[0m" 272 | printf "\033[37m\n * Please apply recommended configuration later or run Releem Agent manually:\033[0m" 273 | printf "\033[32m\n /opt/releem/releem-agent -f \033[0m\n\n" 274 | exit 1; 275 | fi 276 | if ! check_mysql_version; then 277 | printf "\033[31m\n * MySQL version is lower than 5.6.7. Check the documentation https://github.com/Releem/mysqlconfigurer#how-to-apply-the-recommended-configuration for applying the configuration. \033[0m\n" 278 | exit 2 279 | fi 280 | if [ -z "$RELEEM_MYSQL_CONFIG_DIR" -o ! -d "$RELEEM_MYSQL_CONFIG_DIR" ]; then 281 | printf "\033[37m\n * MySQL configuration directory is not found.\033[0m" 282 | printf "\033[37m\n * Try to reinstall Releem Agent, and please set the my.cnf location.\033[0m" 283 | exit 3; 284 | fi 285 | if [ -z "$RELEEM_MYSQL_RESTART_SERVICE" ]; then 286 | printf "\033[37m\n * The command to restart the MySQL service was not found. Try to reinstall Releem Agent.\033[0m" 287 | exit 4; 288 | fi 289 | diff_cmd=$(which diff || true) 290 | if [ -n "$diff_cmd" ];then 291 | diff "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" "$MYSQLCONFIGURER_CONFIGFILE" > /dev/null 2>&1 292 | retVal=$? 293 | if [ $retVal -eq 0 ]; 294 | then 295 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m The new configuration does not differ from the current one applied. No restart is required.!\033[0m\n" 296 | exit 0 297 | fi 298 | fi 299 | 300 | FLAG_RESTART_SERVICE=1 301 | if [ -z "$RELEEM_RESTART_SERVICE" ]; then 302 | read -p "Please confirm the MySQL service restart? (Y/N) " -n 1 -r 303 | echo # move to a new line 304 | if [[ ! $REPLY =~ ^[Yy]$ ]] 305 | then 306 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m A confirmation to restart the service has not been received. Releem recommended configuration has not been applied.\033[0m\n" 307 | FLAG_RESTART_SERVICE=0 308 | fi 309 | elif [ "$RELEEM_RESTART_SERVICE" -eq 0 ]; then 310 | FLAG_RESTART_SERVICE=0 311 | fi 312 | if [ "$FLAG_RESTART_SERVICE" -eq 0 ]; then 313 | exit 5 314 | fi 315 | 316 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Copying file $MYSQLCONFIGURER_CONFIGFILE to directory $RELEEM_MYSQL_CONFIG_DIR/...\033[0m\n" 317 | if [ ! -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" ]; then 318 | yes | cp -f "${RELEEM_MYSQL_CONFIG_DIR}/${MYSQLCONFIGURER_FILE_NAME}" "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" 319 | fi 320 | yes | cp -fr $MYSQLCONFIGURER_CONFIGFILE $RELEEM_MYSQL_CONFIG_DIR/ 321 | chmod 644 $RELEEM_MYSQL_CONFIG_DIR/* 322 | 323 | #echo "-------Test config-------" 324 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[37m Restarting MySQL with the command '$RELEEM_MYSQL_RESTART_SERVICE'...\033[0m\n" 325 | eval "$RELEEM_MYSQL_RESTART_SERVICE" & 326 | wait_restart $! 327 | RESTART_CODE=$? 328 | 329 | if [ $RESTART_CODE -eq 0 ]; 330 | then 331 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m The MySQL service restarted successfully!\033[0m\n" 332 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m Recommended configuration applied successfully!\033[0m\n" 333 | printf "\n`date +%Y%m%d-%H:%M:%S` Releem Score and Unapplied recommendations in the Releem Dashboard will be updated in a few minutes.\n" 334 | rm -f "${MYSQLCONFIGURER_PATH}${MYSQLCONFIGURER_FILE_NAME}.bkp" 335 | elif [ $RESTART_CODE -eq 6 ]; 336 | then 337 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m MySQL service failed to restart in 1200 seconds! \033[0m\n" 338 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m Wait for the MySQL service to start and Check the MySQL error log!\033[0m\n" 339 | 340 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m Try to roll back the configuration application using the command: \033[0m\n" 341 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m bash /opt/releem/mysqlconfigurer.sh -r\033[0m\n\n" 342 | elif [ $RESTART_CODE -eq 7 ]; 343 | then 344 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m MySQL service failed to restart! Check the MySQL error log! \033[0m\n" 345 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[31m Try to roll back the configuration application using the command: \033[0m\n" 346 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m bash /opt/releem/mysqlconfigurer.sh -r\033[0m\n\n" 347 | fi 348 | /opt/releem/releem-agent --event=config_applied > /dev/null 349 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m Notification to Releem Platform was sent successfully!\033[0m\n" 350 | 351 | exit "${RESTART_CODE}" 352 | } 353 | 354 | 355 | function releem_runnig_cron() { 356 | HOUR=$(date +%I) 357 | MINUTE=$(date +%M) 358 | send_metrics 359 | if [ "${HOUR}" == "12" ] && [ "${MINUTE}" == "10" ]; 360 | then 361 | get_config 362 | update_agent 363 | fi 364 | exit 0 365 | } 366 | 367 | function send_metrics() { 368 | #echo -e "\033[37m\n * Checking the environment...\033[0m" 369 | check_env 370 | ##### PARAMETERS ##### 371 | CACHE_TTL="55" 372 | CACHE_FILE_STATUS="/tmp/releem.mysql.status.`echo $MYSQLCONFIGURER_CONFIGFILE | md5sum | cut -d" " -f1`.cache" 373 | CACHE_FILE_VARIABLES="/tmp/releem.mysql.variables.`echo $MYSQLCONFIGURER_CONFIGFILE | md5sum | cut -d" " -f1`.cache" 374 | EXEC_TIMEOUT="1" 375 | NOW_TIME=`date '+%s'` 376 | ##### RUN ##### 377 | # Collect MySQL metrics 378 | #echo -e "\033[37m\n * Collecting metrics...\033[0m" 379 | 380 | if [ -s "${CACHE_FILE_STATUS}" ]; then 381 | CACHE_TIME=`stat -c"%Y" "${CACHE_FILE_STATUS}"` 382 | else 383 | CACHE_TIME=0 384 | fi 385 | DELTA_TIME=$((${NOW_TIME} - ${CACHE_TIME})) 386 | # 387 | if [ ${DELTA_TIME} -lt ${EXEC_TIMEOUT} ]; then 388 | sleep $((${EXEC_TIMEOUT} - ${DELTA_TIME})) 389 | elif [ ${DELTA_TIME} -gt ${CACHE_TTL} ]; then 390 | echo "" >> "${CACHE_FILE_STATUS}" # !!! 391 | DATACACHE=`mysql -sNe "show global status;"` 392 | echo "${DATACACHE}" > "${CACHE_FILE_STATUS}" # !!! 393 | chmod 640 "${CACHE_FILE_STATUS}" 394 | fi 395 | 396 | if [ -s "${CACHE_FILE_VARIABLES}" ]; then 397 | CACHE_TIME=`stat -c"%Y" "${CACHE_FILE_VARIABLES}"` 398 | else 399 | CACHE_TIME=0 400 | fi 401 | DELTA_TIME=$((${NOW_TIME} - ${CACHE_TIME})) 402 | # 403 | if [ ${DELTA_TIME} -lt ${EXEC_TIMEOUT} ]; then 404 | sleep $((${EXEC_TIMEOUT} - ${DELTA_TIME})) 405 | elif [ ${DELTA_TIME} -gt ${CACHE_TTL} ]; then 406 | echo "" >> "${CACHE_FILE_VARIABLES}" # !!! 407 | DATACACHE=`mysql -sNe "show global variables;"` 408 | echo "${DATACACHE}" > "${CACHE_FILE_VARIABLES}" # !!! 409 | chmod 640 "${CACHE_FILE_VARIABLES}" 410 | fi 411 | 412 | QUESTIONS=`cat ${CACHE_FILE_STATUS} | grep -w 'Questions' | awk '{print $2}'` 413 | TIMESTAMP=`stat -c"%Y" "${CACHE_FILE_STATUS}"` 414 | HOSTNAME=`cat ${CACHE_FILE_VARIABLES} | grep -w 'hostname' | awk '{print $2}'` 415 | 416 | JSON_STRING='{"Hostname": "'${HOSTNAME}'", "Timestamp":"'${TIMESTAMP}'", "ReleemMetrics": {"Questions": "'${QUESTIONS}'"}}' 417 | #echo -e "\033[37m\n * Sending metrics to Releem Cloud Platform...\033[0m" 418 | # Send metrics to Releem Platform. The answer is the configuration file for MySQL 419 | curl -s -d "$JSON_STRING" -H "x-releem-api-key: $RELEEM_API_KEY" -H "Content-Type: application/json" -X POST https://api.releem.com/v1/mysql 420 | } 421 | 422 | function check_env() { 423 | # Check RELEEM_API_KEY is not empty 424 | if [ -z "$RELEEM_API_KEY" ]; then 425 | echo >&2 "RELEEM_API_KEY is empty please sign up at https://releem.com/appsignup to get your Releem API key. Aborting." 426 | exit 1; 427 | fi 428 | command -v curl >/dev/null 2>&1 || { echo >&2 "Curl is not installed. Please install Curl. Aborting."; exit 1; } 429 | 430 | } 431 | 432 | function get_config() { 433 | echo -e "\033[37m\n * Checking the environment...\033[0m" 434 | check_env 435 | 436 | command -v perl >/dev/null 2>&1 || { echo >&2 "Perl is not installed. Please install Perl. Aborting."; exit 1; } 437 | perl -e "use JSON;" >/dev/null 2>&1 || { echo >&2 "Perl module JSON is not installed. Please install Perl module JSON. Aborting."; exit 1; } 438 | 439 | # Check if the tmp folder exists 440 | if [ -d "$MYSQLCONFIGURER_PATH" ]; then 441 | # Clear tmp directory 442 | rm $MYSQLCONFIGURER_PATH/* 443 | else 444 | # Create tmp directory 445 | mkdir $MYSQLCONFIGURER_PATH 446 | fi 447 | 448 | # Check if MySQLTuner already downloaded and download if it doesn't exist 449 | if [ ! -f "$MYSQLTUNER_FILENAME" ]; then 450 | # Download latest version of the MySQLTuner 451 | curl -s -o $MYSQLTUNER_FILENAME -L https://raw.githubusercontent.com/major/MySQLTuner-perl/fdd42e76857532002b8037cafddec3e38983dde8/mysqltuner.pl 452 | chmod +x $MYSQLTUNER_FILENAME 453 | fi 454 | 455 | echo -e "\033[37m\n * Collecting metrics to recommend a config...\033[0m" 456 | 457 | # Collect MySQL metrics 458 | if perl $MYSQLTUNER_FILENAME --json --verbose --notbstat --nocolstat --noidxstat --nopfstat --forcemem=$MYSQL_MEMORY_LIMIT --outputfile="$MYSQLTUNER_REPORT" --user=${MYSQL_LOGIN} --pass=${MYSQL_PASSWORD} ${connection_string} > /dev/null; then 459 | 460 | echo -e "\033[37m\n * Sending metrics to Releem Cloud Platform...\033[0m" 461 | 462 | # Send metrics to Releem Platform. The answer is the configuration file for MySQL 463 | curl -s -d @$MYSQLTUNER_REPORT -H "x-releem-api-key: $RELEEM_API_KEY" -H "Content-Type: application/json" -X POST https://api.releem.com/v1/mysql -o "$MYSQLCONFIGURER_CONFIGFILE" 464 | 465 | echo -e "\033[37m\n * Downloading recommended MySQL configuration from Releem Cloud Platform...\033[0m" 466 | 467 | # Show recommended configuration and exit 468 | msg="\n\n#---------------Releem Agent Report-------------\n\n" 469 | printf "${msg}" 470 | 471 | echo -e "1. Recommended MySQL configuration downloaded to ${MYSQLCONFIGURER_CONFIGFILE}" 472 | echo 473 | echo -e "2. To check MySQL Performance Score please visit https://app.releem.com/dashboard?menu=metrics" 474 | echo 475 | echo -e "3. To apply the recommended configuration please read documentation https://app.releem.com/dashboard" 476 | else 477 | # If error then show report and exit 478 | errormsg=" \ 479 | \n\n\n\n--------Releem Agent completed with error--------\n \ 480 | \nCheck $MYSQLTUNER_REPORT for details \n \ 481 | \n--------Please fix the error and run Releem Agent again--------\n" 482 | printf "${errormsg}" >&2 483 | fi 484 | 485 | } 486 | connection_string="" 487 | if test -f $RELEEM_CONF_FILE ; then 488 | . $RELEEM_CONF_FILE 489 | 490 | if [ ! -z $apikey ]; then 491 | RELEEM_API_KEY=$apikey 492 | fi 493 | if [ ! -z $memory_limit ]; then 494 | MYSQL_MEMORY_LIMIT=$memory_limit 495 | fi 496 | if [ ! -z $mysql_cnf_dir ]; then 497 | RELEEM_MYSQL_CONFIG_DIR=$mysql_cnf_dir 498 | fi 499 | if [ ! -z "$mysql_restart_service" ]; then 500 | RELEEM_MYSQL_RESTART_SERVICE=$mysql_restart_service 501 | fi 502 | if [ ! -z "$mysql_user" ]; then 503 | MYSQL_LOGIN=$mysql_user 504 | fi 505 | if [ ! -z "$mysql_password" ]; then 506 | MYSQL_PASSWORD=$mysql_password 507 | fi 508 | if [ ! -z "$mysql_host" ]; then 509 | if [ -S "$mysql_host" ]; then 510 | connection_string="${connection_string} --socket=$mysql_host" 511 | else 512 | connection_string="${connection_string} --host=$mysql_host" 513 | fi 514 | else 515 | connection_string="${connection_string} --host=127.0.0.1" 516 | fi 517 | if [ ! -z "$mysql_port" ]; then 518 | connection_string="${connection_string} --port=$mysql_port" 519 | else 520 | connection_string="${connection_string} --port=3306" 521 | fi 522 | fi 523 | 524 | 525 | # Parse parameters 526 | while getopts "k:m:s:arcpu" option 527 | do 528 | case "${option}" in 529 | k) RELEEM_API_KEY=${OPTARG};; 530 | m) MYSQL_MEMORY_LIMIT=${OPTARG};; 531 | a) releem_apply_manual;; 532 | s) releem_apply_config ${OPTARG};; 533 | r) releem_rollback_config;; 534 | c) get_config;; 535 | p) releem_ps_mysql;; 536 | u) update_agent; exit 0;; 537 | esac 538 | done 539 | 540 | printf "\033[37m\n\033[0m" 541 | printf "\033[37m * To run Releem Agent manually please use the following command:\033[0m\n" 542 | printf "\033[32m /opt/releem/releem-agent -f\033[0m\n\n" -------------------------------------------------------------------------------- /releem.conf: -------------------------------------------------------------------------------- 1 | # Debug bool `hcl:"debug"` 2 | # Used for testing and debugging 3 | debug=false 4 | 5 | # Used to specify the environment: prod, dev 6 | env="prod" 7 | 8 | # ApiKey string `hcl:"apikey"` 9 | # Defaults to 3600 seconds, api key for Releem Platform. 10 | apikey="" 11 | 12 | # Hostname string `hcl:"hostname"` 13 | # Hostname for instance 14 | hostname="" 15 | 16 | # MemoryLimit int `hcl:"memory_limit"` 17 | # Defaults to 0, Mysql memory usage limit. 18 | memory_limit=0 19 | 20 | # MetricsPeriod time.Duration `hcl:"interval_seconds"` 21 | # Defaults to 30 seconds, how often metrics are collected. 22 | interval_seconds=60 23 | 24 | # ReadConfigPeriod time.Duration `hcl:"interval_read_config_seconds"` 25 | # Defaults to 3600 seconds, how often to update the values from the config. 26 | interval_read_config_seconds=3600 27 | 28 | # GenerateConfigPeriod time.Duration `hcl:"interval_generate_config_seconds"` 29 | # Defaults to 43200 seconds, how often to generate recommend the config. 30 | interval_generate_config_seconds=43200 31 | 32 | # QueryOptimization time.Duration `hcl:"interval_query_optimization_seconds"` 33 | # Defaults to 3600 seconds, how often query metrics are collected. 34 | interval_query_optimization_seconds=3600 35 | 36 | # QueryOptimizationCollectSqlTextPeriod time.Duration `hcl:"interval_query_optimization_collect_sqltext_seconds"` 37 | # Defaults to 1 seconds, how often query sql text are collected. 38 | interval_query_optimization_collect_sqltext_seconds=1 39 | 40 | # MysqlUser string`hcl:"mysql_user"` 41 | # Mysql user name for collection metrics. 42 | mysql_user="releem" 43 | 44 | # MysqlPassword string `hcl:"mysql_password"` 45 | # Mysql user password for collection metrics. 46 | mysql_password="releem" 47 | 48 | # MysqlHost string `hcl:"mysql_host"` 49 | # Mysql host for collection metrics. 50 | mysql_host="127.0.0.1" 51 | 52 | # MysqlPort string `hcl:"mysql_port"` 53 | # Mysql port for collection metrics. 54 | mysql_port="3306" 55 | 56 | # CommandRestartService string `hcl:"mysql_restart_service"` 57 | # Defaults to 3600 seconds, command to restart service mysql. 58 | mysql_restart_service=" /bin/systemctl restart mysql" 59 | 60 | # MysqlConfDir string `hcl:"mysql_cnf_dir"` 61 | # Defaults to 3600 seconds, the path to copy the recommended config. 62 | mysql_cnf_dir="/etc/mysql/releem.conf.d" 63 | 64 | # ReleemConfDir string `hcl:"releem_cnf_dir"` 65 | # Defaults to 3600 seconds, Releem Agent configuration path. 66 | releem_cnf_dir="/opt/releem/conf" 67 | 68 | # Collect Explain bool `hcl:"query_optimization"` 69 | # Releem collect explain for query 70 | query_optimization=false 71 | 72 | # databases_query_optimization string `hcl:"databases_query_optimization"` 73 | # List of databases for query optimization 74 | databases_query_optimization="" 75 | 76 | # releem_region string `hcl:"releem_region"` 77 | # Server data storage region - EU or empty. 78 | releem_region="" -------------------------------------------------------------------------------- /repeater/logger.go: -------------------------------------------------------------------------------- 1 | package repeater 2 | 3 | import ( 4 | "io" 5 | "log" 6 | 7 | "github.com/Releem/mysqlconfigurer/models" 8 | logging "github.com/google/logger" 9 | ) 10 | 11 | type LogMetricsRepeater struct { 12 | logger logging.Logger 13 | } 14 | 15 | func (lr LogMetricsRepeater) ProcessMetrics(metrics models.Metric) error { 16 | for _, m := range metrics { 17 | lr.logger.Infof("%s", m) 18 | } 19 | return nil 20 | } 21 | 22 | func NewLogMetricsRepeater() LogMetricsRepeater { 23 | logger := *logging.Init("releem-agent", true, false, io.Discard) 24 | defer logger.Close() 25 | logging.SetFlags(log.LstdFlags | log.Lshortfile) 26 | return LogMetricsRepeater{logger} 27 | } 28 | -------------------------------------------------------------------------------- /repeater/releemConfiguration.go: -------------------------------------------------------------------------------- 1 | package repeater 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "os" 9 | 10 | "github.com/Releem/mysqlconfigurer/config" 11 | "github.com/Releem/mysqlconfigurer/models" 12 | "github.com/Releem/mysqlconfigurer/utils" 13 | logging "github.com/google/logger" 14 | 15 | "time" 16 | ) 17 | 18 | type ReleemConfigurationsRepeater struct { 19 | logger logging.Logger 20 | configuration *config.Config 21 | } 22 | 23 | func (repeater ReleemConfigurationsRepeater) ProcessMetrics(context models.MetricContext, metrics models.Metrics, Mode models.ModeType) (interface{}, error) { 24 | defer utils.HandlePanic(repeater.configuration, repeater.logger) 25 | repeater.logger.V(5).Info(Mode.Name, Mode.Type) 26 | var buffer bytes.Buffer 27 | encoder := json.NewEncoder(&buffer) 28 | if err := encoder.Encode(metrics); err != nil { 29 | repeater.logger.Error("Failed to encode metrics: ", err) 30 | } 31 | repeater.logger.V(5).Info("Result Send data: ", buffer.String()) 32 | var api_domain, subdomain, domain string 33 | env := context.GetEnv() 34 | 35 | if env == "dev2" { 36 | subdomain = "dev2." 37 | } else if env == "dev" { 38 | subdomain = "dev." 39 | } else if env == "stage" { 40 | subdomain = "stage." 41 | } else { 42 | subdomain = "" 43 | } 44 | if repeater.configuration.ReleemRegion == "EU" { 45 | domain = "eu.releem.com" 46 | } else { 47 | domain = "releem.com" 48 | } 49 | if Mode.Name == "TaskSet" && Mode.Type == "queries_optimization" { 50 | api_domain = "https://api.queries." + subdomain + domain + "/v2/" 51 | } else if Mode.Name == "Metrics" { 52 | api_domain = "https://api.queries." + subdomain + domain + "/v2/" 53 | } else { 54 | api_domain = "https://api." + subdomain + domain + "/v2/" 55 | } 56 | 57 | if Mode.Name == "Configurations" { 58 | if Mode.Type == "set" { 59 | api_domain = api_domain + "mysql" 60 | } else if Mode.Type == "get" { 61 | api_domain = api_domain + "config" 62 | } else if Mode.Type == "get-json" { 63 | api_domain = api_domain + "config?json=1" 64 | } else { 65 | api_domain = api_domain + "mysql" 66 | } 67 | } else if Mode.Name == "Metrics" { 68 | if Mode.Type == "QueryOptimization" { 69 | api_domain = api_domain + "queries/metrics" 70 | } else { 71 | api_domain = api_domain + "mysql/metrics" 72 | } 73 | } else if Mode.Name == "Event" { 74 | api_domain = api_domain + "event/" + Mode.Type 75 | } else if Mode.Name == "TaskGet" { 76 | api_domain = api_domain + "task/task_get" 77 | } else if Mode.Name == "TaskSet" { 78 | api_domain = api_domain + "task/" + Mode.Type 79 | } else if Mode.Name == "TaskStatus" { 80 | api_domain = api_domain + "task/task_status" 81 | } 82 | repeater.logger.V(5).Info(api_domain) 83 | 84 | req, err := http.NewRequest(http.MethodPost, api_domain, &buffer) 85 | if err != nil { 86 | repeater.logger.Error("Request: could not create request: ", err) 87 | return nil, err 88 | } 89 | req.Header.Set("x-releem-api-key", context.GetApiKey()) 90 | 91 | client := http.Client{ 92 | Timeout: 10 * time.Minute, 93 | } 94 | res, err := client.Do(req) 95 | if err != nil { 96 | repeater.logger.Error("Request: error making http request: ", err) 97 | return nil, err 98 | } 99 | defer res.Body.Close() 100 | 101 | body_res, err := io.ReadAll(res.Body) 102 | if err != nil { 103 | repeater.logger.Error("Response: error read body request: ", err) 104 | return nil, err 105 | } 106 | if res.StatusCode != 200 && res.StatusCode != 201 { 107 | repeater.logger.Error("Response: status code: ", res.StatusCode) 108 | repeater.logger.Error("Response: body:\n", string(body_res)) 109 | } else { 110 | repeater.logger.V(5).Info("Response: status code: ", res.StatusCode) 111 | repeater.logger.V(5).Info("Response: body:\n", string(body_res)) 112 | 113 | if Mode.Name == "Configurations" { 114 | err = os.WriteFile(context.GetReleemConfDir()+"/z_aiops_mysql.cnf", body_res, 0644) 115 | if err != nil { 116 | repeater.logger.Error("WriteFile: Error write to file: ", err) 117 | return nil, err 118 | } 119 | return string(body_res), err 120 | 121 | } else if Mode.Name == "Metrics" { 122 | return string(body_res), err 123 | } else if Mode.Name == "Event" { 124 | return nil, err 125 | } else if Mode.Name == "TaskGet" { 126 | result_data := models.Task{} 127 | err := json.Unmarshal(body_res, &result_data) 128 | return result_data, err 129 | } else if Mode.Name == "TaskSet" { 130 | return nil, err 131 | } else if Mode.Name == "TaskStatus" { 132 | return nil, err 133 | } 134 | } 135 | return nil, err 136 | } 137 | 138 | func NewReleemConfigurationsRepeater(configuration *config.Config, logger logging.Logger) ReleemConfigurationsRepeater { 139 | return ReleemConfigurationsRepeater{logger, configuration} 140 | } 141 | -------------------------------------------------------------------------------- /tasks/tasks.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "os/exec" 8 | "runtime" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/Releem/mysqlconfigurer/config" 14 | "github.com/Releem/mysqlconfigurer/models" 15 | "github.com/Releem/mysqlconfigurer/utils" 16 | "github.com/aws/aws-sdk-go-v2/service/rds" 17 | "github.com/aws/aws-sdk-go-v2/service/rds/types" 18 | "github.com/aws/aws-sdk-go/aws" 19 | logging "github.com/google/logger" 20 | 21 | config_aws "github.com/aws/aws-sdk-go-v2/config" 22 | ) 23 | 24 | func ProcessTaskFunc(metrics *models.Metrics, repeaters models.MetricsRepeater, gatherers []models.MetricsGatherer, logger logging.Logger, configuration *config.Config) func() { 25 | return func() { 26 | ProcessTask(metrics, repeaters, gatherers, logger, configuration) 27 | } 28 | } 29 | 30 | func ProcessTask(metrics *models.Metrics, repeaters models.MetricsRepeater, gatherers []models.MetricsGatherer, logger logging.Logger, configuration *config.Config) { 31 | defer utils.HandlePanic(configuration, logger) 32 | output := make(models.MetricGroupValue) 33 | //metrics := collectMetrics(gatherers, logger) 34 | var task_output string 35 | task := utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "TaskGet", Type: ""}) 36 | if task.(models.Task).TaskTypeID == nil { 37 | return 38 | } 39 | 40 | TaskTypeID := *task.(models.Task).TaskTypeID 41 | TaskID := *task.(models.Task).TaskID 42 | var stdout, stderr bytes.Buffer 43 | 44 | output["task_id"] = TaskID 45 | output["task_type_id"] = TaskTypeID 46 | output["task_status"] = 3 47 | output["task_output"] = "" 48 | 49 | metrics.ReleemAgent.Tasks = output 50 | utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "TaskStatus", Type: ""}) 51 | logger.Info(" * Task with id - ", TaskID, " and type id - ", TaskTypeID, " is being started...") 52 | 53 | if TaskTypeID == 0 { 54 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/mysqlconfigurer.sh -a", []string{"RELEEM_RESTART_SERVICE=1"}, logger) 55 | output["task_output"] = output["task_output"].(string) + task_output 56 | 57 | if output["task_exit_code"] == 7 { 58 | var rollback_exit_code int 59 | cmd := exec.Command(configuration.ReleemDir+"/mysqlconfigurer.sh", "-r") 60 | cmd.Stdout = &stdout 61 | cmd.Stderr = &stderr 62 | cmd.Env = append(cmd.Environ(), "RELEEM_RESTART_SERVICE=1") 63 | err := cmd.Run() 64 | if err != nil { 65 | output["task_output"] = output["task_output"].(string) + err.Error() 66 | logger.Error(err) 67 | if exiterr, ok := err.(*exec.ExitError); ok { 68 | rollback_exit_code = exiterr.ExitCode() 69 | } else { 70 | rollback_exit_code = 999 71 | } 72 | } else { 73 | rollback_exit_code = 0 74 | } 75 | output["task_output"] = output["task_output"].(string) + stdout.String() + stderr.String() 76 | logger.Info(" * Task rollbacked with code ", rollback_exit_code) 77 | } 78 | 79 | } else if TaskTypeID == 1 { 80 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/releem-agent -f", []string{}, logger) 81 | output["task_output"] = output["task_output"].(string) + task_output 82 | } else if TaskTypeID == 2 { 83 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/mysqlconfigurer.sh -u", []string{}, logger) 84 | output["task_output"] = output["task_output"].(string) + task_output 85 | } else if TaskTypeID == 3 { 86 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/releem-agent --task=queries_optimization", []string{}, logger) 87 | output["task_output"] = output["task_output"].(string) + task_output 88 | } else if TaskTypeID == 4 { 89 | if configuration.InstanceType == "aws/rds" { 90 | output["task_exit_code"], output["task_status"], task_output = ApplyConfAwsRds(repeaters, gatherers, logger, configuration, types.ApplyMethodImmediate) 91 | output["task_output"] = output["task_output"].(string) + task_output 92 | if output["task_exit_code"] == 0 { 93 | output["task_exit_code"], output["task_status"], task_output = ApplyConfAwsRds(repeaters, gatherers, logger, configuration, types.ApplyMethodPendingReboot) 94 | output["task_output"] = output["task_output"].(string) + task_output 95 | } 96 | } else { 97 | switch runtime.GOOS { 98 | case "windows": 99 | output["task_exit_code"] = 0 100 | output["task_status"] = 1 101 | output["task_output"] = output["task_output"].(string) + "Windows is not supported apply configuration.\n" 102 | default: // для Linux и других UNIX-подобных систем 103 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/mysqlconfigurer.sh -s automatic", []string{"RELEEM_RESTART_SERVICE=0"}, logger) 104 | output["task_output"] = output["task_output"].(string) + task_output 105 | } 106 | 107 | if output["task_exit_code"] == 0 { 108 | output["task_exit_code"], output["task_status"], task_output = ApplyConfLocal(metrics, repeaters, gatherers, logger, configuration) 109 | output["task_output"] = output["task_output"].(string) + task_output 110 | } 111 | } 112 | metrics = utils.CollectMetrics(gatherers, logger, configuration) 113 | } else if TaskTypeID == 5 { 114 | if configuration.InstanceType == "aws/rds" { 115 | output["task_exit_code"], output["task_status"], task_output = ApplyConfAwsRds(repeaters, gatherers, logger, configuration, types.ApplyMethodPendingReboot) 116 | output["task_output"] = output["task_output"].(string) + task_output 117 | } else { 118 | output["task_exit_code"], output["task_status"], task_output = execCmd(configuration.ReleemDir+"/mysqlconfigurer.sh -s automatic", []string{"RELEEM_RESTART_SERVICE=1"}, logger) 119 | output["task_output"] = output["task_output"].(string) + task_output 120 | if output["task_exit_code"] == 7 { 121 | var rollback_exit_code int 122 | cmd := exec.Command(configuration.ReleemDir+"/mysqlconfigurer.sh", "-r") 123 | cmd.Stdout = &stdout 124 | cmd.Stderr = &stderr 125 | cmd.Env = append(cmd.Environ(), "RELEEM_RESTART_SERVICE=1") 126 | err := cmd.Run() 127 | if err != nil { 128 | output["task_output"] = output["task_output"].(string) + err.Error() 129 | logger.Error(err) 130 | if exiterr, ok := err.(*exec.ExitError); ok { 131 | rollback_exit_code = exiterr.ExitCode() 132 | } else { 133 | rollback_exit_code = 999 134 | } 135 | } else { 136 | rollback_exit_code = 0 137 | } 138 | output["task_output"] = output["task_output"].(string) + stdout.String() + stderr.String() 139 | logger.Info(" * Task rollbacked with code ", rollback_exit_code) 140 | } 141 | } 142 | } 143 | logger.Info(" * Task with id - ", TaskID, " and type id - ", TaskTypeID, " completed with code ", output["task_exit_code"]) 144 | metrics.ReleemAgent.Tasks = output 145 | utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "TaskStatus", Type: ""}) 146 | 147 | } 148 | 149 | func execCmd(cmd_path string, environment []string, logger logging.Logger) (int, int, string) { 150 | var stdout, stderr bytes.Buffer 151 | var task_exit_code, task_status int 152 | var task_output string 153 | 154 | cmd := exec.Command("sh", "-c", cmd_path) 155 | cmd.Stdout = &stdout 156 | cmd.Stderr = &stderr 157 | for _, env := range environment { 158 | cmd.Env = append(cmd.Environ(), env) 159 | } 160 | err := cmd.Run() 161 | if err != nil { 162 | task_output = task_output + err.Error() 163 | logger.Error(err) 164 | if exiterr, ok := err.(*exec.ExitError); ok { 165 | task_exit_code = exiterr.ExitCode() 166 | } else { 167 | task_exit_code = 999 168 | } 169 | task_status = 4 170 | } else { 171 | task_exit_code = 0 172 | task_status = 1 173 | } 174 | task_output = task_output + stdout.String() + stderr.String() 175 | return task_exit_code, task_status, task_output 176 | } 177 | 178 | func ApplyConfLocal(metrics *models.Metrics, repeaters models.MetricsRepeater, gatherers []models.MetricsGatherer, logger logging.Logger, configuration *config.Config) (int, int, string) { 179 | var task_exit_code, task_status int 180 | var task_output string 181 | 182 | result_data := models.MetricGroupValue{} 183 | // flush_queries := []string{"flush status", "flush statistic"} 184 | need_restart := false 185 | need_privileges := false 186 | need_flush := false 187 | error_exist := false 188 | 189 | recommend_var := utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "Configurations", Type: "get-json"}) 190 | err := json.Unmarshal([]byte(recommend_var.(string)), &result_data) 191 | if err != nil { 192 | logger.Error(err) 193 | } 194 | 195 | for key := range result_data { 196 | logger.Info(key, result_data[key], metrics.DB.Conf.Variables[key]) 197 | 198 | if result_data[key] != metrics.DB.Conf.Variables[key] { 199 | query_set_var := "set global " + key + "=" + result_data[key].(string) 200 | _, err := models.DB.Exec(query_set_var) 201 | if err != nil { 202 | logger.Error(err) 203 | task_output = task_output + err.Error() 204 | if strings.Contains(err.Error(), "is a read only variable") || strings.Contains(err.Error(), "innodb_log_file_size must be at least") { 205 | need_restart = true 206 | } else if strings.Contains(err.Error(), "Access denied") { 207 | need_privileges = true 208 | } else { 209 | error_exist = true 210 | } 211 | } else { 212 | need_flush = true 213 | } 214 | } 215 | } 216 | logger.Info(need_flush, need_restart, need_privileges, error_exist) 217 | if error_exist { 218 | task_exit_code = 8 219 | task_status = 4 220 | } else { 221 | // if need_flush { 222 | // for _, query := range flush_queries { 223 | // _, err := config.DB.Exec(query) 224 | // if err != nil { 225 | // output["task_output"] = output["task_output"].(string) + err.Error() 226 | // logger.Error(err) 227 | // // if exiterr, ok := err.(*exec.ExitError); ok { 228 | // // output["task_exit_code"] = exiterr.ExitCode() 229 | // // } else { 230 | // // output["task_exit_code"] = 999 231 | // // } 232 | // } 233 | // // } else { 234 | // // output["task_exit_code"] = 0 235 | // // } 236 | // } 237 | // } 238 | if need_privileges { 239 | task_exit_code = 9 240 | task_status = 4 241 | } else if need_restart { 242 | task_exit_code = 10 243 | task_status = 1 244 | } else { 245 | task_exit_code = 0 246 | task_status = 1 247 | } 248 | } 249 | time.Sleep(10 * time.Second) 250 | 251 | return task_exit_code, task_status, task_output 252 | } 253 | 254 | func ApplyConfAwsRds(repeaters models.MetricsRepeater, gatherers []models.MetricsGatherer, 255 | logger logging.Logger, configuration *config.Config, apply_method types.ApplyMethod) (int, int, string) { 256 | 257 | var task_exit_code, task_status int = 0, 1 258 | var task_output string 259 | var paramGroup types.DBParameterGroupStatus 260 | var dbInstance types.DBInstance 261 | 262 | metrics := utils.CollectMetrics(gatherers, logger, configuration) 263 | 264 | // Загрузите конфигурацию AWS по умолчанию 265 | cfg, err := config_aws.LoadDefaultConfig(context.TODO(), config_aws.WithRegion(configuration.AwsRegion)) 266 | if err != nil { 267 | logger.Errorf("Load AWS configuration FAILED, %v", err) 268 | task_output = task_output + err.Error() 269 | } else { 270 | logger.Info("AWS configuration loaded SUCCESS") 271 | } 272 | 273 | // Создайте клиент RDS 274 | rdsclient := rds.NewFromConfig(cfg) 275 | 276 | // Prepare request to RDS 277 | input := &rds.DescribeDBInstancesInput{ 278 | DBInstanceIdentifier: &configuration.AwsRDSDB, 279 | } 280 | result, err := rdsclient.DescribeDBInstances(context.TODO(), input) 281 | if err != nil { 282 | logger.Errorf("Failed to describe DB instance: %v", err) 283 | task_output = task_output + err.Error() 284 | } 285 | 286 | // Проверяем статус инстанса и требуются ли изменения 287 | if len(result.DBInstances) > 0 { 288 | dbInstance = result.DBInstances[0] 289 | paramGroup = dbInstance.DBParameterGroups[0] 290 | } else { 291 | logger.Error("No DB instance found.") 292 | task_output = task_output + "No DB instance found.\n" 293 | } 294 | logger.Infof("DB Instance ID: %s, DB Instance Status: %s, Parameter Group Name: %s, Parameter Group Status: %s\n", *dbInstance.DBInstanceIdentifier, *dbInstance.DBInstanceStatus, *paramGroup.DBParameterGroupName, *paramGroup.ParameterApplyStatus) 295 | if aws.StringValue(dbInstance.DBInstanceStatus) != "available" { 296 | logger.Error("DB Instance Status '" + aws.StringValue(dbInstance.DBInstanceStatus) + "' not available(" + aws.StringValue(dbInstance.DBInstanceStatus) + ")") 297 | task_output = task_output + "DB Instance Status '" + aws.StringValue(dbInstance.DBInstanceStatus) + "' not available\n" 298 | task_status = 4 299 | task_exit_code = 1 300 | return task_exit_code, task_status, task_output 301 | } else if configuration.AwsRDSParameterGroup == "" || aws.StringValue(paramGroup.DBParameterGroupName) != configuration.AwsRDSParameterGroup { 302 | logger.Error("Parameter group '" + configuration.AwsRDSParameterGroup + "' not found or empty in DB Instance " + configuration.AwsRDSDB + "(" + aws.StringValue(paramGroup.DBParameterGroupName) + ")") 303 | task_output = task_output + "Parameter group '" + configuration.AwsRDSParameterGroup + "' not found or empty in DB Instance " + configuration.AwsRDSDB + "(" + aws.StringValue(paramGroup.DBParameterGroupName) + ")\n" 304 | task_status = 4 305 | task_exit_code = 3 306 | return task_exit_code, task_status, task_output 307 | } else if aws.StringValue(paramGroup.ParameterApplyStatus) != "in-sync" { 308 | logger.Error("Parameter group status '" + configuration.AwsRDSParameterGroup + "' not in-sync(" + aws.StringValue(paramGroup.ParameterApplyStatus) + ")") 309 | task_output = task_output + "Parameter group status '" + configuration.AwsRDSParameterGroup + "' not in-sync(" + aws.StringValue(paramGroup.ParameterApplyStatus) + ")\n" 310 | task_status = 4 311 | task_exit_code = 2 312 | return task_exit_code, task_status, task_output 313 | } 314 | DbParametersType := make(models.MetricGroupValue) 315 | 316 | if apply_method == types.ApplyMethodImmediate { 317 | // Вызов DescribeDBParameters для получения параметров группы 318 | input := &rds.DescribeDBParametersInput{ 319 | DBParameterGroupName: aws.String(configuration.AwsRDSParameterGroup), 320 | } 321 | 322 | // Итерируем по всем параметрам в группе и выводим ApplyType для каждого 323 | paginator := rds.NewDescribeDBParametersPaginator(rdsclient, input) 324 | for paginator.HasMorePages() { 325 | page, err := paginator.NextPage(context.TODO()) 326 | if err != nil { 327 | logger.Errorf("Failed to retrieve parameters: %v", err) 328 | task_output = task_output + err.Error() 329 | } 330 | for _, param := range page.Parameters { 331 | DbParametersType[*param.ParameterName] = *param.ApplyType 332 | } 333 | } 334 | } 335 | result_data := models.MetricGroupValue{} 336 | recommend_var := utils.ProcessRepeaters(metrics, repeaters, configuration, logger, models.ModeType{Name: "Configurations", Type: "get-json"}) 337 | err = json.Unmarshal([]byte(recommend_var.(string)), &result_data) 338 | if err != nil { 339 | logger.Error(err) 340 | task_output = task_output + err.Error() 341 | } 342 | 343 | var Parameters []types.Parameter 344 | var value string 345 | for key := range result_data { 346 | if result_data[key] != metrics.DB.Conf.Variables[key] { 347 | logger.Info(key, result_data[key], metrics.DB.Conf.Variables[key]) 348 | if key == "innodb_max_dirty_pages_pct" { 349 | i, err := strconv.ParseFloat(result_data[key].(string), 32) 350 | if err != nil { 351 | logger.Error(err) 352 | task_output = task_output + err.Error() 353 | } 354 | value = strconv.Itoa(int(i)) 355 | } else { 356 | value = result_data[key].(string) 357 | } 358 | 359 | if apply_method == types.ApplyMethodImmediate { 360 | val, ok := DbParametersType[key] 361 | if ok && val != "dynamic" { 362 | continue 363 | } 364 | } 365 | Parameters = append(Parameters, types.Parameter{ 366 | ParameterName: aws.String(key), 367 | ParameterValue: aws.String(value), 368 | ApplyMethod: apply_method, 369 | }) 370 | } 371 | if len(Parameters) == 20 { 372 | // Создайте запрос на изменение параметра в группе параметров 373 | input := &rds.ModifyDBParameterGroupInput{ 374 | DBParameterGroupName: aws.String(configuration.AwsRDSParameterGroup), 375 | Parameters: Parameters, 376 | } 377 | 378 | // Вызовите ModifyDBParameterGroup API для изменения параметра 379 | _, err := rdsclient.ModifyDBParameterGroup(context.TODO(), input) 380 | if err != nil { 381 | if strings.Contains(err.Error(), "AccessDenied") { 382 | task_exit_code = 9 383 | task_status = 4 384 | } else { 385 | task_exit_code = 8 386 | task_status = 4 387 | } 388 | logger.Errorf("Parameter group modified unsuccessfully: %v", err) 389 | task_output = task_output + err.Error() 390 | return task_exit_code, task_status, task_output 391 | } else { 392 | logger.Info("Parameter group modified successfully") 393 | } 394 | Parameters = []types.Parameter{} 395 | } 396 | } 397 | if len(Parameters) != 0 { 398 | // Создайте запрос на изменение параметра в группе параметров 399 | input := &rds.ModifyDBParameterGroupInput{ 400 | DBParameterGroupName: aws.String(configuration.AwsRDSParameterGroup), 401 | Parameters: Parameters, 402 | } 403 | 404 | // Вызовите ModifyDBParameterGroup API для изменения параметра 405 | _, err := rdsclient.ModifyDBParameterGroup(context.TODO(), input) 406 | if err != nil { 407 | if strings.Contains(err.Error(), "AccessDenied") { 408 | task_exit_code = 9 409 | task_status = 4 410 | } else { 411 | task_exit_code = 8 412 | task_status = 4 413 | } 414 | logger.Errorf("Parameter group modified unsuccessfully: %v", err) 415 | task_output = task_output + err.Error() 416 | return task_exit_code, task_status, task_output 417 | } else { 418 | logger.Info("Parameter group modified successfully") 419 | } 420 | } 421 | time.Sleep(15 * time.Second) 422 | sum := 1 423 | wait_seconds := 400 424 | for sum < wait_seconds { 425 | // Prepare request to RDS 426 | input = &rds.DescribeDBInstancesInput{ 427 | DBInstanceIdentifier: &configuration.AwsRDSDB, 428 | } 429 | result, err = rdsclient.DescribeDBInstances(context.TODO(), input) 430 | if err != nil { 431 | logger.Errorf("Failed to describe DB instance: %v", err) 432 | task_output = task_output + err.Error() 433 | } 434 | // Проверяем статус инстанса и требуются ли изменения 435 | if len(result.DBInstances) > 0 { 436 | dbInstance = result.DBInstances[0] 437 | paramGroup = dbInstance.DBParameterGroups[0] 438 | } else { 439 | logger.Error("No DB instance found.") 440 | task_output = task_output + "No DB instance found.\n" 441 | } 442 | logger.Infof("DB Instance ID: %s, DB Instance Status: %s, Parameter Group Name: %s, Parameter Group Status: %s\n", *dbInstance.DBInstanceIdentifier, *dbInstance.DBInstanceStatus, *paramGroup.DBParameterGroupName, *paramGroup.ParameterApplyStatus) 443 | 444 | if aws.StringValue(dbInstance.DBInstanceStatus) != "modifying" || aws.StringValue(paramGroup.ParameterApplyStatus) != "applying" { 445 | break 446 | } 447 | time.Sleep(3 * time.Second) 448 | sum = sum + 1 449 | } 450 | 451 | if sum >= wait_seconds && aws.StringValue(dbInstance.DBInstanceStatus) == "modifying" && aws.StringValue(paramGroup.ParameterApplyStatus) == "applying" { 452 | task_exit_code = 6 453 | task_status = 4 454 | } else if aws.StringValue(dbInstance.DBInstanceStatus) == "available" && aws.StringValue(paramGroup.ParameterApplyStatus) == "pending-reboot" { 455 | task_exit_code = 10 456 | task_status = 4 457 | } else if aws.StringValue(dbInstance.DBInstanceStatus) == "available" && aws.StringValue(paramGroup.ParameterApplyStatus) == "in-sync" { 458 | logger.Info("DB Instance Status available, Parameter Group Status in-sync, No pending modifications") 459 | } else { 460 | task_exit_code = 7 461 | task_status = 4 462 | } 463 | time.Sleep(30 * time.Second) 464 | 465 | return task_exit_code, task_status, task_output 466 | } 467 | -------------------------------------------------------------------------------- /update_releem_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NAME_CONTAINER=$1 3 | 4 | printf "`date +%Y%m%d-%H:%M:%S`\033[32m Agent Update Start! \033[0m\n" 5 | 6 | 7 | VERSION=$(docker inspect -f '{{range $i, $v := split .Config.Image ":"}}{{if eq $i 1}}{{println $v}}{{end}}{{end}}' ${NAME_CONTAINER}) 8 | NEW_VER=$(curl -s -L https://releem.s3.amazonaws.com/v2/current_version_agent) 9 | if [ "$VERSION" \< "$NEW_VER" ] 10 | then 11 | printf "`date +%Y%m%d-%H:%M:%S`\033[32m Updating script \e[31;1m%s\e[0m -> \e[32;1m%s\e[0m\n" "$VERSION" "$NEW_VER" 12 | enviroment_docker=$(docker inspect -f '{{ join .Config.Env " -e " }}' ${NAME_CONTAINER}) 13 | docker_run="docker run -d -ti --name ${NAME_CONTAINER} -e ${enviroment_docker} releem/releem-agent:${NEW_VER}" 14 | echo "$docker_run" 15 | docker rm -f $NAME_CONTAINER 16 | eval "$docker_run" 17 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m Releem Agent updated successfully.\033[0m\n" 18 | else 19 | printf "\n`date +%Y%m%d-%H:%M:%S`\033[32m Agent update is not required.\033[0m\n" 20 | fi 21 | 22 | 23 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | // Example of a daemon with echo service 2 | package utils 3 | 4 | import ( 5 | "database/sql" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/Releem/mysqlconfigurer/config" 11 | e "github.com/Releem/mysqlconfigurer/errors" 12 | "github.com/Releem/mysqlconfigurer/models" 13 | _ "github.com/go-sql-driver/mysql" 14 | logging "github.com/google/logger" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | func ProcessRepeaters(metrics *models.Metrics, repeaters models.MetricsRepeater, 19 | configuration *config.Config, logger logging.Logger, Mode models.ModeType) interface{} { 20 | defer HandlePanic(configuration, logger) 21 | 22 | result, err := repeaters.ProcessMetrics(configuration, *metrics, Mode) 23 | if err != nil { 24 | logger.Error("Repeater failed", err) 25 | } 26 | return result 27 | } 28 | 29 | func CollectMetrics(gatherers []models.MetricsGatherer, logger logging.Logger, configuration *config.Config) *models.Metrics { 30 | defer HandlePanic(configuration, logger) 31 | var metrics models.Metrics 32 | for _, g := range gatherers { 33 | err := g.GetMetrics(&metrics) 34 | if err != nil { 35 | logger.Error("Problem getting metrics from gatherer") 36 | return nil 37 | } 38 | } 39 | return &metrics 40 | } 41 | 42 | func HandlePanic(configuration *config.Config, logger logging.Logger) { 43 | if r := recover(); r != nil { 44 | err := errors.WithStack(fmt.Errorf("%v", r)) 45 | logger.Infof("%+v", err) 46 | sender := e.NewReleemErrorsRepeater(configuration, logger) 47 | sender.ProcessErrors(fmt.Sprintf("%+v", err)) 48 | } 49 | } 50 | 51 | func MapJoin(map1, map2 models.MetricGroupValue) models.MetricGroupValue { 52 | for k, v := range map2 { 53 | map1[k] = v 54 | } 55 | return map1 56 | } 57 | 58 | func IsPath(path string, logger logging.Logger) bool { 59 | result_path := strings.Index(path, "/") 60 | if result_path == 0 { 61 | return true 62 | } else { 63 | return false 64 | } 65 | } 66 | 67 | func ConnectionDatabase(configuration *config.Config, logger logging.Logger, DBname string) *sql.DB { 68 | var db *sql.DB 69 | var err error 70 | var TypeConnection, MysqlSslMode string 71 | 72 | if configuration.MysqlSslMode { 73 | MysqlSslMode = "?tls=skip-verify" 74 | } else { 75 | MysqlSslMode = "" 76 | } 77 | if IsPath(configuration.MysqlHost, logger) { 78 | db, err = sql.Open("mysql", configuration.MysqlUser+":"+configuration.MysqlPassword+"@unix("+configuration.MysqlHost+")/"+DBname+MysqlSslMode) 79 | TypeConnection = "unix" 80 | 81 | } else { 82 | db, err = sql.Open("mysql", configuration.MysqlUser+":"+configuration.MysqlPassword+"@tcp("+configuration.MysqlHost+":"+configuration.MysqlPort+")/"+DBname+MysqlSslMode) 83 | TypeConnection = "tcp" 84 | } 85 | if err != nil { 86 | logger.Error("Connection opening to failed", err) 87 | } 88 | 89 | err = db.Ping() 90 | if err != nil { 91 | logger.Error("Connection failed", err) 92 | } else { 93 | if TypeConnection == "unix" { 94 | logger.Info("Connect Success to DB ", DBname, " via unix socket ", configuration.MysqlHost) 95 | } else if TypeConnection == "tcp" { 96 | logger.Info("Connect Success to DB ", DBname, " via tcp ", configuration.MysqlHost) 97 | } 98 | } 99 | return db 100 | } 101 | 102 | func EnableEventsStatementsConsumers(configuration *config.Config, logger logging.Logger, uptime_str string) int { 103 | uptime, err := strconv.Atoi(uptime_str) 104 | if err != nil { 105 | logger.Error(err) 106 | } 107 | count_setup_consumers := 0 108 | if configuration.QueryOptimization && uptime < 120 { 109 | err := models.DB.QueryRow("SELECT count(name) FROM performance_schema.setup_consumers WHERE enabled = 'YES' AND name LIKE 'events_statements_%' AND name != 'events_statements_cpu'").Scan(&count_setup_consumers) 110 | if err != nil { 111 | logger.Error(err) 112 | count_setup_consumers = 0 113 | } 114 | logger.Info("Found enabled performance_schema statements consumers: ", count_setup_consumers) 115 | if count_setup_consumers < 3 && configuration.InstanceType == "aws/rds" { 116 | _, err := models.DB.Query("CALL releem.enable_events_statements_consumers()") 117 | if err != nil { 118 | logger.Error("Failed to enable events_statements consumers", err) 119 | } else { 120 | logger.Info("Enable events_statements_consumers") 121 | } 122 | } 123 | } 124 | return count_setup_consumers 125 | } 126 | --------------------------------------------------------------------------------