├── .gitignore ├── docs ├── result.jpg ├── DOCKER.md └── RELEASE.md ├── packaging ├── postinst ├── prerm └── graphite-ch-optimizer.service ├── Dockerfile ├── .github └── workflows │ ├── tests.yml │ ├── release.yml │ └── codeql-analysis.yml ├── nfpm.yaml ├── LICENSE ├── go.mod ├── Jenkinsfile ├── Makefile ├── go.sum ├── README.md └── graphite-ch-optimizer.go /.gitignore: -------------------------------------------------------------------------------- 1 | graphite-ch-optimizer 2 | out/ 3 | artifact/ 4 | -------------------------------------------------------------------------------- /docs/result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-graphite/graphite-ch-optimizer/HEAD/docs/result.jpg -------------------------------------------------------------------------------- /packaging/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | systemctl enable graphite-ch-optimizer.service 4 | systemctl start graphite-ch-optimizer.service 5 | -------------------------------------------------------------------------------- /packaging/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | systemctl disable graphite-ch-optimizer.service 4 | systemctl stop graphite-ch-optimizer.service 5 | -------------------------------------------------------------------------------- /packaging/graphite-ch-optimizer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Service to optimize stale GraphiteMergeTree tables 3 | After=network.target clickhouse-server.service 4 | Wants=clickhouse-server.service 5 | 6 | [Service] 7 | Type=simple 8 | ExecStart=/usr/bin/graphite-ch-optimizer 9 | Restart=on-failure 10 | User=nobody 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /docs/DOCKER.md: -------------------------------------------------------------------------------- 1 | # Docker images and tags 2 | The images `innogames/graphite-ch-optimizer` are available on docker-hub with the following tags: 3 | 4 | - `${semantic_version}` (`docker/graphite-ch-optimizer/Dockerfile`) - these tags are automatically built from tags with tag regexp `v([.0-9]+)` 5 | - `latest` (`docker/graphite-ch-optimizer/Dockerfile`) - is built automatically from the latest `master` commit 6 | - `builder` (`docker/builder/Dockerfile`) - is built automatically from the latest `master` commit 7 | -------------------------------------------------------------------------------- /docs/RELEASE.md: -------------------------------------------------------------------------------- 1 | # CREATE NEW RELEASE 2 | When the code is ready to new release, create it with the tag `v${major_version}.${minor_version}.${patch_version}`. 3 | We have a [workflow](../.github/workflows/upload-assets.yml), which will upload created DEB and RPM packages together with hash sums as the release assets. 4 | 5 | ## Use Jenkins to upload packages to deb-drop repository 6 | When the release is ready and assets are uploaded, launch the multibranch pipeline job configured against [Jenkinsfile](../Jenkinsfile) with desired version. It will download the package, compare hashsums and upload it to the repository. 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Image which contains the binary artefacts 3 | # 4 | FROM golang:alpine AS builder 5 | 6 | ENV GOPATH=/go 7 | 8 | RUN apk add make git --no-cache 9 | 10 | WORKDIR /go/src/github.com/go-graphite/graphite-ch-optimizer 11 | COPY . . 12 | RUN make -e CGO_ENABLED=0 build 13 | 14 | # 15 | # Application image 16 | # 17 | FROM alpine:latest 18 | 19 | RUN apk --no-cache add ca-certificates tzdata && mkdir /graphite-ch-optimizer 20 | 21 | WORKDIR /graphite-ch-optimizer 22 | 23 | COPY --from=builder \ 24 | /go/src/github.com/go-graphite/graphite-ch-optimizer/graphite-ch-optimizer \ 25 | /go/src/github.com/go-graphite/graphite-ch-optimizer/LICENSE \ 26 | . 27 | 28 | ENTRYPOINT ["./graphite-ch-optimizer"] 29 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Tests 3 | 4 | "on": 5 | push: 6 | branches: [master] 7 | pull_request: 8 | branches: [master] 9 | 10 | jobs: 11 | 12 | tests: 13 | name: Test code 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | go: 18 | - ^1.21 19 | - ^1.22 20 | - ^1.23 21 | - ^1 22 | steps: 23 | 24 | - name: Set up Go 1.x 25 | uses: actions/setup-go@v5 26 | with: 27 | go-version: ${{ matrix.go }} 28 | 29 | - name: Check out code into the Go module directory 30 | uses: actions/checkout@v4 31 | 32 | - name: Test 33 | run: make test 34 | 35 | - name: Build and run version 36 | run: | 37 | make VERSION=testing-version 38 | ./graphite-ch-optimizer --version 39 | -------------------------------------------------------------------------------- /nfpm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ${NAME} 3 | description: ${DESCRIPTION} 4 | 5 | # Common packages config 6 | arch: "${ARCH}" # amd64, arm64 7 | platform: "linux" 8 | version: "${VERSION_STRING}" 9 | maintainer: &m "Mikhail f. Shiryaev " 10 | vendor: *m 11 | homepage: "https://github.com/go-graphite/${NAME}" 12 | license: "MIT" 13 | section: "admin" 14 | priority: "optional" 15 | 16 | contents: 17 | - src: "packaging/${NAME}.service" 18 | dst: "/lib/systemd/system/${NAME}.service" 19 | expand: true 20 | - src: out/config.toml.example 21 | dst: /etc/${NAME}/config.toml.example 22 | type: config|noreplace 23 | expand: true 24 | - src: "out/${NAME}-linux-${ARCH}" 25 | dst: /usr/bin/${NAME} 26 | expand: true 27 | # docs 28 | - src: LICENSE 29 | dst: /usr/share/doc/${NAME}/LICENSE 30 | expand: true 31 | 32 | scripts: 33 | postinstall: packaging/postinst 34 | preremove: packaging/prerm 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019-2024 InnoGames GmbH 4 | Copyright (c) 2024 go-graphite 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-graphite/graphite-ch-optimizer 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.23.3 6 | 7 | require ( 8 | github.com/ClickHouse/clickhouse-go v1.5.4 9 | github.com/pelletier/go-toml/v2 v2.2.3 10 | github.com/sirupsen/logrus v1.9.3 11 | github.com/spf13/pflag v1.0.5 12 | github.com/spf13/viper v1.19.0 13 | ) 14 | 15 | require ( 16 | github.com/cloudflare/golz4 v0.0.0-20240916140612-caecf3c00c06 // indirect 17 | github.com/fsnotify/fsnotify v1.8.0 // indirect 18 | github.com/hashicorp/hcl v1.0.0 // indirect 19 | github.com/magiconair/properties v1.8.7 // indirect 20 | github.com/mitchellh/mapstructure v1.5.0 // indirect 21 | github.com/sagikazarmark/locafero v0.6.0 // indirect 22 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 23 | github.com/sourcegraph/conc v0.3.0 // indirect 24 | github.com/spf13/afero v1.11.0 // indirect 25 | github.com/spf13/cast v1.7.0 // indirect 26 | github.com/subosito/gotenv v1.6.0 // indirect 27 | go.uber.org/multierr v1.11.0 // indirect 28 | golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect 29 | golang.org/x/sys v0.27.0 // indirect 30 | golang.org/x/text v0.20.0 // indirect 31 | gopkg.in/ini.v1 v1.67.0 // indirect 32 | gopkg.in/yaml.v3 v3.0.1 // indirect 33 | ) 34 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/groovy 2 | library 'adminsLib@master' 3 | 4 | properties([ 5 | parameters([ 6 | string(defaultValue: '', description: 'deb-drop repository, see https://github.com/innogames/deb-drop/', name: 'REPO_NAME', trim: false), 7 | ]) 8 | ]) 9 | 10 | // Remove builds in presented status, default is ['ABORTED', 'NOT_BUILT'] 11 | jobCommon.cleanNotFinishedBuilds() 12 | 13 | node() { 14 | ansiColor('xterm') { 15 | // Checkout repo and get info about current stage 16 | sh 'echo Initial env; env | sort' 17 | env.PACKAGE_NAME = 'graphite-ch-optimizer' 18 | try { 19 | stage('Checkout') { 20 | gitSteps checkout: true, changeBuildName: false 21 | sh 'set +x; echo "Environment variables after checkout:"; env|sort' 22 | env.NEW_VERSION = sh(returnStdout: true, script: 'make version').trim() 23 | currentBuild.displayName = "${currentBuild.number}: version ${env.NEW_VERSION}" 24 | } 25 | stage('Upload to deb-drop') { 26 | when(env.GIT_BRANCH_OR_TAG == 'tag' && jobCommon.launchedByUser() && env.REPO_NAME != '') { 27 | deb_package = "graphite-ch-optimizer_${env.NEW_VERSION}_amd64.deb" 28 | [deb_package, 'md5sum', 'sha256sum'].each { file-> 29 | sh "set -ex; wget https://github.com/innogames/graphite-ch-optimizer/releases/download/${env.GIT_BRANCH}/${file}" 30 | } 31 | ['md5sum', 'sha256sum'].each { sum-> 32 | sh "set -ex; ${sum} --ignore-missing --status -c ${sum}" 33 | } 34 | withCredentials([string(credentialsId: 'DEB_DROP_TOKEN', variable: 'DebDropToken')]) { 35 | jobCommon.uploadPackage file: deb_package, repo: env.REPO_NAME, token: DebDropToken 36 | } 37 | } 38 | } 39 | cleanWs(notFailBuild: true) 40 | } 41 | catch (all) { 42 | currentBuild.result = 'FAILURE' 43 | error "Something wrong, exception is: ${all}" 44 | jobCommon.processException(all) 45 | } 46 | finally { 47 | jobCommon.postSlack() 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Create new release 3 | 'on': 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | inputs: 8 | ref: 9 | description: 'Git tag to push the image' 10 | required: true 11 | type: string 12 | jobs: 13 | docker: 14 | name: Build image 15 | runs-on: ubuntu-latest 16 | # https://docs.github.com/en/actions/use-cases-and-examples/publishing-packages/publishing-docker-images#publishing-images-to-github-packages 17 | permissions: 18 | packages: write 19 | contents: read 20 | attestations: write 21 | id-token: write 22 | 23 | steps: 24 | - name: Check out code 25 | uses: actions/checkout@v4 26 | with: 27 | ref: ${{ inputs.ref }} 28 | - name: Docker meta 29 | id: meta 30 | uses: docker/metadata-action@v5 31 | with: 32 | context: ${{ inputs.ref && 'git' || 'workflow' }} 33 | images: ghcr.io/${{ github.repository }} 34 | # create latest tag for branch events 35 | flavor: | 36 | latest=${{ inputs.ref && 'false' || 'auto' }} 37 | tags: | 38 | type=semver,pattern={{version}},value=${{inputs.ref}} 39 | type=semver,pattern={{major}}.{{minor}},value=${{inputs.ref}} 40 | type=semver,pattern={{major}}.{{minor}}.{{patch}},value=${{inputs.ref}} 41 | - name: Login to ghcr.io 42 | uses: docker/login-action@v3 43 | with: 44 | registry: ghcr.io 45 | username: ${{ github.actor }} 46 | password: ${{ secrets.GITHUB_TOKEN }} 47 | - name: Build and push 48 | id: docker_build 49 | uses: docker/build-push-action@v5 50 | with: 51 | # push for non-pr events 52 | push: ${{ github.event_name != 'pull_request' }} 53 | context: . 54 | tags: ${{ steps.meta.outputs.tags }} 55 | labels: ${{ steps.meta.outputs.labels }} 56 | 57 | build: 58 | name: Publish assets and packages 59 | runs-on: ubuntu-latest 60 | permissions: 61 | contents: write 62 | 63 | steps: 64 | - uses: actions/checkout@v4 65 | name: Checkout 66 | with: 67 | # Otherwise there's a risk to not get latest tag 68 | # We hope, that the current commit at 69 | # least 100 commits close to the latest release 70 | fetch-depth: 100 71 | fetch-tags: ${{ inputs.ref != '' }} 72 | ref: ${{ inputs.ref }} 73 | - name: Set up Go 1 74 | uses: actions/setup-go@v5 75 | with: 76 | go-version: ^1 77 | - name: Build packages 78 | id: build 79 | run: | 80 | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.40.0 81 | make -e CGO_ENABLED=0 packages 82 | - name: Upload release assets 83 | env: 84 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 85 | run: | 86 | TAG="${{ inputs.ref && inputs.ref || github.event.release.tag_name }}" 87 | gh release upload --clobber --repo ${{ github.repository }} "$TAG" \ 88 | out/*.deb out/*.rpm out/*sum 89 | - name: Upload packages to packagecloud.com 90 | env: 91 | PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} 92 | run: | 93 | go install github.com/mlafeldt/pkgcloud/cmd/pkgcloud-push@e79e9efc 94 | make packagecloud-stable 95 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME = graphite-ch-optimizer 2 | MODULE = github.com/go-graphite/$(NAME) 3 | VERSION = $(shell git describe --always --tags 2>/dev/null | sed 's:^v::; s/\([^-]*-g\)/c\1/; s|-|.|g') 4 | define DESCRIPTION = 5 | 'Service to optimize stale GraphiteMergeTree tables 6 | This software looking for tables with GraphiteMergeTree engine and evaluate if some of partitions should be optimized. It could work both as one-shot script and background daemon.' 7 | endef 8 | PKG_FILES = $(wildcard out/*$(VERSION)*.deb out/*$(VERSION)*.rpm ) 9 | SUM_FILES = out/sha256sum out/md5sum 10 | 11 | GO ?= go 12 | GO_VERSION = -ldflags "-X 'main.version=$(VERSION)'" 13 | ifeq ("$(CGO_ENABLED)", "0") 14 | GOFLAGS += -ldflags=-extldflags=-static 15 | endif 16 | export GO111MODULE := on 17 | 18 | SRCS:=$(shell find . -name '*.go') 19 | 20 | .PHONY: all clean docker test version _nfpm 21 | 22 | all: $(NAME) 23 | build: $(NAME) 24 | $(NAME): $(SRCS) 25 | $(GO) build $(GO_VERSION) -o $@ . 26 | 27 | version: 28 | @echo $(VERSION) 29 | 30 | clean: 31 | rm -rf artifact 32 | rm -rf $(NAME) 33 | rm -rf out 34 | 35 | rebuild: clean all 36 | 37 | # Run tests 38 | test: 39 | $(GO) vet $(MODULE) 40 | $(GO) test $(MODULE) 41 | 42 | static: 43 | CGO_ENABLED=0 $(MAKE) $(NAME) 44 | 45 | docker: 46 | docker build --label 'org.opencontainers.image.source=https://$(MODULE)' -t ghcr.io/go-graphite/$(NAME):latest . 47 | 48 | # we need it static 49 | .PHONY: gox-build 50 | gox-build: 51 | @CGO_ENABLED=0 $(MAKE) out/$(NAME)-linux-amd64 out/$(NAME)-linux-arm64 52 | 53 | out/$(NAME)-linux-%: $(SRCS) | out 54 | GOOS=linux GOARCH=$* $(GO) build $(GO_VERSION) -o $@ $(MODULE) 55 | 56 | out: out/done 57 | out/done: 58 | mkdir -p out/done 59 | 60 | ######################################################### 61 | # Prepare artifact directory and set outputs for upload # 62 | ######################################################### 63 | github_artifact: $(foreach art,$(PKG_FILES) $(SUM_FILES), artifact/$(notdir $(art))) 64 | 65 | artifact: 66 | mkdir $@ 67 | 68 | # Link artifact to directory with setting step output to filename 69 | artifact/%: ART=$(notdir $@) 70 | artifact/%: TYPE=$(lastword $(subst ., ,$(ART))) 71 | artifact/%: out/% | artifact 72 | cp -l $< $@ 73 | @echo '::set-output name=$(TYPE)::$(ART)' 74 | 75 | ####### 76 | # END # 77 | ####### 78 | 79 | ############# 80 | # Packaging # 81 | ############# 82 | 83 | # Prepare everything for packaging 84 | out/config.toml.example: $(NAME) | out 85 | ./$(NAME) --print-defaults > $@ 86 | 87 | .ONESHELL: 88 | nfpm: 89 | @$(MAKE) _nfpm ARCH=amd64 PACKAGER=deb 90 | @$(MAKE) _nfpm ARCH=arm64 PACKAGER=deb 91 | @$(MAKE) _nfpm ARCH=amd64 PACKAGER=rpm 92 | @$(MAKE) _nfpm ARCH=arm64 PACKAGER=rpm 93 | 94 | _nfpm: nfpm.yaml out/config.toml.example | out/done gox-build 95 | @NAME=$(NAME) DESCRIPTION=$(DESCRIPTION) ARCH=$(ARCH) VERSION_STRING=$(VERSION) nfpm package --packager $(PACKAGER) --target out/ 96 | 97 | packages: nfpm $(SUM_FILES) 98 | 99 | # md5 and sha256 sum-files for packages 100 | $(SUM_FILES): COMMAND = $(notdir $@) 101 | $(SUM_FILES): PKG_FILES_NAME = $(notdir $(PKG_FILES)) 102 | $(SUM_FILES): nfpm 103 | cd out && $(COMMAND) $(PKG_FILES_NAME) > $(COMMAND) 104 | ####### 105 | # END # 106 | ####### 107 | 108 | ############## 109 | # PUBLISHING # 110 | ############## 111 | 112 | # Use `go install github.com/mlafeldt/pkgcloud/cmd/pkgcloud-push` 113 | 114 | 115 | .ONESHELL: 116 | packagecloud-push-rpm: $(wildcard out/$(NAME)-$(VERSION)*.rpm) 117 | pkgcloud-push $(REPO)/rpm_any/rpm_any $^ || true 118 | 119 | .ONESHELL: 120 | packagecloud-push-deb: $(wildcard out/$(NAME)_$(VERSION)*.deb) 121 | pkgcloud-push $(REPO)/any/any $^ || true 122 | 123 | packagecloud-push: nfpm 124 | @$(MAKE) packagecloud-push-rpm 125 | @$(MAKE) packagecloud-push-deb 126 | 127 | packagecloud-autobuilds: 128 | $(MAKE) packagecloud-push REPO=go-graphite/autobuilds 129 | 130 | packagecloud-stable: 131 | $(MAKE) packagecloud-push REPO=go-graphite/stable 132 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL Advanced" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | branches: [ "master" ] 19 | schedule: 20 | - cron: '15 15 * * 6' 21 | 22 | jobs: 23 | analyze: 24 | name: Analyze (${{ matrix.language }}) 25 | # Runner size impacts CodeQL analysis time. To learn more, please see: 26 | # - https://gh.io/recommended-hardware-resources-for-running-codeql 27 | # - https://gh.io/supported-runners-and-hardware-resources 28 | # - https://gh.io/using-larger-runners (GitHub.com only) 29 | # Consider using larger runners or machines with greater resources for possible analysis time improvements. 30 | runs-on: 'ubuntu-latest' 31 | permissions: 32 | # required for all workflows 33 | security-events: write 34 | 35 | # required to fetch internal or private CodeQL packs 36 | packages: read 37 | 38 | # only required for workflows in private repositories 39 | actions: read 40 | contents: read 41 | 42 | strategy: 43 | fail-fast: false 44 | matrix: 45 | include: 46 | - language: go 47 | build-mode: autobuild 48 | # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' 49 | # Use `c-cpp` to analyze code written in C, C++ or both 50 | # Use 'java-kotlin' to analyze code written in Java, Kotlin or both 51 | # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both 52 | # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, 53 | # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. 54 | # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how 55 | # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages 56 | steps: 57 | - name: Checkout repository 58 | uses: actions/checkout@v4 59 | 60 | # Initializes the CodeQL tools for scanning. 61 | - name: Initialize CodeQL 62 | uses: github/codeql-action/init@v3 63 | with: 64 | languages: ${{ matrix.language }} 65 | build-mode: ${{ matrix.build-mode }} 66 | # If you wish to specify custom queries, you can do so here or in a config file. 67 | # By default, queries listed here will override any specified in a config file. 68 | # Prefix the list here with "+" to use these queries and those in the config file. 69 | 70 | # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 71 | # queries: security-extended,security-and-quality 72 | 73 | # If the analyze step fails for one of the languages you are analyzing with 74 | # "We were unable to automatically build your code", modify the matrix above 75 | # to set the build mode to "manual" for that language. Then modify this step 76 | # to build your code. 77 | # ℹ️ Command-line programs to run using the OS shell. 78 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 79 | - if: matrix.build-mode == 'manual' 80 | shell: bash 81 | run: | 82 | echo 'If you are using a "manual" build mode for one or more of the' \ 83 | 'languages you are analyzing, replace this with the commands to build' \ 84 | 'your code, for example:' 85 | echo ' make bootstrap' 86 | echo ' make release' 87 | exit 1 88 | 89 | - name: Perform CodeQL Analysis 90 | uses: github/codeql-action/analyze@v3 91 | with: 92 | category: "/language:${{matrix.language}}" 93 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= 2 | github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= 3 | github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= 4 | github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= 5 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= 6 | github.com/cloudflare/golz4 v0.0.0-20240916140612-caecf3c00c06 h1:6aQNgrBLzcUBaJHQjMk4X+jDo9rQtu5E0XNLhRV6pOk= 7 | github.com/cloudflare/golz4 v0.0.0-20240916140612-caecf3c00c06/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 11 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 13 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 14 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 15 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 16 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 17 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 18 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 19 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 20 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 21 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 22 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 23 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 24 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 25 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 26 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 27 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 28 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 29 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 30 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 31 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 32 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 33 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 34 | github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= 35 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 36 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 37 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 38 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 39 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 40 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 41 | github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= 42 | github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= 43 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 44 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 45 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 46 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 47 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 48 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 49 | github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= 50 | github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= 51 | github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= 52 | github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 53 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 54 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 55 | github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= 56 | github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= 57 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 58 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 59 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 60 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 61 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 62 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 63 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 64 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 65 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 66 | golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= 67 | golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= 68 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= 70 | golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 71 | golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= 72 | golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= 73 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 74 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 75 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 76 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 77 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 78 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 79 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 80 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![deb](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) 2 | [![rpm](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/go-graphite/stable) 3 | 4 | # Service to optimize stale GraphiteMergeTree tables 5 | When you use [GraphiteMergeTree](https://clickhouse.yandex/docs/en/operations/table_engines/graphitemergetree) in ClickHouse DBMS, it applies retention policies from `system.graphite_retentions` configuration during merge processes. Unfortunately, ClickHouse doesn't launch merges for partitions a) without active inserts or b) with only one part in. It means, that it never will watch for the actual retention scheme applied to partitions. 6 | This software looking for tables with GraphiteMergeTree engine and evaluate if some of partitions should be optimized. It could work both as one-shot script and background daemon. 7 | 8 | ## Build 9 | To build a binary just run `make`. 10 | 11 | You should have make, golang and [fpm](https://github.com/jordansissel/fpm) installed to build packages. To build packages run one of the following: 12 | 13 | ``` 14 | make packages 15 | make deb 16 | make rpm 17 | ``` 18 | 19 | ### Publish new version 20 | 21 | After checks are green for the latest commit in master, create a tag with format `v1.2.3` with 1 as major, 2 as minor and 3 as patch version. 22 | The workflow will build packages and upload them as release's assets. 23 | 24 | In the same time, the Docker Hub will build the images for the latest and `1.2.3` tags. 25 | 26 | ## Docker 27 | 28 | To build docker image locally run: 29 | `make docker` 30 | 31 | To launch the container run the following command on the host with a running ClickHouse server: 32 | `docker run --net=host --rm ghcr.io/go-graphite/graphite-ch-optimizer:latest` 33 | 34 | Other versions are available on the [packages](https://github.com/go-graphite/graphite-ch-optimizer/pkgs/container/graphite-ch-optimizer) page. 35 | 36 | ## FAQ 37 | * The `go` version 1.13 or newer is required 38 | * Daemon mode is preferable over one-shot script for the normal work 39 | * It's safe to run it on the cluster hosts 40 | * You could either run it on the one of replicated host or just over the all hosts 41 | * There are two different parameters in DSN, that should be adjusted together to fix timeouts for big partitions (month or something like this) optimizing: 42 | * `read_timeout` - `clickhouse-go` parameter. Controls the timeout between the `graphite-ch-optimizer` and ClickHouse server. 43 | * `receive_timeout` - ClickHouse parameter, used when the OPTIMIZE query is applied in the cluster. See [comment](https://github.com/ClickHouse/ClickHouse/issues/4831#issuecomment-708721042) in the issue. 44 | * `optimize_throw_if_noop=1` is not mandatory, but good to have. 45 | * The next picture demonstrates the result of running the daemon for the first time on ~3 years old GraphiteMergeTree table: 46 | example 47 | 48 | ### Details 49 | The next query is executed with some additional conditions as search for the partitions to optimize: 50 | 51 | ```sql 52 | SELECT 53 | concat(p.database, '.', p.table) AS table, 54 | p.partition_id AS partition_id, 55 | p.partition AS partition, 56 | max(g.age) AS age, 57 | countDistinct(p.name) AS parts, 58 | toDateTime(max(p.max_date + 1)) AS max_time, 59 | max_time + age AS rollup_time, 60 | min(p.modification_time) AS modified_at 61 | FROM system.parts AS p 62 | INNER JOIN 63 | ( 64 | SELECT 65 | Tables.database AS database, 66 | Tables.table AS table, 67 | age 68 | FROM system.graphite_retentions 69 | ARRAY JOIN Tables 70 | GROUP BY 71 | database, 72 | table, 73 | age 74 | ) AS g ON (p.table = g.table) AND (p.database = g.database) 75 | WHERE p.active AND ((toDateTime(p.max_date + 1) + g.age) < now()) 76 | GROUP BY 77 | table, 78 | partition, 79 | partition_id 80 | HAVING (modified_at < rollup_time) OR (parts > 1) 81 | ORDER BY 82 | table ASC, 83 | partition ASC, 84 | age ASC 85 | ``` 86 | 87 | #### The next queries could be executed before and after the daemon running 88 | 89 | * Detailed info about each partition of GraphiteMergeTree tables: 90 | 91 | ```sql 92 | SELECT 93 | database, 94 | table, 95 | count() AS parts, 96 | active, 97 | partition, 98 | min(min_date) AS min_date, 99 | max(max_date) AS max_date, 100 | formatReadableSize(sum(bytes_on_disk)) AS size, 101 | sum(rows) AS rows 102 | FROM system.parts AS p 103 | INNER JOIN 104 | ( 105 | SELECT 106 | Tables.database AS database, 107 | Tables.table AS table 108 | FROM system.graphite_retentions 109 | ARRAY JOIN Tables 110 | GROUP BY 111 | database, 112 | table 113 | ) AS g USING (database, table) 114 | GROUP BY 115 | database, 116 | table, 117 | partition, 118 | active 119 | ORDER BY 120 | database, 121 | table, 122 | partition, 123 | active ASC 124 | ``` 125 | 126 | * Summary about each GraphiteMergeTree table: 127 | 128 | ```sql 129 | SELECT 130 | database, 131 | table, 132 | count() AS parts, 133 | active, 134 | min(min_date) AS min_date, 135 | max(max_date) AS max_date, 136 | formatReadableSize(sum(bytes_on_disk)) AS size, 137 | sum(rows) AS rows 138 | FROM system.parts AS p 139 | INNER JOIN 140 | ( 141 | SELECT 142 | Tables.database AS database, 143 | Tables.table AS table 144 | FROM system.graphite_retentions 145 | ARRAY JOIN Tables 146 | GROUP BY 147 | database, 148 | table 149 | ) AS g USING (database, table) 150 | GROUP BY 151 | database, 152 | table, 153 | active 154 | ORDER BY 155 | database ASC, 156 | table ASC, 157 | active ASC 158 | ``` 159 | 160 | They will show general info about every GraphiteMergeTree table on the server. 161 | 162 | ## Run the graphite-ch-optimizer 163 | If you run the ClickHouse locally, you could just run `graphite-ch-optimizer -n --log-level debug` and see how many partitions on the instance are able to be merged automatically. 164 | 165 | Default config: 166 | 167 | ```toml 168 | [clickhouse] 169 | optimize-interval = "72h0m0s" 170 | server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600" 171 | 172 | [daemon] 173 | dry-run = false 174 | loop-interval = "1h0m0s" 175 | one-shot = false 176 | 177 | [logging] 178 | log-level = "info" 179 | output = "-" 180 | ``` 181 | 182 | Possible command line arguments: 183 | 184 | ``` 185 | Usage of graphite-ch-optimizer: 186 | -c, --config string Filename of the custom config. CLI arguments override it (default "/etc/graphite-ch-optimizer/config.toml") 187 | --print-defaults Print default config values and exit 188 | -v, --version Print version and exit 189 | --optimize-interval duration The partition will be merged after having no writes for more than the given duration (default 72h0m0s) 190 | -s, --server-dsn string DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600") 191 | -n, --dry-run Will print how many partitions would be merged without actions 192 | --loop-interval duration Daemon will check if there partitions to merge once per this interval (default 1h0m0s) 193 | --one-shot Program will make only one optimization instead of working in the loop (true if dry-run) 194 | --log-level string Valid options are: panic, fatal, error, warn, warning, info, debug, trace 195 | --output string The logs file. '-' is accepted as STDOUT (default "-") 196 | 197 | Version: version-string 198 | ``` 199 | -------------------------------------------------------------------------------- /graphite-ch-optimizer.go: -------------------------------------------------------------------------------- 1 | // Package main provides the watcher for the in time merged partitions 2 | // Copyright (C) 2019 InnoGames GmbH 3 | package main 4 | 5 | import ( 6 | "database/sql" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/ClickHouse/clickhouse-go" 17 | "github.com/pelletier/go-toml/v2" 18 | "github.com/sirupsen/logrus" 19 | "github.com/spf13/pflag" 20 | "github.com/spf13/viper" 21 | ) 22 | 23 | var version = "development" 24 | 25 | // SelectUnmerged is the query to create the temporary table with 26 | // partitions and the retention age, which should be applied. 27 | // Table name should be with backquotes to be able to OPTIMIZE `database`.`.inner.table` 28 | // for MaterializedView engines 29 | const SelectUnmerged = ` 30 | SELECT 31 | concat(` + "'`', p.database, '`.`', p.table, '`'" + `) AS table, 32 | p.partition_id AS partition_id, 33 | p.partition AS partition_name, 34 | max(g.age) AS age, 35 | countDistinct(p.name) AS parts, 36 | toDateTime(max(p.max_date + 1)) AS max_time, 37 | max_time + age AS rollup_time, 38 | min(p.modification_time) AS modified_at 39 | FROM system.parts AS p 40 | INNER JOIN 41 | ( 42 | SELECT 43 | Tables.database AS database, 44 | Tables.table AS table, 45 | age 46 | FROM system.graphite_retentions 47 | ARRAY JOIN Tables 48 | GROUP BY 49 | database, 50 | table, 51 | age 52 | ) AS g ON (p.table = g.table) AND (p.database = g.database) 53 | -- toDateTime(p.max_date + 1) + g.age AS unaggregated rollup_time 54 | WHERE p.active AND ((toDateTime(p.max_date + 1) + g.age) < now()) 55 | GROUP BY 56 | table, 57 | partition_name, 58 | partition_id 59 | -- modified_at < rollup_time: the merge has not been applied for the current retention policy 60 | -- parts > 1: merge should be applied because of new parts 61 | -- modified_at < (now() - @Interval): we want to merge active partitions only once per interval, 62 | -- so do not touch partitions with current active inserts 63 | HAVING ((modified_at < rollup_time) OR (parts > 1)) 64 | AND (modified_at < (now() - @Interval)) 65 | ORDER BY 66 | table ASC, 67 | partition_name ASC, 68 | age ASC 69 | ` 70 | 71 | type merge struct { 72 | table string 73 | partitionID string 74 | partitionName string 75 | } 76 | 77 | type clickHouse struct { 78 | ServerDsn string `mapstructure:"server-dsn" toml:"server-dsn"` 79 | OptimizeInterval time.Duration `mapstructure:"optimize-interval" toml:"optimize-interval"` 80 | connect *sql.DB 81 | } 82 | 83 | type daemon struct { 84 | LoopInterval time.Duration `mapstructure:"loop-interval" toml:"loop-interval"` 85 | OneShot bool `mapstructure:"one-shot" toml:"one-shot"` 86 | DryRun bool `mapstructure:"dry-run" toml:"dry-run"` 87 | } 88 | 89 | type logging struct { 90 | // List of files to write. '-' is token as os.Stdout 91 | Output string `mapstructure:"output" toml:"output"` 92 | Level string `mapstructure:"log-level" toml:"level"` 93 | } 94 | 95 | // Config for the graphite-ch-optimizer binary 96 | type Config struct { 97 | ClickHouse clickHouse `mapstructure:"clickhouse" toml:"clickhouse"` 98 | Daemon daemon `mapstructure:"daemon" toml:"daemon"` 99 | Logging logging `mapstructure:"logging" toml:"logging"` 100 | } 101 | 102 | var cfg Config 103 | 104 | func init() { 105 | var err error 106 | cfg = getConfig() 107 | 108 | // Set logging 109 | formatter := logrus.TextFormatter{ 110 | TimestampFormat: "2006-01-02 15:04:05 MST", 111 | FullTimestamp: true, 112 | } 113 | logrus.SetFormatter(&formatter) 114 | var output io.Writer 115 | switch cfg.Logging.Output { 116 | case "-": 117 | output = os.Stdout 118 | default: 119 | output, err = os.OpenFile(cfg.Logging.Output, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 120 | if err != nil { 121 | logrus.Fatalf("Unable to open file %s for writing: %v", cfg.Logging.Output, err) 122 | } 123 | } 124 | logrus.SetOutput(output) 125 | clickhouse.SetLogOutput(output) 126 | level, err := logrus.ParseLevel(cfg.Logging.Level) 127 | if err != nil { 128 | logrus.Fatalf("Fail to parse log level: %v", err) 129 | } 130 | logrus.SetLevel(level) 131 | 132 | configString, err := toml.Marshal(cfg) 133 | if err != nil { 134 | logrus.Fatalf("Failed to marshal TOML config: %v", err) 135 | } 136 | logrus.Tracef("The config is:\n%v", string(configString)) 137 | } 138 | 139 | // setDefaultConfig sets default config parameters 140 | func setDefaultConfig() { 141 | viper.SetDefault("clickhouse", map[string]interface{}{ 142 | // See ClickHouse documentation for further options. 143 | // As well, take a look into README.md to see the difference between different timeout arguments, 144 | // and why both of them are necessary. 145 | "server-dsn": "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600", 146 | // Ignore partitions which were merged less than 3 days before 147 | "optimize-interval": time.Duration(72) * time.Hour, 148 | }) 149 | viper.SetDefault("daemon", map[string]interface{}{ 150 | "one-shot": false, 151 | "loop-interval": time.Duration(1) * time.Hour, 152 | "dry-run": false, 153 | }) 154 | viper.SetDefault("logging", map[string]interface{}{ 155 | "output": "-", 156 | "log-level": "info", 157 | }) 158 | } 159 | 160 | func processFlags() error { 161 | // Parse command line arguments in differend flag groups 162 | pflag.CommandLine.SortFlags = false 163 | defaultConfig := "/etc/" + filepath.Base(os.Args[0]) + "/config.toml" 164 | customConfig := pflag.StringP("config", "c", defaultConfig, "Filename of the custom config. CLI arguments override it") 165 | pflag.Bool("print-defaults", false, "Print default config values and exit") 166 | pflag.BoolP("version", "v", false, "Print version and exit") 167 | 168 | // ClickHouse set 169 | fc := pflag.NewFlagSet("clickhouse", 0) 170 | fc.StringP("server-dsn", "s", viper.GetString("clickhouse.server-dsn"), "DSN to connect to ClickHouse server") 171 | fc.Duration("optimize-interval", viper.GetDuration("clickhouse.optimize-interval"), "The partition will be merged after having no writes for more than the given duration") 172 | // Daemon set 173 | fd := pflag.NewFlagSet("daemon", 0) 174 | fd.Bool("one-shot", viper.GetBool("daemon.one-shot"), "Program will make only one optimization instead of working in the loop (true if dry-run)") 175 | fd.Duration("loop-interval", viper.GetDuration("daemon.loop-interval"), "Daemon will check if there partitions to merge once per this interval") 176 | fd.BoolP("dry-run", "n", viper.GetBool("daemon.dry-run"), "Will print how many partitions would be merged without actions") 177 | // Logging set 178 | fl := pflag.NewFlagSet("logging", 0) 179 | fl.String("output", viper.GetString("logging.output"), "The logs file. '-' is accepted as STDOUT") 180 | fl.String("log-level", viper.GetString("logging.level"), "Valid options are: panic, fatal, error, warn, warning, info, debug, trace") 181 | 182 | pflag.CommandLine.AddFlagSet(fc) 183 | pflag.CommandLine.AddFlagSet(fd) 184 | pflag.CommandLine.AddFlagSet(fl) 185 | 186 | pflag.ErrHelp = fmt.Errorf("\nVersion: %s", version) 187 | pflag.Parse() 188 | // We must read config files before the setting of the config config to flags' values 189 | err := readConfigFile(*customConfig) 190 | if err != nil { 191 | return err 192 | } 193 | 194 | // Parse flag groups into viper config 195 | fc.VisitAll(func(f *pflag.Flag) { 196 | if err := viper.BindPFlag("clickhouse."+f.Name, f); err != nil { 197 | logrus.Fatalf("Failed to bind key clickhouse.%s: %v", f.Name, err) 198 | } 199 | }) 200 | fd.VisitAll(func(f *pflag.Flag) { 201 | if err := viper.BindPFlag("daemon."+f.Name, f); err != nil { 202 | logrus.Fatalf("Failed to bind key daemon.%s: %v", f.Name, err) 203 | } 204 | }) 205 | fl.VisitAll(func(f *pflag.Flag) { 206 | if err := viper.BindPFlag("logging."+f.Name, f); err != nil { 207 | logrus.Fatalf("Failed to bind key logging.%s: %v", f.Name, err) 208 | } 209 | }) 210 | 211 | // If it's dry run, then it should be done only once 212 | if viper.GetBool("daemon.dry-run") { 213 | viper.Set("daemon.one-shot", true) 214 | } 215 | 216 | return nil 217 | } 218 | 219 | // readConfigFile set file as the config name if it's not empty and reads the config from Viper.configPaths 220 | func readConfigFile(file string) error { 221 | var cfgNotFound viper.ConfigFileNotFoundError 222 | var perr *os.PathError 223 | viper.SetConfigFile(file) 224 | err := viper.ReadInConfig() 225 | if err != nil { 226 | if errors.As(err, &cfgNotFound) || errors.As(err, &perr) { 227 | logrus.Debug("No config files were found, use defaults and flags") 228 | return nil 229 | } 230 | return fmt.Errorf("Failed to read viper config: %w", err) 231 | } 232 | return nil 233 | } 234 | 235 | func getConfig() Config { 236 | viper.SetConfigName("config") 237 | viper.SetConfigType("toml") 238 | exeName := filepath.Base(os.Args[0]) 239 | 240 | // Set config files 241 | if userConfig, err := os.UserConfigDir(); err == nil { 242 | viper.AddConfigPath(filepath.Join(userConfig, exeName)) 243 | } 244 | viper.AddConfigPath(filepath.Join("/etc", exeName)) 245 | 246 | setDefaultConfig() 247 | defaultConfig := viper.AllSettings() 248 | 249 | err := processFlags() 250 | if err != nil { 251 | logrus.Fatalf("Failed to process flags: %v", err) 252 | } 253 | 254 | // Prints version and exit 255 | printVersion, err := pflag.CommandLine.GetBool("version") 256 | if err != nil { 257 | logrus.Fatal("Can't get '--version' value") 258 | } 259 | if printVersion { 260 | fmt.Println(version) 261 | os.Exit(0) 262 | } 263 | 264 | // Prints default config and exits 265 | printDefaults, err := pflag.CommandLine.GetBool("print-defaults") 266 | if err != nil { 267 | logrus.Fatal("Can't get '--print-defaults' value") 268 | } 269 | if printDefaults { 270 | t := new(strings.Builder) 271 | encoder := toml.NewEncoder(t) 272 | encoder.SetIndentSymbol(" ") 273 | encoder.SetIndentTables(true) 274 | err := encoder.Encode(defaultConfig) 275 | if err != nil { 276 | logrus.Fatal(err) 277 | } 278 | fmt.Println(t.String()) 279 | os.Exit(0) 280 | } 281 | 282 | c := Config{} 283 | if err := viper.Unmarshal(&c); err != nil { 284 | logrus.Fatalf("Failed to unmarshal config: %v", err) 285 | } 286 | return c 287 | } 288 | 289 | func main() { 290 | if cfg.Daemon.OneShot { 291 | err := optimize() 292 | if err != nil { 293 | logrus.Fatalf("Optimization failed: %s", err) 294 | } 295 | os.Exit(0) 296 | } 297 | 298 | go func() { 299 | logrus.Trace("Starting loop function") 300 | for { 301 | err := optimize() 302 | if err != nil { 303 | logrus.Errorf("Optimization failed: %s", err) 304 | } 305 | logrus.Infof("Optimizations round is over, going to sleep for %v", cfg.Daemon.LoopInterval) 306 | time.Sleep(cfg.Daemon.LoopInterval) 307 | } 308 | }() 309 | 310 | var wg sync.WaitGroup 311 | wg.Add(1) 312 | wg.Wait() 313 | } 314 | 315 | func optimize() error { 316 | // Getting connection pool and check it for work 317 | connect, err := sql.Open("clickhouse", cfg.ClickHouse.ServerDsn) 318 | if err != nil { 319 | logrus.Fatalf("Failed to open connection to %s: %v ClickHouse", cfg.ClickHouse.ServerDsn, err) 320 | } 321 | cfg.ClickHouse.connect = connect 322 | defer connect.Close() 323 | err = connect.Ping() 324 | if checkErr(err) != nil { 325 | logrus.Fatalf("Ping ClickHouse server failed: %v", err) 326 | } 327 | 328 | // Getting the rows with tables and partitions to optimize 329 | rows, err := connect.Query( 330 | SelectUnmerged, 331 | sql.Named("Interval", cfg.ClickHouse.OptimizeInterval.Seconds()), 332 | ) 333 | if checkErr(err) != nil { 334 | return err 335 | } 336 | 337 | merges := []merge{} 338 | var ( 339 | age uint64 340 | parts uint64 341 | maxTime time.Time 342 | rollupTime time.Time 343 | modifiedAt time.Time 344 | ) 345 | 346 | // Parse the data from DB into `merges` 347 | for rows.Next() { 348 | var m merge 349 | err = rows.Scan(&m.table, &m.partitionID, &m.partitionName, &age, &parts, &maxTime, &rollupTime, &modifiedAt) 350 | if checkErr(err) != nil { 351 | return err 352 | } 353 | merges = append(merges, m) 354 | logrus.WithFields(logrus.Fields{ 355 | "table": m.table, 356 | "partition_id": m.partitionID, 357 | "partition_name": m.partitionName, 358 | "age": age, 359 | "parts": parts, 360 | "max_time": maxTime, 361 | "rollup_time": rollupTime, 362 | "modified_at": modifiedAt, 363 | }).Debug("Merge to be applied") 364 | } 365 | 366 | if cfg.Daemon.DryRun { 367 | logrus.Infof("DRY RUN. Merges would be applied: %d", len(merges)) 368 | return nil 369 | } 370 | logrus.Infof("Merges will be applied: %d", len(merges)) 371 | 372 | for _, m := range merges { 373 | m := m 374 | err = applyMerge(&m) 375 | if checkErr(err) != nil { 376 | return err 377 | } 378 | } 379 | return nil 380 | } 381 | 382 | func applyMerge(m *merge) error { 383 | logrus.Infof("Going to merge TABLE %s PARTITION %s", m.table, m.partitionName) 384 | _, err := cfg.ClickHouse.connect.Exec( 385 | fmt.Sprintf( 386 | "OPTIMIZE TABLE %s PARTITION ID '%s' FINAL", 387 | m.table, 388 | m.partitionID, 389 | ), 390 | ) 391 | if err == nil { 392 | return nil 393 | } 394 | 395 | var chExc *clickhouse.Exception 396 | if errors.As(err, &chExc) && chExc.Code == 388 && strings.Contains(chExc.Message, "has already been assigned a merge into") { 397 | logrus.WithFields(logrus.Fields{ 398 | "table": m.table, 399 | "partition_name": m.partitionName, 400 | }).Info("The partition is already merging:") 401 | return nil 402 | } 403 | return fmt.Errorf("Fail to merge partition %v: %w", m.partitionName, checkErr(err)) 404 | } 405 | 406 | func checkErr(err error) error { 407 | var chExc *clickhouse.Exception 408 | if err == nil { 409 | return nil 410 | } 411 | if !errors.As(err, &chExc) { 412 | logrus.Errorf("Fail: %v", err) 413 | return err 414 | } 415 | logrus.Errorf( 416 | "[%d] %s \n%s\n", 417 | chExc.Code, 418 | chExc.Message, 419 | chExc.StackTrace, 420 | ) 421 | return err 422 | } 423 | --------------------------------------------------------------------------------