├── .codecov.yml ├── .editorconfig ├── .editorconfig-checker.json ├── .gitattributes ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.yaml │ ├── config.yml │ ├── enhancement.yaml │ └── question.yaml ├── PULL_REQUEST_TEMPLATE.md ├── release.yml └── workflows │ ├── ci.yaml │ ├── pr-check.yaml │ ├── stale.yaml │ ├── tag.yaml │ └── wiki.yaml ├── .gitignore ├── .golangci.yaml ├── .goreleaser.yaml ├── .idea ├── dictionaries │ └── project.xml ├── go.imports.xml ├── inspectionProfiles │ └── Project_Default.xml └── vcs.xml ├── AGENTS.md ├── CODE_OF_CONDUCT.md ├── DEVELOPER.md ├── Dockerfile ├── LICENSE.txt ├── Makefile ├── README.md ├── cmd └── access-log-exporter │ ├── it_test.go │ ├── main.go │ └── main_test.go ├── contrib └── grafana-dashboard.json ├── docs ├── Configuration.md ├── Grafana.md ├── Home.md ├── Installation.md ├── Prometheus.md ├── README.md ├── Webserver.md ├── _Footer.md └── demo │ ├── docker-compose.yaml │ ├── grafana │ ├── dashboards.yaml │ └── datasource.yaml │ ├── load-generator │ └── main.go │ ├── nginx.conf │ └── prometheus │ └── prometheus.yml ├── go.mod ├── go.sum ├── internal ├── collector │ ├── collector.go │ ├── line.go │ └── types.go ├── config │ ├── config.go │ ├── config_test.go │ ├── defaults.go │ ├── env_test.go │ ├── errors.go │ ├── flags.go │ ├── types.go │ ├── types │ │ ├── slice.go │ │ ├── slice_test.go │ │ ├── url.go │ │ └── url_test.go │ ├── validate.go │ └── validate_test.go ├── metric │ ├── metric.go │ ├── metric_bench_test.go │ ├── metric_test.go │ └── types.go ├── nginx │ ├── collector.go │ └── collector_test.go ├── syslog │ ├── syslog.go │ ├── syslog_bench_test.go │ └── syslog_test.go └── useragent │ └── useragent.go ├── packaging ├── apt │ └── access-log-exporter.sources ├── etc │ └── access-log-exporter │ │ └── config.yaml ├── scripts │ ├── postinst.sh │ ├── postremove.sh │ ├── preinst.sh │ └── preremove.sh └── usr │ └── lib │ ├── systemd │ └── system │ │ └── access-log-exporter.service │ └── sysusers.d │ └── access-log-exporter.conf └── renovate.json /.codecov.yml: -------------------------------------------------------------------------------- 1 | --- 2 | coverage: 3 | status: 4 | project: false 5 | patch: off 6 | 7 | ignore: 8 | - "docs/**/*" 9 | - "tests/**/*" 10 | - "pkg/**/*" 11 | - "wiki/**/*" 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | insert_final_newline = true 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | 9 | [{Makefile,*.go,go.mod,go.sum}] 10 | indent_style = tab 11 | indent_size = tab 12 | max_line_length = 160 13 | 14 | [*_test.go] 15 | max_line_length = off 16 | 17 | [*.{yml,yaml}] 18 | indent_size = 2 19 | 20 | [*.sh] 21 | indent_size = 2 22 | 23 | [{*.md,*.css}] 24 | indent_size = 2 25 | 26 | [.run/*.xml] 27 | insert_final_newline = false 28 | -------------------------------------------------------------------------------- /.editorconfig-checker.json: -------------------------------------------------------------------------------- 1 | { 2 | "Exclude": ["\\.go$","\\.md$","\\.idea\\.*"] 3 | } 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | Makefile text eol=lf 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: jkroepke 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yaml: -------------------------------------------------------------------------------- 1 | name: 🐞 Bug 2 | description: Something is not working as indended. 3 | labels: [ 🐞 bug ] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thanks for taking the time to fill out this bug report! 9 | 10 | Before you submit this issue, please make sure you have read the documentation and searched for similar issues. 11 | If you have done that, please fill out the template below. 12 | 13 | - type: textarea 14 | attributes: 15 | label: Current Behavior 16 | description: A concise description of what you're experiencing. 17 | placeholder: | 18 | When I do , happens and I see the error message attached below: 19 | ```...``` 20 | validations: 21 | required: true 22 | 23 | - type: textarea 24 | attributes: 25 | label: Expected Behavior 26 | description: A concise description of what you expected to happen. 27 | placeholder: When I do , should happen instead. 28 | validations: 29 | required: true 30 | 31 | - type: textarea 32 | attributes: 33 | label: Steps To Reproduce 34 | description: Steps to reproduce the behavior. 35 | placeholder: | 36 | 1. In this environment... 37 | 2. With this config... 38 | 3. Run '...' 39 | 4. See error... 40 | render: Markdown 41 | validations: 42 | required: false 43 | 44 | - type: textarea 45 | attributes: 46 | label: Environment 47 | description: | 48 | examples: 49 | - **access-log-exporter Version**: 1.5.1 50 | value: | 51 | - access-log-exporter Version: 52 | validations: 53 | required: true 54 | 55 | 56 | - type: textarea 57 | attributes: 58 | label: Anything else? 59 | description: | 60 | Links? References? Anything that will give us more context about the issue you are encountering! 61 | 62 | Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. 63 | validations: 64 | required: false 65 | 66 | - type: checkboxes 67 | id: documentation 68 | attributes: 69 | label: Preflight Checklist 70 | options: 71 | - required: true 72 | #language=markdown 73 | label: | 74 | I could not find a solution in the [documentation](https://github.com/jkroepke/access-log-exporter/wiki), 75 | the [FAQ](https://github.com/jkroepke/access-log-exporter/wiki/FAQ), the existing issues or discussions. 76 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.yaml: -------------------------------------------------------------------------------- 1 | name: ✨ Enhancement / Feature / Task 2 | description: Some feature is missing or incomplete. 3 | labels: [ ✨ enhancement ] 4 | body: 5 | - type: textarea 6 | attributes: 7 | label: Problem Statement 8 | description: Without specifying a solution, describe what the project is missing today. 9 | placeholder: | 10 | The rotating project logo has a fixed size and color. 11 | There is no way to make it larger and more shiny. 12 | validations: 13 | required: false 14 | - type: textarea 15 | attributes: 16 | label: Proposed Solution 17 | description: Describe the proposed solution to the problem above. 18 | placeholder: | 19 | - Implement 2 new flags CLI: ```--logo-color=FFD700``` and ```--logo-size=100``` 20 | - Let these flags control the size of the rotating project logo. 21 | validations: 22 | required: false 23 | - type: textarea 24 | attributes: 25 | label: Additional information 26 | placeholder: | 27 | We considered adjusting the logo size to the phase of the moon, but there was no 28 | reliable data source in air-gapped environments. 29 | validations: 30 | required: false 31 | - type: textarea 32 | attributes: 33 | label: Acceptance Criteria 34 | placeholder: | 35 | - [ ] As a user, I can control the size of the rotating logo using a CLI flag. 36 | - [ ] As a user, I can control the color of the rotating logo using a CLI flag. 37 | - [ ] Defaults are reasonably set. 38 | - [ ] New settings are appropriately documented. 39 | - [ ] No breaking change for current users of the rotating logo feature. 40 | validations: 41 | required: false 42 | 43 | - type: checkboxes 44 | id: documentation 45 | attributes: 46 | label: Preflight Checklist 47 | options: 48 | - required: true 49 | #language=markdown 50 | label: | 51 | I could not find a solution in the [documentation](https://github.com/jkroepke/access-log-exporter/wiki), 52 | the [FAQ](https://github.com/jkroepke/access-log-exporter/wiki/FAQ), the existing issues or discussions. 53 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.yaml: -------------------------------------------------------------------------------- 1 | name: ❓ Question 2 | description: Something is not clear. 3 | labels: [ ❓ question ] 4 | body: 5 | - type: textarea 6 | attributes: 7 | label: Problem Statement 8 | description: Without specifying a solution, describe what the project is missing today. 9 | placeholder: | 10 | The rotating project logo has a fixed size and color. 11 | There is no way to make it larger and more shiny. 12 | validations: 13 | required: false 14 | 15 | - type: textarea 16 | attributes: 17 | label: Environment 18 | description: | 19 | examples: 20 | - **access-log-exporter Version**: 1.5.1 21 | value: | 22 | - access-log-exporter Version: 23 | validations: 24 | required: false 25 | 26 | - type: checkboxes 27 | id: documentation 28 | attributes: 29 | label: Preflight Checklist 30 | options: 31 | - required: true 32 | #language=markdown 33 | label: | 34 | I could not find a solution in the [documentation](https://github.com/jkroepke/access-log-exporter/wiki), 35 | the [FAQ](https://github.com/jkroepke/access-log-exporter/wiki/FAQ), the existing issues or discussions. 36 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### What this PR does / why we need it 2 | 3 | #### Which issue this PR fixes 4 | 5 | *(optional, in `fixes #(, fixes #, ...)` format, will close that issue when PR gets merged)*: fixes # 6 | 7 | - fixes # 8 | 9 | #### Special notes for your reviewer 10 | 11 | #### Particularly user-facing changes 12 | 13 | #### Checklist 14 | 15 | Complete these before marking the PR as `ready to review`: 16 | 17 | 18 | 19 | - [ ] [DCO](https://github.com/prometheus-community/helm-charts/blob/main/CONTRIBUTING.md#sign-off-your-work) signed 20 | - [ ] The PR title has a summary of the changes 21 | - [ ] The PR body has a summary to reflect any significant (and particularly user-facing) changes introduced by this PR 22 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | labels: 4 | - chore 5 | categories: 6 | - title: 💥 Breaking Changes 7 | labels: 8 | - 💥 breaking-change 9 | - title: ✨ Exciting New Features 10 | labels: 11 | - ✨ enhancement 12 | - title: 🐞 Bug Fixes 13 | labels: 14 | - 🐞 bug 15 | - title: 🛠️ Dependencies 16 | labels: 17 | - 🛠️ dependencies 18 | - title: 📖 Documentation 19 | labels: 20 | - 📖 docs 21 | - title: Other Changes 22 | labels: 23 | - "*" 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | tags: 8 | - 'v*' 9 | branches: 10 | - main 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-24.04 15 | name: Build & Test 16 | steps: 17 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 18 | 19 | - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 20 | with: 21 | go-version-file: 'go.mod' 22 | cache: "${{ github.actor != 'renovate[bot]' && github.actor != 'mend[bot]' }}" 23 | 24 | - run: go build ./cmd/access-log-exporter 25 | - run: go test ./... -timeout 20s -race -covermode=atomic -coverprofile=coverage.out -coverpkg=./... 26 | - run: go test ./... -timeout 20s -run='^$' -bench=. -benchmem -count 3 27 | 28 | - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 29 | env: 30 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 31 | 32 | goreleaser: 33 | runs-on: ubuntu-24.04 34 | name: Test goreleaser 35 | steps: 36 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 37 | with: 38 | fetch-depth: 0 39 | 40 | - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 41 | with: 42 | go-version-file: 'go.mod' 43 | cache: "${{ github.actor != 'renovate[bot]' && github.actor != 'mend[bot]' }}" 44 | 45 | - uses: anchore/sbom-action/download-syft@f8bdd1d8ac5e901a77a92f111440fdb1b593736b # v0.20.6 46 | 47 | - name: Write gpg sign key 48 | if: env.GPG_KEY != null 49 | run: echo "$GPG_KEY" > "$GPG_KEY_PATH" 50 | env: 51 | GPG_KEY_PATH: "${{ secrets.GPG_KEY_PATH }}" 52 | GPG_KEY: ${{ secrets.GPG_KEY }} 53 | 54 | - name: Set up Docker Buildx 55 | uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 56 | 57 | - name: go build (with goreleaser) 58 | uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 59 | with: 60 | # renovate: github=goreleaser/goreleaser 61 | version: v2.12.5 62 | args: release --snapshot 63 | env: 64 | GITHUB_TOKEN: "" 65 | GPG_KEY_PATH: "" 66 | 67 | - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 68 | with: 69 | name: dists 70 | path: dist/ 71 | lint: 72 | name: lint 73 | runs-on: ubuntu-24.04 74 | steps: 75 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 76 | - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 77 | with: 78 | go-version-file: 'go.mod' 79 | cache: "${{ github.actor != 'renovate[bot]' && github.actor != 'mend[bot]' }}" 80 | 81 | - run: go mod tidy -diff 82 | 83 | - name: golangci-lint 84 | uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 85 | with: 86 | # renovate: github=golangci/golangci-lint 87 | version: v2.5.0 88 | args: "--max-same-issues=0" 89 | 90 | super-lint: 91 | name: super-lint 92 | runs-on: ubuntu-24.04 93 | permissions: 94 | contents: read 95 | steps: 96 | - name: Checkout Code 97 | uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 98 | with: 99 | fetch-depth: 0 100 | 101 | - name: Lint Code Base 102 | uses: super-linter/super-linter/slim@7bba2eeb89d01dc9bfd93c497477a57e72c83240 # v8.2.0 103 | env: 104 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 105 | MULTI_STATUS: false 106 | LINTER_RULES_PATH: . 107 | VALIDATE_ALL_CODEBASE: true 108 | VALIDATE_BASH: true 109 | VALIDATE_BASH_EXEC: true 110 | VALIDATE_EDITORCONFIG: true 111 | VALIDATE_ENV: true 112 | # VALIDATE_GO_RELEASER: true 113 | VALIDATE_GITHUB_ACTIONS: true 114 | VALIDATE_HTML: true 115 | VALIDATE_JSON: true 116 | VALIDATE_NATURAL_LANGUAGE: true 117 | # VALIDATE_MARKDOWN: false 118 | VALIDATE_RENOVATE: true 119 | VALIDATE_SHELL_SHFMT: true 120 | VALIDATE_XML: true 121 | VALIDATE_YAML: true 122 | publish: 123 | name: Publish package 124 | if: >- 125 | github.event_name == 'push' 126 | && startsWith(github.ref, 'refs/tags/v') 127 | needs: 128 | - build 129 | - lint 130 | - goreleaser 131 | - super-lint 132 | runs-on: ubuntu-24.04 133 | permissions: 134 | contents: write 135 | id-token: write 136 | packages: write 137 | attestations: write 138 | steps: 139 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 140 | with: 141 | fetch-depth: 0 142 | 143 | - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 144 | with: 145 | go-version-file: 'go.mod' 146 | cache: true 147 | 148 | - uses: anchore/sbom-action/download-syft@f8bdd1d8ac5e901a77a92f111440fdb1b593736b # v0.20.6 149 | 150 | - uses: sigstore/cosign-installer@d7543c93d881b35a8faa02e8e3605f69b7a1ce62 # v3.10.0 151 | 152 | - name: Login to GitHub Container Registry 153 | uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 154 | with: 155 | registry: ghcr.io 156 | username: ${{ github.actor }} 157 | password: ${{ secrets.GITHUB_TOKEN }} 158 | 159 | - name: GPG configuration 160 | run: |- 161 | echo "$GPG_KEY" > "$GPG_KEY_PATH" 162 | mkdir -p "$HOME/.gnupg" 163 | chmod 0700 "$HOME/.gnupg" 164 | echo "use-agent" > "$HOME/.gnupg/gpg.conf" 165 | echo "pinentry-mode loopback" >> "$HOME/.gnupg/gpg.conf" 166 | echo "allow-loopback-pinentry" > "$HOME/.gnupg/gpg-agent.conf" 167 | echo "max-cache-ttl 86400" >> "$HOME/.gnupg/gpg-agent.conf" 168 | echo "default-cache-ttl 86400" >> "$HOME/.gnupg/gpg-agent.conf" 169 | gpgconf --kill gpg-agent 170 | gpgconf --launch gpg-agent 171 | echo "$GPG_PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 --import "$GPG_KEY_PATH" 172 | echo "1F34F95B4F30BC5B06E0D7CC3F619F17002790D8:6:" | gpg --import-ownertrust 173 | env: 174 | GPG_KEY_ID: ${{ vars.GPG_KEY_ID }} 175 | GPG_KEY: ${{ secrets.GPG_KEY }} 176 | GPG_PASSPHRASE: ${{ secrets.NFPM_PASSPHRASE }} 177 | GPG_KEY_PATH: "${{ secrets.GPG_KEY_PATH }}" 178 | 179 | - name: Set up Docker Buildx 180 | uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 181 | 182 | - name: Run GoReleaser 183 | id: goreleaser 184 | uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 185 | with: 186 | # renovate: github=goreleaser/goreleaser 187 | version: v2.12.5 188 | args: release --clean 189 | env: 190 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 191 | GORELEASER_CURRENT_TAG: ${{ github.ref_name }} 192 | NFPM_ACCESS-LOG-EXPORTER_PASSPHRASE: ${{ secrets.NFPM_PASSPHRASE }} 193 | GPG_KEY_PATH: ${{ secrets.GPG_KEY_PATH }} 194 | 195 | - name: Release APT repository 196 | run: | 197 | set -x 198 | gh release download "${GITHUB_REF_NAME}" -p "*.deb" -D tmp 199 | pushd tmp 200 | apt-ftparchive packages . | tee Packages | xz > Packages.xz 201 | apt-ftparchive release . > Release 202 | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --clearsign -o InRelease Release 203 | gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" --armor --detach-sign --sign -o Release.gpg Release 204 | gh release upload "${GITHUB_REF_NAME}" InRelease Packages Packages.xz Release Release.gpg --clobber 205 | popd 206 | env: 207 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 208 | GPG_PASSPHRASE: ${{ secrets.NFPM_PASSPHRASE }} 209 | 210 | - name: Publish Release 211 | run: gh release edit "${GITHUB_REF_NAME}" --draft=false 212 | env: 213 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 214 | 215 | # parse artifacts to the format required for image attestation 216 | # https://github.com/goreleaser/goreleaser/issues/4852#issuecomment-2122790132 217 | - run: | 218 | echo "digest=$(echo "$ARTIFACTS" | jq -r '[.[]|select(.type=="Docker Image")][0]|.extra.Digest')" | tee /dev/stderr >> "$GITHUB_OUTPUT" 219 | echo "name=$(echo "$ARTIFACTS" | jq -r '[.[]|select(.type=="Docker Image")][0]|.name|split(":")[0]')" | tee /dev/stderr >> "$GITHUB_OUTPUT" 220 | id: image_metadata 221 | env: 222 | ARTIFACTS: ${{steps.goreleaser.outputs.artifacts}} 223 | 224 | - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 225 | with: 226 | subject-checksums: ./dist/checksums.txt 227 | 228 | # attest image 229 | - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 230 | with: 231 | subject-digest: ${{steps.image_metadata.outputs.digest}} 232 | subject-name: ${{steps.image_metadata.outputs.name}} 233 | push-to-registry: true 234 | -------------------------------------------------------------------------------- /.github/workflows/pr-check.yaml: -------------------------------------------------------------------------------- 1 | name: Validate Pull Request 2 | on: 3 | pull_request: 4 | types: 5 | - opened 6 | - reopened 7 | - synchronize 8 | - labeled 9 | - unlabeled 10 | 11 | jobs: 12 | required-labels-missing: 13 | name: required labels missing 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: check 17 | if: >- 18 | !contains(github.event.pull_request.labels.*.name, '💥 breaking-change') 19 | && !contains(github.event.pull_request.labels.*.name, '✨ enhancement') 20 | && !contains(github.event.pull_request.labels.*.name, '🐞 bug') 21 | && !contains(github.event.pull_request.labels.*.name, '📖 docs') 22 | && !contains(github.event.pull_request.labels.*.name, 'chore') 23 | && !contains(github.event.pull_request.labels.*.name, '🛠️ dependencies') 24 | run: >- 25 | echo One of the following labels is missing on this PR: 26 | breaking-change 27 | enhancement 28 | bug 29 | docs 30 | chore 31 | && exit 1 32 | -------------------------------------------------------------------------------- /.github/workflows/stale.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: 'Close stale issues and PRs' 3 | on: 4 | schedule: 5 | - cron: '30 1 * * *' 6 | 7 | permissions: 8 | issues: write 9 | pull-requests: write 10 | 11 | jobs: 12 | stale: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0 16 | with: 17 | stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' 18 | days-before-stale: 30 19 | days-before-close: 5 20 | enable-statistics: true 21 | exempt-issue-labels: 'lifecycle/frozen' 22 | stale-issue-label: 'lifecycle/stale' 23 | -------------------------------------------------------------------------------- /.github/workflows/tag.yaml: -------------------------------------------------------------------------------- 1 | name: Create Tag 2 | run-name: "${{ inputs.tag }}" 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | tag: 7 | description: 'Tag to create (e.g. v1.2.3)' 8 | required: true 9 | type: string 10 | 11 | permissions: {} 12 | 13 | jobs: 14 | create: 15 | runs-on: ubuntu-24.04 16 | name: Create Tag 17 | if: startsWith(inputs.tag, 'v') 18 | steps: 19 | # Using a GitHub App token, because GitHub Actions doesn't run on commits from github-actions bot 20 | # Used App: 21 | # https://github.com/organizations/prometheus-community/settings/apps/helm-charts-renovate-helper. 22 | # Ref: https://github.com/prometheus-community/helm-charts/issues/5213. 23 | - uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 24 | id: app-token 25 | with: 26 | app-id: 1248576 27 | private-key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }} 28 | 29 | - name: Create Tag 30 | uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 31 | with: 32 | github-token: ${{ steps.app-token.outputs.token }} 33 | script: | 34 | github.rest.git.createRef({ 35 | owner: context.repo.owner, 36 | repo: context.repo.repo, 37 | ref: 'refs/tags/${{ inputs.tag }}', 38 | sha: context.sha, 39 | }); 40 | -------------------------------------------------------------------------------- /.github/workflows/wiki.yaml: -------------------------------------------------------------------------------- 1 | name: Push to GH Wiki 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - docs/** 10 | - .github/workflows/wiki.yaml 11 | 12 | permissions: 13 | contents: write 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 21 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 22 | with: 23 | repository: ${{ github.repository }}.wiki 24 | path: wiki.git 25 | 26 | - name: sync wiki 27 | run: rsync -av --delete --exclude=README.md --exclude=.git docs/ wiki.git/ 28 | 29 | - name: remove header line 30 | run: sed -i '1d' wiki.git/*.md 31 | 32 | - name: Commit files 33 | run: | 34 | git config --local user.email "action@github.com" 35 | git config --local user.name "GitHub Action" 36 | git add . 37 | git diff-index --quiet HEAD || git commit -m "Automatically publish wiki" && git push 38 | working-directory: wiki.git 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.out 3 | 4 | /.idea/* 5 | !/.idea/inspectionProfiles/ 6 | /.idea/inspectionProfiles/* 7 | !/.idea/inspectionProfiles/Project_Default.xml 8 | !/.idea/dictionaries/ 9 | /.idea/dictionaries/* 10 | !/.idea/dictionaries/project.xml 11 | /.idea/copyright/* 12 | !/.idea/copyright/profiles_settings.xml 13 | !/.idea/copyright/windows_exporter.xml 14 | !/.idea/vcs.xml 15 | !/.idea/go.imports.xml 16 | 17 | tests/* 18 | !tests/docker-compose.yaml 19 | !tests/Dockerfile 20 | !tests/.dockerignore 21 | !tests/systemd/ 22 | tests/systemd/prometheus-nginx-exporter.conf 23 | 24 | /access-log-exporter 25 | /access-log-exporter.exe 26 | dist/ 27 | bin/ 28 | 29 | ### macOS template 30 | # General 31 | .DS_Store 32 | .AppleDouble 33 | .LSOverride 34 | 35 | # Icon must end with two \r 36 | Icon 37 | 38 | # Thumbnails 39 | ._* 40 | 41 | # Files that might appear in the root of a volume 42 | .DocumentRevisions-V100 43 | .fseventsd 44 | .Spotlight-V100 45 | .TemporaryItems 46 | .Trashes 47 | .VolumeIcon.icns 48 | .com.apple.timemachine.donotpresent 49 | 50 | # Directories potentially created on remote AFP share 51 | .AppleDB 52 | .AppleDesktop 53 | Network Trash Folder 54 | Temporary Items 55 | .apdisk 56 | 57 | ### Windows template 58 | # Windows thumbnail cache files 59 | Thumbs.db 60 | Thumbs.db:encryptable 61 | ehthumbs.db 62 | ehthumbs_vista.db 63 | 64 | # Dump file 65 | *.stackdump 66 | 67 | # Folder config file 68 | [Dd]esktop.ini 69 | 70 | # Recycle Bin used on file shares 71 | $RECYCLE.BIN/ 72 | 73 | # Windows Installer files 74 | *.cab 75 | *.msi 76 | *.msix 77 | *.msm 78 | *.msp 79 | 80 | # Windows shortcuts 81 | *.lnk 82 | 83 | /3rdpartylicenses/ 84 | /local/ 85 | /tests/**/*.sysconfig 86 | Test*.te* 87 | *.test.exe 88 | *.test 89 | *.gpg 90 | venv/ 91 | kodata/ 92 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: all 4 | disable: 5 | - depguard 6 | - err113 7 | - exhaustruct 8 | - funlen 9 | - ireturn 10 | - mnd 11 | - noinlineerr 12 | - recvcheck 13 | - wsl 14 | settings: 15 | forbidigo: 16 | forbid: 17 | - pattern: "^(fmt\\.Print(|f|ln)|print|println)$" 18 | - pattern: "os.Stderr" 19 | msg: "Direct use of os.Stderr is forbidden. Use a logger instead." 20 | - pattern: "os.Stdout" 21 | msg: "Direct use of os.Stdout is forbidden. Use a logger instead." 22 | gomoddirectives: 23 | toolchain-forbidden: true 24 | gosec: 25 | excludes: 26 | - G101 27 | govet: 28 | enable-all: true 29 | disable: 30 | - shadow 31 | lll: 32 | line-length: 160 33 | revive: 34 | rules: 35 | - name: argument-limit 36 | arguments: 37 | - 7 38 | - name: atomic 39 | - name: bare-return 40 | - name: blank-imports 41 | - name: bool-literal-in-expr 42 | - name: call-to-gc 43 | - name: comment-spacings 44 | arguments: 45 | - 'nolint:' 46 | - name: confusing-naming 47 | - name: constant-logical-expr 48 | - name: context-as-argument 49 | - name: context-keys-type 50 | - name: datarace 51 | - name: deep-exit 52 | - name: defer 53 | - name: dot-imports 54 | - name: duplicated-imports 55 | - name: early-return 56 | - name: empty-block 57 | - name: empty-lines 58 | - name: enforce-map-style 59 | arguments: 60 | - make 61 | exclude: 62 | - TEST 63 | - name: enforce-repeated-arg-type-style 64 | arguments: 65 | - short 66 | - name: enforce-slice-style 67 | arguments: 68 | - make 69 | - name: error-naming 70 | - name: error-return 71 | - name: error-strings 72 | - name: errorf 73 | - name: get-return 74 | - name: identical-branches 75 | - name: if-return 76 | - name: import-alias-naming 77 | - name: import-shadowing 78 | - name: increment-decrement 79 | - name: indent-error-flow 80 | - name: modifies-parameter 81 | - name: modifies-value-receiver 82 | - name: optimize-operands-order 83 | - name: package-comments 84 | - name: range 85 | - name: range-val-address 86 | - name: range-val-in-closure 87 | - name: receiver-naming 88 | - name: redefines-builtin-id 89 | - name: redundant-import-alias 90 | - name: string-format 91 | arguments: 92 | - - fmt.Errorf[0],errors.New[0] 93 | - /^([^A-Z]|$)/ 94 | - Error string must not start with a capital letter. 95 | - - fmt.Errorf[0],errors.New[0] 96 | - /(^|[^\.!?])$/ 97 | - Error string must not end in punctuation. 98 | - - panic 99 | - /^[^\n]*$/ 100 | - Must not contain line breaks. 101 | - name: string-of-int 102 | - name: struct-tag 103 | - name: superfluous-else 104 | - name: time-equal 105 | - name: time-naming 106 | - name: unconditional-recursion 107 | - name: unexported-naming 108 | - name: unnecessary-stmt 109 | - name: unreachable-code 110 | - name: unused-parameter 111 | - name: var-declaration 112 | - name: var-naming 113 | arguments: 114 | - [] # AllowList - do not remove as args for the rule are positional and won't work without lists first 115 | - [] # DenyList 116 | - - skip-package-name-checks: true 117 | - name: waitgroup-by-value 118 | sloglint: 119 | no-mixed-args: true 120 | kv-only: false 121 | attr-only: true 122 | no-global: all 123 | context: scope 124 | static-msg: false 125 | no-raw-keys: false 126 | key-naming-case: snake 127 | forbidden-keys: 128 | - time 129 | - level 130 | - msg 131 | - source 132 | args-on-sep-lines: true 133 | tagliatelle: 134 | case: 135 | rules: 136 | yaml: camel 137 | json: camel 138 | varnamelen: 139 | ignore-names: 140 | - tc 141 | - ch 142 | ignore-decls: 143 | - i int 144 | - n int 145 | - a ...any 146 | - err error 147 | - ok bool 148 | - w http.ResponseWriter 149 | - r *http.Request 150 | - rt http.RoundTripper 151 | - l net.Listener 152 | - t reflect.Type 153 | - wg sync.WaitGroup 154 | - wg *sync.WaitGroup 155 | - sb strings.Builder 156 | - mu sync.Mutex 157 | exclusions: 158 | generated: lax 159 | paths: 160 | - "^examples/" 161 | - "^docs/" 162 | presets: 163 | - comments 164 | - common-false-positives 165 | - legacy 166 | - std-error-handling 167 | rules: 168 | - linters: 169 | - contextcheck 170 | - cyclop 171 | - dogsled 172 | - dupl 173 | - dupword 174 | - funlen 175 | - gocognit 176 | - gocyclo 177 | - lll 178 | - maintidx 179 | - wrapcheck 180 | path: _test\.go 181 | - path: _test\.go 182 | text: "fieldalignment:" 183 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 2 | 3 | version: 2 4 | before: 5 | hooks: 6 | - rm -rf 3rdpartylicenses 7 | - make 3rdpartylicenses 8 | 9 | builds: 10 | - id: "access-log-exporter" 11 | main: ./cmd/access-log-exporter 12 | binary: access-log-exporter 13 | goos: 14 | - linux 15 | - freebsd 16 | - openbsd 17 | goarch: 18 | - amd64 19 | - arm64 20 | mod_timestamp: '{{ .CommitTimestamp }}' 21 | env: 22 | - CGO_ENABLED=0 23 | flags: 24 | - -trimpath 25 | ldflags: 26 | - >- 27 | -s -w 28 | -X github.com/prometheus/common/version.Version={{.Version}} 29 | -X github.com/prometheus/common/version.Revision={{.Commit}} 30 | -X github.com/prometheus/common/version.Branch={{.Branch}} 31 | -X github.com/prometheus/common/version.BuildDate={{.Date}} 32 | 33 | archives: 34 | - id: access-log-exporter 35 | ids: 36 | - access-log-exporter 37 | formats: ['tar.xz'] 38 | files: 39 | - LICENSE.txt 40 | - 3rdpartylicenses/**/* 41 | 42 | nfpms: 43 | - id: access-log-exporter 44 | ids: 45 | - access-log-exporter 46 | homepage: https://github.com/jkroepke/access-log-exporter 47 | maintainer: Jan-Otto Kröpke 48 | section: net 49 | description: | 50 | access-log-exporter is a management client for OpenVPN that handles the authentication of connecting users against OIDC providers like Azure AD or Keycloak. 51 | license: MIT License 52 | formats: 53 | - deb 54 | - rpm 55 | provides: 56 | - access-log-exporter 57 | recommends: 58 | - openvpn 59 | contents: 60 | - src: packaging/usr/lib/sysusers.d/ 61 | dst: /usr/lib/sysusers.d/ 62 | - src: packaging/usr/lib/systemd/system/ 63 | dst: /usr/lib/systemd/system/ 64 | - dst: /etc/access-log-exporter/ 65 | type: dir 66 | file_info: 67 | owner: root 68 | group: access-log-exporter 69 | mode: 0750 70 | - dst: /etc/access-log-exporter/client-config/ 71 | type: dir 72 | file_info: 73 | owner: root 74 | group: access-log-exporter 75 | mode: 0750 76 | - src: packaging/etc/access-log-exporter/config.yaml 77 | dst: /etc/access-log-exporter/config.yaml 78 | type: "config|noreplace" 79 | file_info: 80 | owner: root 81 | group: access-log-exporter 82 | mode: 0640 83 | - src: 3rdpartylicenses/ 84 | dst: /usr/share/doc/access-log-exporter/3rdpartylicenses/ 85 | type: tree 86 | file_info: 87 | owner: root 88 | group: root 89 | mode: 0644 90 | scripts: 91 | preinstall: "packaging/scripts/preinst.sh" 92 | postinstall: "packaging/scripts/postinst.sh" 93 | preremove: "packaging/scripts/preremove.sh" 94 | postremove: "packaging/scripts/postremove.sh" 95 | rpm: 96 | compression: xz 97 | signature: 98 | key_file: "{{ .Env.GPG_KEY_PATH }}" 99 | deb: 100 | #compression: xz 101 | signature: 102 | key_file: "{{ .Env.GPG_KEY_PATH }}" 103 | 104 | dockers_v2: 105 | - images: 106 | - ghcr.io/jkroepke/access-log-exporter 107 | tags: 108 | - '{{ .Version }}' 109 | - '{{ if and .Tag (not .Prerelease) (not .IsSnapshot) (ne .Branch "main") }}latest{{end}}' 110 | - '{{ if eq .Branch "main" }}main{{end}}' 111 | extra_files: 112 | - packaging/etc/access-log-exporter/config.yaml 113 | labels: 114 | "org.opencontainers.image.authors": 'Jan-Otto Kröpke https://github.com/jkroepke/' 115 | "org.opencontainers.image.created": '{{.Date}}' 116 | "org.opencontainers.image.description": 'A Prometheus exporter that receives access logs through the syslog protocol and converts them into metrics.' 117 | "org.opencontainers.image.documentation": 'https://github.com/jkroepke/access-log-exporter/wiki' 118 | "org.opencontainers.image.licenses": 'Apache-2.0' 119 | "org.opencontainers.image.revision": '{{.FullCommit}}' 120 | "org.opencontainers.image.source": 'https://github.com/jkroepke/access-log-exporter' 121 | "org.opencontainers.image.title": '{{.ProjectName}}' 122 | "org.opencontainers.image.vendor": 'Jan-Otto Kröpke' 123 | "org.opencontainers.image.version": '{{.Version}}' 124 | annotations: 125 | "org.opencontainers.image.authors": 'Jan-Otto Kröpke https://github.com/jkroepke/' 126 | "org.opencontainers.image.created": '{{.Date}}' 127 | "org.opencontainers.image.description": 'A Prometheus exporter that receives access logs through the syslog protocol and converts them into metrics.' 128 | "org.opencontainers.image.documentation": 'https://github.com/jkroepke/access-log-exporter/wiki' 129 | "org.opencontainers.image.licenses": 'Apache-2.0' 130 | "org.opencontainers.image.revision": '{{.FullCommit}}' 131 | "org.opencontainers.image.source": 'https://github.com/jkroepke/access-log-exporter' 132 | "org.opencontainers.image.title": '{{.ProjectName}}' 133 | "org.opencontainers.image.vendor": 'Jan-Otto Kröpke' 134 | "org.opencontainers.image.version": '{{.Version}}' 135 | platforms: 136 | - linux/amd64 137 | - linux/arm64 138 | # flags: 139 | # - '{{ if not .IsSnapshot }}--sbom=true{{end}}' 140 | # - '{{ if not .IsSnapshot }}--provenance=false{{end}}' 141 | 142 | docker_signs: 143 | - artifacts: '' 144 | output: true 145 | cmd: cosign 146 | env: 147 | - COSIGN_EXPERIMENTAL=1 148 | args: 149 | - sign 150 | - '--oidc-issuer={{if index .Env "CI"}}https://token.actions.githubusercontent.com{{else}}https://oauth2.sigstore.dev/auth{{end}}' 151 | - '--yes' 152 | - '${artifact}@${digest}' 153 | 154 | checksum: 155 | name_template: "checksums.txt" 156 | 157 | report_sizes: true 158 | 159 | metadata: 160 | mod_timestamp: "{{ .CommitTimestamp }}" 161 | 162 | gomod: 163 | proxy: true 164 | 165 | release: 166 | draft: true 167 | replace_existing_draft: true 168 | prerelease: auto 169 | 170 | changelog: 171 | use: github-native 172 | 173 | sboms: 174 | - artifacts: archive 175 | -------------------------------------------------------------------------------- /.idea/dictionaries/project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | httpbin 5 | opencontainers 6 | rdpartylicenses 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/go.imports.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 11 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /AGENTS.md: -------------------------------------------------------------------------------- 1 | # Instructions for AI Agents 2 | 3 | The following guidelines apply to all files in this repository. 4 | 5 | Before you start contributing, read [`DEVELOPER.md`](DEVELOPER.md) for a basic 6 | understanding of how the project is structured and works. 7 | 8 | Ensure that that the local go version matches the one specified in 9 | [`go.mod`](go.mod). 10 | Never update the Go version in `go.mod`. 11 | 12 | ## Programmatic checks 13 | 14 | Before committing any changes, always run: 15 | 16 | 1. `make fmt` – formats all Go code. 17 | 2. `make lint` – runs the linter. 18 | 3. `make test` – executes the test suite. 19 | 20 | If a command fails because of missing dependencies or network restrictions, note this in the PR's Testing section using the provided disclaimer. 21 | 22 | ## Pull requests 23 | 24 | Summarise your changes and cite relevant lines in the repository. Mention the output of the programmatic checks. 25 | 26 | ## Program overview 27 | 28 | `access-log-export` is written in Go and acts as a prometheus exporter for 29 | access logs. 30 | It collects and exports metrics from access logs collected through syslog protocol. 31 | 32 | Configuration is usually done through a YAML file or environment variables. The 33 | project's `docs/` directory contains detailed guides such as 34 | [`docs/Configuration.md`](docs/Configuration.md) and 35 | [`docs/Home.md`](docs/Home.md). 36 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, caste, color, religion, or sexual 10 | identity and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the overall 26 | community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or advances of 31 | any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email address, 35 | without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official email address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | GitHub Issues. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series of 86 | actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or permanent 93 | ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within the 113 | community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.1, available at 119 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 123 | 124 | For answers to common questions about this code of conduct, see the FAQ at 125 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 126 | [https://www.contributor-covenant.org/translations][translations]. 127 | 128 | [homepage]: https://www.contributor-covenant.org 129 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 130 | [Mozilla CoC]: https://github.com/mozilla/diversity 131 | [FAQ]: https://www.contributor-covenant.org/faq 132 | [translations]: https://www.contributor-covenant.org/translations 133 | -------------------------------------------------------------------------------- /DEVELOPER.md: -------------------------------------------------------------------------------- 1 | # Developer Guide 2 | 3 | This document provides a technical overview of the project and highlights the most important packages and concepts. 4 | 5 | ## Overview 6 | 7 | `access-log-exporter` is a high-performance Go application that acts as a Prometheus exporter for access logs. It receives log messages via syslog protocol, parses them according to configurable rules, and exposes the extracted metrics in Prometheus format. 8 | 9 | ### Key Features 10 | 11 | - **High-throughput processing**: Concurrent worker architecture for processing thousands of log lines per second 12 | - **Flexible configuration**: YAML-based configuration with support for multiple metric presets 13 | - **Multiple metric types**: Support for Prometheus counters, gauges, and histograms 14 | - **Label processing**: Advanced label extraction with regular expression replacements and user agent parsing 15 | - **Upstream support**: Special handling for load balancer upstream servers 16 | - **Memory efficient**: Uses sync.Pool for object reuse to minimize garbage collection pressure 17 | - **Thread-safe**: Designed for concurrent access across multiple goroutines 18 | 19 | ## Architecture 20 | 21 | The application follows a pipeline architecture with the following main components: 22 | 23 | ``` 24 | Syslog Server → Message Buffer → Worker Pool → Metric Processing → Prometheus Export 25 | ``` 26 | 27 | ### Core Components 28 | 29 | 1. **Syslog Server** (`internal/syslog`): Receives and parses syslog messages 30 | 2. **Collector** (`internal/collector`): Manages worker pool and coordinates metric processing 31 | 3. **Metric Engine** (`internal/metric`): Processes log lines and updates Prometheus metrics 32 | 4. **Configuration** (`internal/config`): Handles YAML configuration and validation 33 | 5. **HTTP Server**: Exposes `/metrics` endpoint for Prometheus scraping 34 | 35 | ## How It Works 36 | 37 | ### 1. Startup and Initialization 38 | 39 | The application starts by: 40 | 41 | 1. **Configuration Loading**: Reads YAML configuration file or environment variables 42 | 2. **Preset Selection**: Chooses the active metric preset from configuration 43 | 3. **Syslog Server Setup**: Creates UDP or Unix socket listener for syslog messages 44 | 4. **Metric Initialization**: Creates Prometheus metric collectors based on configuration 45 | 5. **Worker Pool**: Spawns concurrent workers (defaults to number of CPU cores) 46 | 6. **HTTP Server**: Starts web server for `/metrics` and `/health` endpoints 47 | 48 | ### 2. Message Flow 49 | 50 | #### Syslog Reception 51 | ```go 52 | // Syslog server receives messages on UDP/Unix socket 53 | // Strips syslog headers and extracts the actual log message 54 | // Sends cleaned message to buffered channel 55 | ``` 56 | 57 | The syslog component: 58 | - Listens on UDP or Unix domain sockets 59 | - Parses syslog format messages (e.g., `<34>Oct 11 22:14:15 nginx: actual_log_message`) 60 | - Extracts the actual log message after the third colon 61 | - Uses a buffer pool to minimize memory allocations 62 | 63 | #### Concurrent Processing 64 | ```go 65 | // Multiple worker goroutines process messages concurrently 66 | func (c *Collector) lineHandlerWorker(ctx context.Context, logger *slog.Logger, messageCh <-chan string) { 67 | for msg := range messageCh { 68 | // Split message into fields (tab-separated) 69 | line := strings.Split(msg, "\t") 70 | 71 | // Process each configured metric 72 | for _, metric := range c.metrics { 73 | metric.Parse(line) 74 | } 75 | } 76 | } 77 | ``` 78 | 79 | Workers operate independently and process messages from a shared channel, providing high throughput through parallel processing. 80 | 81 | #### Metric Processing 82 | ```go 83 | // Each metric processes the log line according to its configuration 84 | func (m *Metric) Parse(line []string) error { 85 | // 1. Validate line format and extract value 86 | // 2. Get labels map from sync.Pool (thread-safe reuse) 87 | // 3. Process each configured label 88 | // 4. Apply transformations (user agent parsing, regular expression replacements) 89 | // 5. Set metric value (counter, gauge, or histogram) 90 | // 6. Return labels map to pool 91 | } 92 | ``` 93 | 94 | ### 3. Configuration System 95 | 96 | The configuration system supports: 97 | 98 | #### Presets 99 | ```yaml 100 | presets: 101 | nginx: 102 | metrics: 103 | - name: http_requests_total 104 | type: counter 105 | help: "Total HTTP requests" 106 | labels: 107 | - name: host 108 | lineIndex: 0 109 | - name: method 110 | lineIndex: 1 111 | ``` 112 | 113 | #### Metric Types 114 | - **Counter**: Monotonically increasing values (request counts, error counts) 115 | - **Gauge**: Values that can go up and down (response times, queue sizes) 116 | - **Histogram**: Distribution of values (response time distributions) 117 | 118 | #### Advanced Features 119 | - **Math transformations**: Apply multiplication/division to metric values 120 | - **Label replacements**: Use regular expression to transform label values 121 | - **User agent parsing**: Extract browser family from user agent strings 122 | - **Upstream handling**: Special support for load balancer upstream servers 123 | 124 | ### 4. Performance Optimizations 125 | 126 | #### Memory Management 127 | - **sync.Pool**: Reuses `prometheus.Labels` maps across goroutines to reduce allocations 128 | - **Buffer pooling**: Syslog server reuses byte buffers for reading messages 129 | - **Pre-sized allocations**: Maps and slices are allocated with known capacity 130 | 131 | #### Concurrency 132 | - **Worker pool**: Parallel processing across multiple CPU cores 133 | - **Thread-safe design**: All components designed for concurrent access 134 | - **Channel-based communication**: Non-blocking message passing between components 135 | 136 | #### Parsing Optimizations 137 | - **Bounds check elimination**: Uses Go compiler hints to eliminate array bounds checks 138 | - **Efficient string operations**: Uses `strings.IndexByte` for fast comma parsing 139 | - **Regular expression optimization**: Only calls regular expression replacement when match is found 140 | 141 | ### 5. Key Packages 142 | 143 | #### `internal/metric` 144 | Core metric processing engine: 145 | - `Parse()`: Main entry point for processing log lines 146 | - `setMetric()`: Handles value parsing and Prometheus metric updates 147 | - `labelValueReplacements()`: Applies regular expression transformations to labels 148 | - Thread-safe design using sync.Pool for label map reuse 149 | 150 | #### `internal/collector` 151 | Manages the worker pool and coordinates metric processing: 152 | - `lineHandlerWorkers()`: Creates concurrent worker goroutines 153 | - `lineHandlerWorker()`: Individual worker that processes messages 154 | - Implements Prometheus collector interface 155 | 156 | #### `internal/syslog` 157 | Handles syslog protocol reception: 158 | - Supports UDP and Unix domain sockets 159 | - Parses syslog format and extracts log messages 160 | - Uses buffer pooling for high-performance message processing 161 | 162 | #### `internal/config` 163 | Configuration management and validation: 164 | - YAML/JSON configuration parsing 165 | - Environment variable support 166 | - Configuration validation and defaults 167 | - Support for multiple metric presets 168 | 169 | ### 6. Monitoring and Observability 170 | 171 | The exporter includes built-in metrics: 172 | - `log_parse_errors_total`: Counter of parsing errors 173 | - `log_last_received_timestamp_seconds`: Timestamp of last received message 174 | - Standard Go runtime metrics (memory, GC, goroutines) 175 | - Optional nginx stub_status metrics 176 | 177 | ### 7. Testing and Benchmarking 178 | 179 | The project includes comprehensive benchmarks: 180 | - `BenchmarkMetricParseSimple`: Tests basic metric parsing performance 181 | - `BenchmarkMetricParseUserAgent`: Tests user agent parsing overhead 182 | - `BenchmarkMetricParseUpstream`: Tests upstream processing performance 183 | 184 | Performance targets: 185 | - Zero allocations in the hot path for simple metrics 186 | - Sub-microsecond processing time per log line 187 | - Scales linearly with number of CPU cores 188 | 189 | ## Development Workflow 190 | 191 | 1. **Code formatting**: Run `make fmt` to format Go code 192 | 2. **Linting**: Run `make lint` to check code quality 193 | 3. **Testing**: Run `make test` to execute test suite 194 | 4. **Benchmarking**: Use `go test -bench=.` to measure performance 195 | 196 | The application is designed for high-throughput log processing environments where performance and memory efficiency are critical requirements. 197 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | ARG TARGETPLATFORM 3 | ENTRYPOINT ["/access-log-exporter"] 4 | COPY packaging/etc/access-log-exporter/config.yaml /config.yaml 5 | COPY $TARGETPLATFORM/access-log-exporter / 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## 2 | # Console Colors 3 | ## 4 | GREEN := $(shell printf "\033[0;32m") 5 | YELLOW := $(shell printf "\033[0;33m") 6 | WHITE := $(shell printf "\033[0;37m") 7 | CYAN := $(shell printf "\033[0;36m") 8 | RESET := $(shell printf "\033[0m") 9 | 10 | # Get the current working directory 11 | CURRENT_DIR := $(CURDIR) 12 | 13 | # Get the directory name of the current working directory 14 | PROJECT_NAME := $(notdir $(CURRENT_DIR)) 15 | 16 | # Get the GOOS value 17 | GOOS := $(shell go env GOOS) 18 | 19 | # Determine the output file extension based on the GOOS value 20 | ifeq ($(GOOS),windows) 21 | EXT := .exe 22 | else 23 | EXT := 24 | endif 25 | 26 | ## 27 | # Targets 28 | ## 29 | .PHONY: help 30 | help: ## show this help. 31 | @echo "Project: $(PROJECT_NAME)" 32 | @echo 'Usage:' 33 | @echo " ${GREEN}make${RESET} ${YELLOW}${RESET}" 34 | @echo '' 35 | @echo 'Targets:' 36 | @awk 'BEGIN {FS = ":.*?## "} { \ 37 | if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf " ${GREEN}%-21s${YELLOW}%s${RESET}\n", $$1, $$2} \ 38 | else if (/^## .*$$/) {printf " ${CYAN}%s${RESET}\n", substr($$1,4)} \ 39 | }' $(MAKEFILE_LIST) | sort 40 | 41 | .PHONY: clean 42 | clean: ## clean builds dir 43 | @rm -rf "$(PROJECT_NAME)" "$(PROJECT_NAME).exe" dist/ 44 | 45 | .PHONY: check 46 | check: test lint golangci ## Run all checks locally 47 | 48 | .PHONY: update 49 | update: ## Run dependency updates 50 | @go get -u ./... 51 | @go mod tidy 52 | 53 | .PHONY: build ## Build the project 54 | build: clean $(PROJECT_NAME) 55 | 56 | $(PROJECT_NAME): 57 | @go build -o $(PROJECT_NAME)$(EXT) ./cmd/$(PROJECT_NAME) 58 | 59 | .PHONY: test 60 | test: ## Test the project 61 | @go test -race ./... 62 | 63 | .PHONY: lint 64 | lint: golangci ## Run linter 65 | 66 | .PHONY: fmt ## Format code 67 | fmt: 68 | @-go fmt ./... 69 | @-go mod tidy 70 | @-go run github.com/daixiang0/gci@v0.13.7 write . 71 | @-go run mvdan.cc/gofumpt@v0.9.1 -l -w . 72 | @-go run golang.org/x/tools/cmd/goimports@v0.37.0 -l -w . 73 | @-go run github.com/bombsimon/wsl/v5/cmd/wsl@v5.2.0 -fix ./... 74 | @-go run github.com/catenacyber/perfsprint@v0.9.1 --fix ./... 75 | @-go run github.com/tetafro/godot/cmd/godot@v1.4.20 -w . 76 | @-go run github.com/4meepo/tagalign/cmd/tagalign@v1.4.3 -fix -sort ./... 77 | @-go run golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@v0.37.0 -test=false -fix ./... 78 | @-go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 run ./... 79 | 80 | .PHONY: golangci 81 | golangci: 82 | @go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 run ./... 83 | 84 | .PHONY: 3rdpartylicenses 85 | 3rdpartylicenses: 86 | @go run github.com/google/go-licenses/v2@v2.0.1 save ./cmd/access-log-exporter --save_path=3rdpartylicenses 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CI](https://github.com/jkroepke/access-log-exporter/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/jkroepke/access-log-exporter/actions/workflows/ci.yaml) 2 | [![GitHub license](https://img.shields.io/github/license/jkroepke/access-log-exporter)](https://github.com/jkroepke/access-log-exporter/blob/master/LICENSE.txt) 3 | [![Current Release](https://img.shields.io/github/release/jkroepke/access-log-exporter.svg?logo=github)](https://github.com/jkroepke/access-log-exporter/releases/latest) 4 | [![GitHub Repo stars](https://img.shields.io/github/stars/jkroepke/access-log-exporter?style=flat&logo=github)](https://github.com/jkroepke/access-log-exporter/stargazers) 5 | [![GitHub all releases](https://img.shields.io/github/downloads/jkroepke/access-log-exporter/total?logo=github)](https://github.com/jkroepke/access-log-exporter/releases/latest) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/jkroepke/access-log-exporter)](https://goreportcard.com/report/github.com/jkroepke/access-log-exporter) 7 | [![codecov](https://codecov.io/gh/jkroepke/access-log-exporter/graph/badge.svg?token=TJRPHF5BVX)](https://codecov.io/gh/jkroepke/access-log-exporter) 8 | 9 | # access-log-exporter 10 | 11 | ⭐ Don't forget to star this repository! ⭐ 12 | 13 | A Prometheus exporter that receives web server access logs via syslog and converts them into metrics. 14 | 15 | access-log-exporter processes logs from multiple web servers and has undergone testing with Nginx and Apache HTTP Server 2.4. It supports flexible metric configuration through presets and provides comprehensive monitoring capabilities for web traffic analysis. 16 | 17 | ## Features 18 | 19 | - **Multi-server support**: Works with Nginx and Apache HTTP Server, 20 | - **Syslog protocol**: Receives logs via UDP/TCP syslog for real-time processing, 21 | - **Flexible configuration**: Customizable presets for different monitoring needs, 22 | - **Built-in presets**: Ready-to-use configurations for common scenarios, 23 | - **Upstream metrics**: Support for Nginx upstream server monitoring, 24 | - **Nginx status metrics**: Built-in Nginx stub_status scraping eliminates the need for a separate nginx-prometheus-exporter, 25 | - **Label transformation**: Regex-based label value normalization, 26 | - **Mathematical operations**: Unit conversion for proper Prometheus base units, 27 | - **High performance**: Configurable buffering and worker threads. 28 | 29 | ## Quick Start 30 | 31 | ### Installation 32 | 33 | **Using package managers:** 34 | ```bash 35 | # Debian/Ubuntu 36 | curl -L https://raw.githubusercontent.com/jkroepke/access-log-exporter/refs/heads/main/packaging/apt/access-log-exporter.sources | sudo tee /etc/apt/sources.list.d/access-log-exporter.sources 37 | sudo apt update && sudo apt install access-log-exporter 38 | 39 | # Manual download 40 | # Download the latest release from GitHub releases page 41 | ``` 42 | 43 | **Using Docker:** 44 | ```bash 45 | docker run -p 4040:4040 -p 8514:8514/udp ghcr.io/jkroepke/access-log-exporter:latest 46 | ``` 47 | 48 | ### Basic Configuration 49 | 50 | **1. Configure your web server to send logs via syslog:** 51 | 52 | For **Nginx**, add to your configuration: 53 | ```nginx 54 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent'; 55 | access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 56 | ``` 57 | 58 | For **Apache2**, add to your configuration: 59 | ```apache 60 | LogFormat "%v\t%m\t%>s\tOK\t%{ms}T\t%I\t%O" accesslog_exporter 61 | CustomLog "|/usr/bin/logger --rfc3164 --server 127.0.0.1 --port 8514 --udp" accesslog_exporter 62 | ``` 63 | 64 | **2. Start access-log-exporter:** 65 | ```bash 66 | access-log-exporter --preset simple 67 | ``` 68 | 69 | **3. Access metrics:** 70 | ```bash 71 | curl http://localhost:4040/metrics 72 | ``` 73 | 74 | ## Available Presets 75 | 76 | - **`simple`**: Basic HTTP metrics (requests, response times, sizes) - compatible with both Nginx and Apache 77 | - **`simple_upstream`**: Includes upstream server metrics - Nginx only 78 | - **`simple_uri_upstream`**: Extends `simple_upstream` with request URI tracking and path normalization - Nginx only 79 | 80 | ## Configuration 81 | 82 | access-log-exporter supports configuration via: 83 | - Command-line flags 84 | - Environment variables 85 | - YAML configuration files 86 | 87 | **Example command-line usage:** 88 | ```bash 89 | # Use different preset 90 | access-log-exporter --preset simple_upstream 91 | 92 | # Custom syslog port 93 | access-log-exporter --syslog.listen-address udp://0.0.0.0:9514 94 | 95 | # Custom metrics port 96 | access-log-exporter --web.listen-address :9090 97 | ``` 98 | 99 | **Example configuration file:** 100 | ```yaml 101 | preset: "simple" 102 | syslog: 103 | listenAddress: "udp://[::]:8514" 104 | web: 105 | listenAddress: ":4040" 106 | bufferSize: 1000 107 | workerCount: 4 108 | ``` 109 | 110 | ## Metrics Examples 111 | 112 | With the `simple` preset, you get metrics like: 113 | 114 | ```prometheus 115 | # HELP http_requests_total The total number of client requests 116 | # TYPE http_requests_total counter 117 | http_requests_total{host="example.com",method="GET",status="200"} 1234 118 | 119 | # HELP http_requests_completed_total The total number of completed requests 120 | # TYPE http_requests_completed_total counter 121 | http_requests_completed_total{host="example.com",method="GET",status="200"} 1234 122 | 123 | # HELP http_request_duration_seconds The time spent on receiving and response the response to the client 124 | # TYPE http_request_duration_seconds histogram 125 | http_request_duration_seconds_bucket{host="example.com",method="GET",status="200",le="0.005"} 123 126 | http_request_duration_seconds_bucket{host="example.com",method="GET",status="200",le="0.01"} 234 127 | ``` 128 | 129 | ## Nginx Status Metrics 130 | 131 | In addition to processing access logs, access-log-exporter can also collect Nginx server status metrics by scraping the `stub_status` module. This provides additional insights into your Nginx server's performance and connection handling. 132 | 133 | ### Enabling Nginx Status Collection 134 | 135 | To enable Nginx status metrics collection, use the `--nginx.scrape-url` flag: 136 | 137 | ```bash 138 | # HTTP endpoint 139 | access-log-exporter --nginx.scrape-url http://127.0.0.1/stub_status 140 | 141 | # Unix domain socket 142 | access-log-exporter --nginx.scrape-url unix:///var/run/nginx-status.sock 143 | ``` 144 | 145 | ### Nginx Configuration 146 | 147 | First, enable the `stub_status` module in your Nginx configuration: 148 | 149 | ```nginx 150 | server { 151 | listen 127.0.0.1:8080; 152 | server_name localhost; 153 | 154 | location /stub_status { 155 | stub_status on; 156 | access_log off; 157 | allow 127.0.0.1; 158 | deny all; 159 | } 160 | } 161 | ``` 162 | 163 | **Important:** Ensure the stub_status endpoint is only accessible from localhost or trusted networks for security. 164 | 165 | For detailed information about the metrics exposed and configuration options, see the [Nginx Status Metrics section](https://github.com/jkroepke/access-log-exporter/wiki/Configuration#nginx-status-metrics) in the Configuration Guide. 166 | 167 | ### Configuration File Example 168 | 169 | You can also configure the Nginx scrape URL in your YAML configuration file: 170 | 171 | ```yaml 172 | nginx: 173 | scrapeUri: "http://127.0.0.1:8080/stub_status" 174 | 175 | preset: "simple" 176 | syslog: 177 | listenAddress: "udp://[::]:8514" 178 | web: 179 | listenAddress: ":4040" 180 | ``` 181 | 182 | ## Documentation 183 | 184 | For detailed documentation, please refer to: 185 | 186 | - **[Installation Guide](https://github.com/jkroepke/access-log-exporter/wiki/Installation)**: Package installation, manual builds, Kubernetes deployment 187 | - **[Configuration Guide](https://github.com/jkroepke/access-log-exporter/wiki/Configuration)**: Complete configuration reference and custom presets 188 | - **[Webserver Setup](https://github.com/jkroepke/access-log-exporter/wiki/Webserver)**: Nginx and Apache configuration examples 189 | - **[Wiki](https://github.com/jkroepke/access-log-exporter/wiki)**: Additional guides and examples 190 | 191 | ## Requirements 192 | 193 | - Go 1.21+ (for building from source) 194 | - Web server with syslog support (Nginx, Apache) 195 | - Network connectivity between web server and access-log-exporter 196 | 197 | ## Contributing 198 | 199 | Contributions welcome! Please read our [Code of Conduct](CODE_OF_CONDUCT.md) and submit pull requests to help improve the project. 200 | 201 | ## Related Projects 202 | 203 | * [martin-helmich/prometheus-nginxlog-exporter](https://github.com/martin-helmich/prometheus-nginxlog-exporter). 204 | * [ozonru/accesslog-exporter](https://github.com/ozonru/accesslog-exporter) 205 | 206 | ## Copyright and license 207 | 208 | © 2025 Jan-Otto Kröpke (jkroepke) 209 | 210 | Licensed under the [Apache License, Version 2.0](LICENSE.txt). 211 | 212 | ## Open Source Sponsors 213 | 214 | Thanks to all sponsors! 215 | 216 | * [@hegawa](https://github.com/hegawa) (25$) onetime 217 | * [@Zero-Down-Time](https://github.com/Zero-Down-Time) (25$) onetime 218 | * [@k0ste](https://github.com/k0ste) (25$) onetime 219 | 220 | ## Acknowledgements 221 | 222 | Thanks to JetBrains IDEs for their support. 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 242 | 243 | 244 |
JetBrains IDEs
233 |

234 | 235 | 236 | 237 | 238 | 239 | 240 |

241 |
245 | -------------------------------------------------------------------------------- /cmd/access-log-exporter/it_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "syscall" 13 | "testing" 14 | "time" 15 | 16 | "github.com/docker/docker/api/types/container" 17 | "github.com/stretchr/testify/require" 18 | "github.com/testcontainers/testcontainers-go" 19 | "github.com/testcontainers/testcontainers-go/wait" 20 | ) 21 | 22 | const nginxConfig = ` 23 | user nginx; 24 | worker_processes auto; 25 | 26 | pid /run/nginx.pid; 27 | 28 | events { 29 | worker_connections 1024; 30 | } 31 | 32 | http { 33 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent'; 34 | access_log syslog:server=host.docker.internal:8514,nohostname accesslog_exporter; 35 | 36 | server { 37 | listen 8080; 38 | server_name localhost; 39 | 40 | location = /200 { 41 | return 200 "OK"; 42 | } 43 | location = /204 { 44 | return 204 "No Content"; 45 | } 46 | location = /404 { 47 | return 404 "Not Found"; 48 | } 49 | location = /500 { 50 | return 500 "Internal Server Error"; 51 | } 52 | 53 | location /proxy/ { 54 | proxy_pass http://127.0.0.1:8080/; 55 | } 56 | 57 | location = /stub_status { 58 | stub_status; 59 | } 60 | } 61 | } 62 | ` 63 | 64 | func TestIT(t *testing.T) { 65 | t.Parallel() 66 | 67 | termCh := make(chan os.Signal) 68 | returnCodeCh := make(chan ReturnCode, 1) 69 | 70 | dockerImage := "nginx" 71 | if dockerImageEnv, ok := os.LookupEnv("DOCKER_IMAGE"); ok { 72 | dockerImage = dockerImageEnv 73 | } 74 | 75 | nginx, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ 76 | ContainerRequest: testcontainers.ContainerRequest{ 77 | Image: dockerImage, 78 | ConfigModifier: func(config *container.Config) { 79 | config.Cmd = []string{ 80 | "nginx-debug", "-g", "daemon off;", 81 | } 82 | }, 83 | ExposedPorts: []string{ 84 | "8080/tcp", 85 | }, 86 | Env: map[string]string{ 87 | "NGINX_ENTRYPOINT_QUIET_LOGS": "true", 88 | }, 89 | Labels: map[string]string{ 90 | "testcontainers": "true", 91 | }, 92 | HostConfigModifier: func(hostConfig *container.HostConfig) { 93 | hostConfig.ExtraHosts = []string{"host.docker.internal:host-gateway"} 94 | }, 95 | Files: []testcontainers.ContainerFile{ 96 | { 97 | Reader: strings.NewReader(nginxConfig), 98 | ContainerFilePath: "/etc/nginx/nginx.conf", 99 | FileMode: 0o644, 100 | }, 101 | }, 102 | WaitingFor: wait.ForListeningPort("8080/tcp").WithStartupTimeout(time.Second * 5), 103 | }, 104 | Started: true, 105 | }) 106 | 107 | testcontainers.CleanupContainer(t, nginx) 108 | 109 | containerLogs, _ := getContainerLogs(t, nginx) 110 | require.NoError(t, err, containerLogs) 111 | 112 | endpoint, err := nginx.PortEndpoint(t.Context(), "8080/tcp", "http") 113 | require.NoError(t, err, containerLogs) 114 | 115 | stdout := &bytes.Buffer{} 116 | 117 | wd, err := os.Getwd() 118 | require.NoError(t, err) 119 | 120 | moduleRoot, err := findModuleRoot(wd) 121 | require.NoError(t, err) 122 | 123 | go func() { 124 | returnCodeCh <- run(t.Context(), []string{ 125 | "--config=" + moduleRoot + "/packaging/etc/access-log-exporter/config.yaml", 126 | "--nginx.scrape-url=" + endpoint + "/stub_status", 127 | "--web.listen-address=127.0.0.1:54321", 128 | "--debug.enable=true", 129 | }, stdout, termCh) 130 | }() 131 | 132 | time.Sleep(1 * time.Second) 133 | 134 | t.Cleanup(func() { 135 | require.Equal(t, ReturnCodeOK, <-returnCodeCh, stdout.String()) 136 | }) 137 | 138 | for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} { 139 | for _, code := range []string{"200", "204", "404", "500"} { 140 | req, err := http.NewRequestWithContext(t.Context(), method, endpoint+"/"+code, nil) 141 | require.NoError(t, err) 142 | 143 | resp, err := http.DefaultClient.Do(req) 144 | require.NoError(t, err) 145 | 146 | _, err = io.Copy(io.Discard, resp.Body) 147 | require.NoError(t, err) 148 | 149 | err = resp.Body.Close() 150 | require.NoError(t, err) 151 | 152 | req, err = http.NewRequestWithContext(t.Context(), method, endpoint+"/proxy/"+code, nil) 153 | require.NoError(t, err) 154 | 155 | resp, err = http.DefaultClient.Do(req) 156 | require.NoError(t, err) 157 | 158 | _, err = io.Copy(io.Discard, resp.Body) 159 | require.NoError(t, err) 160 | 161 | err = resp.Body.Close() 162 | require.NoError(t, err) 163 | } 164 | } 165 | 166 | time.Sleep(1 * time.Second) 167 | 168 | req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://127.0.0.1:54321/metrics", nil) 169 | require.NoError(t, err) 170 | 171 | resp, err := http.DefaultClient.Do(req) 172 | require.NoError(t, err) 173 | 174 | body, err := io.ReadAll(resp.Body) 175 | require.NoError(t, err) 176 | 177 | err = resp.Body.Close() 178 | require.NoError(t, err) 179 | 180 | metrics := strings.TrimSpace(string(body)) 181 | 182 | time.Sleep(1 * time.Second) // Wait for the exporter to process the logs 183 | 184 | require.Equal(t, 1, strings.Count(metrics, "log_parse_errors_total 0"), metrics) 185 | require.Equal(t, 448, strings.Count(metrics, "http_request_duration_seconds_"), metrics) 186 | require.Equal(t, 322, strings.Count(metrics, "http_request_size_bytes"), metrics) 187 | require.Equal(t, 34, strings.Count(metrics, "http_requests_completed_total"), metrics) 188 | require.Equal(t, 34, strings.Count(metrics, "http_requests_total"), metrics) 189 | require.Equal(t, 320, strings.Count(metrics, "http_response_size_bytes_"), metrics) 190 | require.Equal(t, 21, strings.Count(metrics, "nginx_"), metrics) 191 | 192 | termCh <- syscall.SIGTERM 193 | } 194 | 195 | func findModuleRoot(start string) (string, error) { 196 | dir := start 197 | for { 198 | if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { 199 | return dir, nil 200 | } 201 | 202 | parent := filepath.Dir(dir) 203 | if parent == dir { 204 | return "", errors.New("go.mod not found") 205 | } 206 | 207 | dir = parent 208 | } 209 | } 210 | 211 | func getContainerLogs(t *testing.T, ctr testcontainers.Container) (string, error) { 212 | t.Helper() 213 | 214 | cli, err := testcontainers.NewDockerClientWithOpts(t.Context()) 215 | if err != nil { 216 | return "", fmt.Errorf("failed to create Docker client: %w", err) 217 | } 218 | 219 | logReader, err := cli.ContainerLogs(t.Context(), ctr.GetContainerID(), container.LogsOptions{ 220 | ShowStderr: true, 221 | ShowStdout: true, 222 | }) 223 | if err != nil { 224 | return "", fmt.Errorf("failed to get container logs: %w", err) 225 | } 226 | 227 | containerLogs, err := io.ReadAll(logReader) 228 | if err != nil { 229 | return "", fmt.Errorf("error reading container logs: %w", err) 230 | } 231 | 232 | return string(containerLogs), nil 233 | } 234 | -------------------------------------------------------------------------------- /cmd/access-log-exporter/main.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: Apache-2.0 2 | // 3 | // Copyright Jan-Otto Kröpke 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "flag" 22 | "fmt" 23 | "io" 24 | "log/slog" 25 | "net/http" 26 | "net/http/pprof" 27 | "os" 28 | "os/signal" 29 | "runtime" 30 | "runtime/debug" 31 | "sync" 32 | "syscall" 33 | "time" 34 | 35 | "github.com/KimMachineGun/automemlimit/memlimit" 36 | "github.com/jkroepke/access-log-exporter/internal/collector" 37 | "github.com/jkroepke/access-log-exporter/internal/config" 38 | "github.com/jkroepke/access-log-exporter/internal/nginx" 39 | "github.com/jkroepke/access-log-exporter/internal/syslog" 40 | "github.com/prometheus/client_golang/prometheus" 41 | "github.com/prometheus/client_golang/prometheus/collectors" 42 | versioncollector "github.com/prometheus/client_golang/prometheus/collectors/version" 43 | "github.com/prometheus/client_golang/prometheus/promhttp" 44 | "github.com/prometheus/common/version" 45 | ) 46 | 47 | type ReturnCode = int 48 | 49 | const ( 50 | // ReturnCodeNoError indicates that the program should continue running. 51 | ReturnCodeNoError ReturnCode = -2 52 | // ReturnCodeReload indicates that the configuration should be reloaded. 53 | ReturnCodeReload ReturnCode = -1 54 | // ReturnCodeOK indicates a successful execution of the program. 55 | ReturnCodeOK ReturnCode = 0 56 | // ReturnCodeError indicates an error during execution. 57 | ReturnCodeError ReturnCode = 1 58 | ) 59 | 60 | var ErrReload = errors.New("reload") 61 | 62 | func main() { 63 | termCh := make(chan os.Signal, 1) 64 | signal.Notify(termCh, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGUSR1) 65 | 66 | os.Exit(execute(os.Args, os.Stdout, termCh)) //nolint:forbidigo // entry point 67 | } 68 | 69 | // execute is the main entry point for the daemon. 70 | func execute(args []string, stdout io.Writer, termCh <-chan os.Signal) int { 71 | ctx := context.Background() 72 | 73 | for { 74 | if returnCode := run(ctx, args, stdout, termCh); returnCode != ReturnCodeReload { 75 | return returnCode 76 | } 77 | } 78 | } 79 | 80 | // run runs the main program logic of the daemon. 81 | // 82 | //nolint:cyclop,gocognit 83 | func run(ctx context.Context, args []string, stdout io.Writer, termCh <-chan os.Signal) ReturnCode { 84 | conf, logger, rc := initializeConfigAndLogger(args, stdout) 85 | if rc != ReturnCodeNoError { 86 | return rc 87 | } 88 | 89 | // initialize the root context with a cancel function 90 | ctx, cancel := context.WithCancelCause(ctx) 91 | defer cancel(nil) 92 | 93 | logger.LogAttrs(ctx, slog.LevelDebug, "config", slog.String("config", conf.String())) 94 | 95 | if conf.VerifyConfig { 96 | return ReturnCodeOK 97 | } 98 | 99 | _, err := memlimit.SetGoMemLimitWithOpts( 100 | memlimit.WithLogger(logger), 101 | ) 102 | if err != nil { 103 | logger.LogAttrs(ctx, slog.LevelWarn, "error setting GOMEMLIMIT", slog.Any("error", err)) 104 | } 105 | 106 | syslogMessageBuffer := make(chan string, conf.BufferSize) 107 | 108 | syslogServer, err := syslog.New(ctx, logger, conf.Syslog.ListenAddress, syslogMessageBuffer) 109 | if err != nil { 110 | logger.LogAttrs(ctx, slog.LevelError, "error creating syslog server", slog.Any("error", err)) 111 | 112 | return ReturnCodeError 113 | } 114 | 115 | go func() { 116 | logger.InfoContext(ctx, "syslog server started", slog.String("address", conf.Syslog.ListenAddress)) 117 | 118 | cancel(syslogServer.Start()) 119 | }() 120 | 121 | prometheusCollector, err := collector.New(ctx, logger, conf.Presets[conf.Preset], conf.WorkerCount, syslogMessageBuffer) 122 | if err != nil { 123 | logger.LogAttrs(ctx, slog.LevelError, "error creating collector", slog.Any("error", err)) 124 | 125 | return ReturnCodeError 126 | } 127 | 128 | reg := setupPrometheusRegistry(conf, logger, prometheusCollector) 129 | server := setupServer(conf, logger, reg) 130 | 131 | wg := &sync.WaitGroup{} 132 | defer wg.Wait() 133 | 134 | wg.Go(func() { 135 | if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { 136 | cancel(err) 137 | } 138 | }) 139 | 140 | for { 141 | select { 142 | case <-ctx.Done(): 143 | err := syslogServer.Close(ctx) 144 | if err != nil { 145 | logger.ErrorContext(ctx, "error shutting down syslog server", 146 | slog.String("address", conf.Syslog.ListenAddress), 147 | slog.Any("error", err), 148 | ) 149 | } 150 | 151 | prometheusCollector.Close() 152 | 153 | logger.InfoContext(ctx, "shutting down syslog server", 154 | slog.String("address", conf.Syslog.ListenAddress), 155 | ) 156 | 157 | close(syslogMessageBuffer) 158 | 159 | serverShutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 160 | 161 | server.RegisterOnShutdown(cancel) 162 | 163 | //nolint:contextcheck 164 | if err := server.Shutdown(serverShutdownCtx); err != nil { 165 | logger.LogAttrs(ctx, slog.LevelError, "error shutting down server", slog.Any("error", err)) 166 | } else { 167 | logger.LogAttrs(ctx, slog.LevelInfo, "server shutdown gracefully") 168 | } 169 | 170 | cancel() 171 | 172 | err = context.Cause(ctx) 173 | if err != nil { 174 | if errors.Is(err, context.Canceled) { 175 | return ReturnCodeOK 176 | } 177 | 178 | if errors.Is(err, ErrReload) { 179 | return ReturnCodeReload 180 | } 181 | 182 | logger.ErrorContext(ctx, err.Error()) 183 | 184 | return ReturnCodeError 185 | } 186 | 187 | return ReturnCodeOK 188 | case sig := <-termCh: 189 | logger.LogAttrs(ctx, slog.LevelInfo, "receiving signal: "+sig.String()) 190 | 191 | switch sig { 192 | case syscall.SIGHUP: 193 | logger.LogAttrs(ctx, slog.LevelInfo, "reloading configuration") 194 | cancel(ErrReload) 195 | default: 196 | cancel(nil) 197 | } 198 | } 199 | } 200 | } 201 | 202 | func setupPrometheusRegistry(conf config.Config, logger *slog.Logger, prometheusCollector *collector.Collector) *prometheus.Registry { 203 | prometheus.DefaultGatherer = nil // Disable default gatherer to avoid conflicts with custom registry 204 | prometheus.DefaultRegisterer = nil // Disable default registerer to avoid conflicts with custom registry 205 | 206 | reg := prometheus.NewRegistry() 207 | reg.MustRegister( 208 | collectors.NewGoCollector(), 209 | collectors.NewBuildInfoCollector(), 210 | collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), 211 | versioncollector.NewCollector("access_log_exporter"), 212 | prometheusCollector, 213 | ) 214 | 215 | if !conf.Nginx.ScrapeURL.IsEmpty() { 216 | reg.MustRegister(nginx.New(logger, conf.Nginx.ScrapeURL.String())) 217 | } 218 | 219 | return reg 220 | } 221 | 222 | // setupServer initializes the HTTP server with the given configuration and logger. 223 | func setupServer(conf config.Config, logger *slog.Logger, reg *prometheus.Registry) *http.Server { 224 | mux := http.NewServeMux() 225 | mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { 226 | w.WriteHeader(http.StatusOK) 227 | }) 228 | 229 | mux.Handle("GET /metrics", promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor( 230 | prometheus.Gatherers{reg}, 231 | promhttp.HandlerOpts{ 232 | ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), 233 | ErrorHandling: promhttp.ContinueOnError, 234 | Registry: reg, 235 | EnableOpenMetrics: true, 236 | }, 237 | ))) 238 | 239 | // Start debug listener if enabled 240 | if conf.Debug.Enable { 241 | mux.Handle("GET /", http.RedirectHandler("/debug/pprof/", http.StatusTemporaryRedirect)) 242 | mux.HandleFunc("GET /debug/pprof/", pprof.Index) 243 | mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline) 244 | mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile) 245 | mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol) 246 | mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace) 247 | } 248 | 249 | server := &http.Server{ 250 | Addr: conf.Web.ListenAddress, 251 | ReadHeaderTimeout: 3 * time.Second, 252 | ReadTimeout: 3 * time.Second, 253 | WriteTimeout: 10 * time.Second, 254 | ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), 255 | Handler: mux, 256 | } 257 | 258 | return server 259 | } 260 | 261 | // initializeConfigAndLogger handles configuration parsing and logger setup. 262 | func initializeConfigAndLogger(args []string, stdout io.Writer) (config.Config, *slog.Logger, ReturnCode) { 263 | conf, err := setupConfiguration(args, stdout) 264 | if err != nil { 265 | if errors.Is(err, flag.ErrHelp) { 266 | return config.Config{}, nil, ReturnCodeOK 267 | } 268 | 269 | if errors.Is(err, config.ErrVersion) { 270 | printVersion(stdout) 271 | 272 | return config.Config{}, nil, ReturnCodeOK 273 | } 274 | 275 | _, _ = fmt.Fprintln(stdout, err.Error()) 276 | 277 | return config.Config{}, nil, ReturnCodeError 278 | } 279 | 280 | logger, err := setupLogger(conf, stdout) 281 | if err != nil { 282 | _, _ = fmt.Fprintln(stdout, fmt.Errorf("error setup logging: %w", err).Error()) 283 | 284 | return config.Config{}, nil, ReturnCodeError 285 | } 286 | 287 | return conf, logger, ReturnCodeNoError 288 | } 289 | 290 | // setupConfiguration parses the command line arguments and loads the configuration. 291 | func setupConfiguration(args []string, logWriter io.Writer) (config.Config, error) { 292 | conf, err := config.New(args, logWriter) 293 | if err != nil { 294 | return config.Config{}, fmt.Errorf("configuration error: %w", err) 295 | } 296 | 297 | if err = config.Validate(conf); err != nil { 298 | return config.Config{}, fmt.Errorf("configuration validation error: %w", err) 299 | } 300 | 301 | return conf, nil 302 | } 303 | 304 | func printVersion(writer io.Writer) { 305 | //goland:noinspection GoBoolExpressions 306 | if version.Version == "" { 307 | if buildInfo, ok := debug.ReadBuildInfo(); ok { 308 | _, _ = fmt.Fprintf(writer, "version: %s\ncommit: %v\ngo: %s\n", buildInfo.Main.Version, version.GetRevision(), buildInfo.GoVersion) 309 | 310 | return 311 | } 312 | } 313 | 314 | _, _ = fmt.Fprintf(writer, "version: %s\ncommit: %s\ndate: %s\ngo: %s\n", version.Version, version.GetRevision(), version.BuildDate, runtime.Version()) 315 | } 316 | 317 | // setupLogger initializes the logger based on the configuration. 318 | func setupLogger(conf config.Config, writer io.Writer) (*slog.Logger, error) { 319 | opts := &slog.HandlerOptions{ 320 | AddSource: false, 321 | Level: conf.Log.Level, 322 | } 323 | 324 | switch conf.Log.Format { 325 | case "json": 326 | return slog.New(slog.NewJSONHandler(writer, opts)), nil 327 | case "console": 328 | return slog.New(slog.NewTextHandler(writer, opts)), nil 329 | default: 330 | return nil, fmt.Errorf("unknown log format: %s", conf.Log.Format) 331 | } 332 | } 333 | -------------------------------------------------------------------------------- /cmd/access-log-exporter/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestHelpFlag(t *testing.T) { 12 | t.Parallel() 13 | 14 | stdout := &bytes.Buffer{} 15 | 16 | rt := run(t.Context(), []string{"access-log-exporter", "--help"}, stdout, nil) 17 | require.Equal(t, ReturnCodeOK, rt, stdout) 18 | require.Contains(t, stdout.String(), "Documentation available at") 19 | } 20 | 21 | func TestVersionFlag(t *testing.T) { 22 | t.Parallel() 23 | 24 | stdout := &bytes.Buffer{} 25 | 26 | rt := run(t.Context(), []string{"access-log-exporter", "--version"}, stdout, nil) 27 | require.Equal(t, ReturnCodeOK, rt, stdout) 28 | require.Contains(t, stdout.String(), "version") 29 | } 30 | 31 | func TestConfigFileNotFound(t *testing.T) { 32 | t.Parallel() 33 | 34 | stdout := &bytes.Buffer{} 35 | 36 | rt := run(t.Context(), []string{"access-log-exporter", "--config=invalid"}, stdout, nil) 37 | require.Equal(t, ReturnCodeError, rt, stdout) 38 | require.Contains(t, stdout.String(), "error opening config file invalid") 39 | } 40 | 41 | func TestDefaultConfigFile(t *testing.T) { 42 | t.Parallel() 43 | 44 | stdout := &bytes.Buffer{} 45 | 46 | rt := run(t.Context(), []string{"access-log-exporter"}, stdout, nil) 47 | require.Equal(t, ReturnCodeError, rt, stdout) 48 | require.Contains(t, stdout.String(), "error opening config file config.yaml") 49 | } 50 | 51 | func TestEmptyConfigFile(t *testing.T) { 52 | stdout := &bytes.Buffer{} 53 | 54 | createTemp, err := os.CreateTemp(t.TempDir(), "config.yaml") 55 | require.NoError(t, err) 56 | 57 | t.Setenv("CONFIG_FILE", createTemp.Name()) 58 | 59 | t.Cleanup(func() { 60 | require.NoError(t, createTemp.Close()) 61 | }) 62 | 63 | rt := run(t.Context(), []string{"access-log-exporter"}, stdout, nil) 64 | require.Equal(t, ReturnCodeError, rt, stdout) 65 | require.Contains(t, stdout.String(), "configuration file is empty") 66 | } 67 | 68 | func TestInvalidPreset(t *testing.T) { 69 | t.Parallel() 70 | 71 | stdout := &bytes.Buffer{} 72 | 73 | wd, err := os.Getwd() 74 | require.NoError(t, err) 75 | 76 | moduleRoot, err := findModuleRoot(wd) 77 | require.NoError(t, err) 78 | 79 | returnCode := run(t.Context(), []string{ 80 | "access-log-exporter", 81 | "--config=" + moduleRoot + "/packaging/etc/access-log-exporter/config.yaml", 82 | "--preset", "invalid", 83 | }, stdout, nil) 84 | require.Equal(t, ReturnCodeError, returnCode, stdout) 85 | require.Contains(t, stdout.String(), "preset 'invalid' not found in configuration") 86 | } 87 | 88 | func TestInvalidLogFormat(t *testing.T) { 89 | t.Parallel() 90 | 91 | stdout := &bytes.Buffer{} 92 | 93 | wd, err := os.Getwd() 94 | require.NoError(t, err) 95 | 96 | moduleRoot, err := findModuleRoot(wd) 97 | require.NoError(t, err) 98 | 99 | returnCode := run(t.Context(), []string{ 100 | "access-log-exporter", 101 | "--config=" + moduleRoot + "/packaging/etc/access-log-exporter/config.yaml", 102 | "--log.format", "invalid", 103 | }, stdout, nil) 104 | require.Equal(t, ReturnCodeError, returnCode, stdout) 105 | require.Contains(t, stdout.String(), "unknown log format: invalid") 106 | } 107 | 108 | func TestVerifyConfig(t *testing.T) { 109 | t.Parallel() 110 | 111 | stdout := &bytes.Buffer{} 112 | 113 | wd, err := os.Getwd() 114 | require.NoError(t, err) 115 | 116 | moduleRoot, err := findModuleRoot(wd) 117 | require.NoError(t, err) 118 | 119 | returnCode := run(t.Context(), []string{ 120 | "access-log-exporter", 121 | "--config=" + moduleRoot + "/packaging/etc/access-log-exporter/config.yaml", 122 | "--log.format=json", 123 | "--verify-config", 124 | }, stdout, nil) 125 | require.Equal(t, ReturnCodeOK, returnCode, stdout) 126 | } 127 | -------------------------------------------------------------------------------- /docs/Grafana.md: -------------------------------------------------------------------------------- 1 | # Grafana Integration 2 | 3 | This guide explains how to visualize access-log-exporter metrics in Grafana. 4 | 5 | ## Quick Setup 6 | 7 | ### 1. Import Dashboard 8 | 9 | access-log-exporter includes a pre-built Grafana dashboard for visualizing web server metrics. 10 | 11 | **Import the dashboard:** 12 | 13 | 1. Download the dashboard JSON: [grafana-dashboard.json](https://github.com/jkroepke/access-log-exporter/blob/main/contrib/grafana-dashboard.json) 14 | 2. In Grafana, go to **Dashboards** → **Import** 15 | 3. Upload the JSON file or paste the content 16 | 4. Click **Import** 17 | 18 | ### 2. Dashboard Features 19 | 20 | The included dashboard provides: 21 | 22 | - **Request Rate**: Requests per second by host and status 23 | - **Response Times**: P50, P95, P99 response time percentiles 24 | - **Error Rates**: 4xx and 5xx error percentages 25 | - **Traffic Volume**: Request/response size metrics 26 | - **Upstream Performance**: Connection and response times (for upstream presets) 27 | - **URI Analytics**: Top endpoints by traffic (for `simple_uri_upstream` preset) 28 | 29 | ## Demo Setup 30 | 31 | For a complete demo environment with Grafana, see the demo configuration in `docs/demo/`: 32 | 33 | ```bash 34 | cd docs/demo 35 | docker-compose up -d 36 | ``` 37 | 38 | This starts: 39 | - access-log-exporter on port 4040 40 | - Nginx with sample traffic on port 8080 41 | - Prometheus on port 9090 42 | - Grafana on port 3000 43 | 44 | The dashboard will be automatically imported and configured. 45 | -------------------------------------------------------------------------------- /docs/Home.md: -------------------------------------------------------------------------------- 1 | # Home 2 | 3 | Welcome to the access-log-exporter wiki! 4 | 5 | ## Links 6 | 7 | 1. [Installation](Installation) 8 | 2. [Configuration](Configuration) 9 | 3. [Webserver Configuration](Webserver) 10 | 4. [Prometheus](Prometheus) 11 | 4. [Grafana](Grafana) 12 | -------------------------------------------------------------------------------- /docs/Installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | This document provides detailed instructions for installing `access-log-exporter`. 4 | 5 | ## Installing via Linux Packages 6 | 7 | DEB/RPM packages for Linux distributions exist. You can download the latest package from the [release page](https://github.com/jkroepke/access-log-exporter/releases). 8 | 9 | ### For Debian-based distributions: 10 | 11 | `access-log-exporter` provides an APT repository for Debian-based distributions. Add the repository and install the package using the following commands: 12 | 13 | ```bash 14 | curl -L https://raw.githubusercontent.com/jkroepke/access-log-exporter/refs/heads/main/packaging/apt/access-log-exporter.sources | sudo tee /etc/apt/sources.list.d/access-log-exporter.sources 15 | sudo apt update 16 | sudo apt install access-log-exporter 17 | ``` 18 | 19 | Note: The APT repository contains only the latest release. 20 | To pin a specific version, use `https://github.com/jkroepke/access-log-exporter/releases/download/vX.Y.Z` as URIs in the sources file. 21 | 22 | **Alternatively, you can install the DEB package manually:** 23 | 24 | 1. Download the DEB package from the releases page. 25 | 2. Open a terminal. 26 | 3. Navigate to the directory where you downloaded the package. 27 | 4. Install the package using the following command: 28 | 29 | ```bash 30 | sudo dpkg -i .deb 31 | ``` 32 | 33 | Replace `` with the name of the downloaded file. 34 | 35 | ### For RedHat-based distributions: 36 | 37 | 1. Download the RPM package from the releases page. 38 | 2. Open a terminal. 39 | 3. Navigate to the directory where you downloaded the package. 40 | 4. Install the package using the following command: 41 | 42 | ```bash 43 | sudo yum localinstall .rpm 44 | ``` 45 | 46 | Replace `` with the name of the downloaded file. 47 | 48 | ## Running as Kubernetes Sidecar 49 | 50 | When using Kubernetes, you can run `access-log-exporter` as a sidecar container in your pod. This allows it to access the logs of your main application container. 51 | 52 | To do this, add the following configuration to your pod's YAML file. The configuration varies slightly depending on your Kubernetes version. 53 | 54 | ### Kubernetes 1.33 and higher: Run as a sidecar container 55 | 56 |
57 | 58 | Click to expand 59 | 60 | ```yaml 61 | apiVersion: apps/v1 62 | kind: Deployment 63 | metadata: 64 | name: nginx 65 | spec: 66 | replicas: 1 67 | selector: 68 | matchLabels: 69 | app: nginx 70 | template: 71 | metadata: 72 | labels: 73 | app: nginx 74 | spec: 75 | initContainers: 76 | - name: access-log-exporter 77 | image: ghcr.io/jkroepke/access-log-exporter:latest 78 | ports: 79 | - containerPort: 4040 80 | name: metrics 81 | - containerPort: 8514 82 | name: syslog 83 | restartPolicy: Always 84 | securityContext: 85 | allowPrivilegeEscalation: false 86 | readOnlyRootFilesystem: true 87 | capabilities: 88 | drop: ["all"] 89 | privileged: false 90 | runAsNonRoot: true 91 | runAsUser: 65532 92 | runAsGroup: 65532 93 | 94 | containers: 95 | - name: nginx 96 | image: nginx:latest 97 | ``` 98 | 99 |
100 | 101 | For more information on how to configure the sidecar container, refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/). 102 | 103 | ### Kubernetes 1.32 and lower: Run as an extra container 104 | 105 |
106 | 107 | Click to expand 108 | 109 | ```yaml 110 | apiVersion: apps/v1 111 | kind: Deployment 112 | metadata: 113 | name: nginx 114 | spec: 115 | replicas: 1 116 | selector: 117 | matchLabels: 118 | app: nginx 119 | template: 120 | metadata: 121 | labels: 122 | app: nginx 123 | spec: 124 | containers: 125 | - name: nginx 126 | image: nginx:latest 127 | 128 | - name: access-log-exporter 129 | image: ghcr.io/jkroepke/access-log-exporter:latest 130 | ports: 131 | - containerPort: 4040 132 | name: metrics 133 | - containerPort: 8514 134 | name: syslog 135 | securityContext: 136 | allowPrivilegeEscalation: false 137 | readOnlyRootFilesystem: true 138 | capabilities: 139 | drop: ["all"] 140 | privileged: false 141 | runAsNonRoot: true 142 | runAsUser: 65532 143 | runAsGroup: 65532 144 | ``` 145 | 146 |
147 | 148 | ## Manual Installation 149 | 150 | To build the binary yourself, follow these steps: 151 | 1. Ensure you have [Go](https://go.dev/doc/install) and Make installed on your system. 152 | 2. Download the source code from our [releases page](https://github.com/jkroepke/access-log-exporter/releases/latest). 153 | 3. Open a terminal. 154 | 4. Navigate to the directory where you downloaded the source code. 155 | 5. Build the binary using the following command: 156 | ```bash 157 | make build 158 | ``` 159 | This creates a binary file named access-log-exporter. 160 | 6. Move the `access-log-exporter` binary to /usr/bin/ using the following command: 161 | ```bash 162 | sudo mv access-log-exporter /usr/bin/ 163 | ``` 164 | 165 | 7. Verify the installation by checking the version: 166 | ```bash 167 | access-log-exporter --version 168 | ``` 169 | 170 | Continue with the [Configuration Guide](./Configuration.md) to set up your provider details. 171 | -------------------------------------------------------------------------------- /docs/Prometheus.md: -------------------------------------------------------------------------------- 1 | # Prometheus Integration 2 | 3 | This guide explains how to integrate access-log-exporter with Prometheus. 4 | 5 | ## Quick Setup 6 | 7 | ### 1. Start access-log-exporter 8 | 9 | ```bash 10 | access-log-exporter --preset simple 11 | ``` 12 | 13 | By default, metrics are exposed on `http://localhost:4040/metrics` 14 | 15 | ### 2. Configure Prometheus 16 | 17 | Add this to your `prometheus.yml`: 18 | 19 | ```yaml 20 | scrape_configs: 21 | - job_name: 'access-log-exporter' 22 | static_configs: 23 | - targets: ['localhost:4040'] 24 | scrape_interval: 30s 25 | ``` 26 | 27 | ### 3. Verify 28 | 29 | Check Prometheus targets at `http://your-prometheus:9090/targets` 30 | 31 | ## Available Metrics 32 | 33 | ### Basic Metrics (simple preset) 34 | - `http_requests_total` - Request counter with labels: host, method, status 35 | - `http_request_duration_seconds` - Response time histogram 36 | - `http_request_size_bytes` - Request size histogram 37 | - `http_response_size_bytes` - Response size histogram 38 | 39 | ### Upstream Metrics (simple_upstream preset) 40 | - `http_upstream_connect_duration_seconds` - Upstream connection time 41 | - `http_upstream_header_duration_seconds` - Upstream header time 42 | - `http_upstream_request_duration_seconds` - Upstream response time 43 | 44 | ### URI Tracking (simple_uri_upstream preset) 45 | All metrics include `request_uri` label with path normalization: 46 | 47 | ```prometheus 48 | http_requests_total{host="example.com",method="GET",status="200",path="/api/users/.+"} 49 | ``` 50 | 51 | ## Useful Queries 52 | 53 | ```promql 54 | # Requests per second 55 | rate(http_requests_total[5m]) 56 | 57 | # 95th percentile response time 58 | histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) 59 | 60 | # Error rate 61 | sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) 62 | ``` 63 | 64 | ## Docker Compose Example 65 | 66 | ```yaml 67 | version: '3.8' 68 | services: 69 | access-log-exporter: 70 | image: ghcr.io/jkroepke/access-log-exporter:latest 71 | ports: 72 | - "4040:4040" 73 | - "8514:8514/udp" 74 | 75 | prometheus: 76 | image: prom/prometheus:latest 77 | ports: 78 | - "9090:9090" 79 | volumes: 80 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 81 | ``` 82 | 83 | ## Troubleshooting 84 | 85 | - **No metrics**: Check if access-log-exporter is receiving logs from your web server 86 | - **High cardinality**: Use `simple_uri_upstream` preset for automatic path normalization 87 | - **Performance**: Use 30-60 second scrape intervals for web metrics 88 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation of access-log-exporter 2 | 3 | Please note, all Markdown files are synced with the access-log-exporter wiki to improve the readability of the pages. 4 | 5 | https://github.com/jkroepke/access-log-exporter/wiki 6 | -------------------------------------------------------------------------------- /docs/Webserver.md: -------------------------------------------------------------------------------- 1 | # Webserver Configuration 2 | 3 | The access-log-exporter works with various web servers, including Nginx and Apache. This guide provides instructions for setting up the exporter with these servers. 4 | 5 | By default, the exporter listens on port 8514/udp for incoming log messages in RFC3164 format. 6 | We recommend using UDP for sending logs to the exporter. 7 | 8 | access-log-exporter exclusively supports RFC3164 format for incoming logs. 9 | Nginx and Apache2, among others, support this format. 10 | 11 | The log format must use tab separation (`\t`) to avoid parsing issues. The exporter does not support JSON or other formats. 12 | 13 | ## nginx 14 | 15 | Nginx can generate access logs compatible with access-log-exporter. This requires defining a custom log format in the Nginx configuration file. 16 | 17 | Common locations for this file include `/etc/nginx/nginx.conf` or `/etc/nginx/conf.d/default.conf`, depending on the operating system. 18 | You can also create an additional configuration file in the `/etc/nginx/conf.d/` directory. 19 | 20 | access-log-exporter includes multiple presets. We recommend the `simple` or `simple_upstream` presets for most use cases. The `simple` preset logs the request method, status code, response time, and request/response sizes. The `simple_upstream` preset additionally logs upstream response times and status codes. 21 | 22 | To configure Nginx, add the following lines to the configuration file. 23 | 24 | ```nginx 25 | # Use only one of the presets below, depending on your needs. 26 | # simple preset 27 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent'; 28 | access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 29 | 30 | # simple_upstream preset 31 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent\t$upstream_addr\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time'; 32 | access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 33 | ``` 34 | 35 | References: 36 | - [Nginx Documentation: Logging](https://nginx.org/en/docs/http/ngx_http_log_module.html) 37 | - [Nginx Documentation: Syslog](https://nginx.org/en/docs/syslog.html) 38 | 39 | ### Exclude specific requests for access-log-exporter 40 | 41 | To exclude specific requests from logging by access-log-exporter, use the `if` directive in Nginx. For example, to exclude requests to `/health` and `/metrics`, add the following lines: 42 | 43 | ```nginx 44 | # Exclude specific requests from logging 45 | map $request_uri $loggable { 46 | default 1; 47 | ~^/health 0; 48 | ~^/metrics 0; 49 | } 50 | 51 | # Use only one of the presets below, depending on your needs. 52 | # simple preset with exclusion 53 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent'; 54 | access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname if=$loggable; 55 | 56 | # simple_upstream preset with exclusion 57 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent\t$upstream_addr\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time'; 58 | access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname if=$loggable; 59 | ``` 60 | 61 | ## Apache2 62 | 63 | Apache can generate access logs compatible with access-log-exporter. This requires the `mod_log_config` module to define a custom log format. Recording incoming and outgoing request sizes requires the `mod_logio` module. 64 | 65 | Add the following lines to the Apache configuration file. Common locations for this file include `/etc/apache2/apache2.conf` and `/etc/httpd/conf/httpd.conf`, depending on the operating system. 66 | 67 | Adjust the binary path for the logger command if your system uses a different location. 68 | 69 | ```apache 70 | # Configuration for the access-log-exporter 71 | LogFormat "%v\t%m\t%>s\tOK\t%{ms}T\t%I\t%O" accesslog_exporter 72 | CustomLog "|/usr/bin/logger --rfc3164 --server 127.0.0.1 --port 8514 --udp" accesslog_exporter 73 | ``` 74 | 75 | ### Important Considerations 76 | 77 | Apache does not natively log information about upstream servers. To track upstream response times or status codes, integrate additional modules or external tools. 78 | -------------------------------------------------------------------------------- /docs/_Footer.md: -------------------------------------------------------------------------------- 1 | 2 | This wiki is synced with the [`docs`](https://github.com/jkroepke/access-log-exporter/tree/main/docs) folder from the code repository! To improve the wiki, create a [pull request](https://github.com/jkroepke/access-log-exporter/pulls) against the code repository with the suggested changes. 3 | -------------------------------------------------------------------------------- /docs/demo/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | nginx: 3 | image: nginx:1-alpine-slim 4 | healthcheck: 5 | test: [ "CMD", "wget", "-q", "http://127.0.0.1:8090/stub_status","-O","/dev/null" ] 6 | interval: 30s 7 | timeout: 10s 8 | retries: 3 9 | start_period: 5s 10 | depends_on: 11 | - httpbin 12 | - access-log-exporter 13 | environment: 14 | NGINX_ENTRYPOINT_QUIET_LOGS: "1" 15 | ports: 16 | - "8090:8090/tcp" 17 | networks: 18 | access-log-exporter: 19 | aliases: 20 | - nginx 21 | volumes: 22 | - ./nginx.conf:/etc/nginx/nginx.conf 23 | httpbin: 24 | image: mccutchen/go-httpbin 25 | networks: 26 | access-log-exporter: 27 | aliases: 28 | - httpbin 29 | environment: 30 | PORT: 8080 31 | access-log-exporter: 32 | image: ghcr.io/jkroepke/access-log-exporter:latest 33 | networks: 34 | access-log-exporter: 35 | aliases: 36 | - access-log-exporter 37 | ports: 38 | - "4040:4040/tcp" 39 | command: 40 | - --preset=simple_uri_upstream 41 | - --nginx.scrape-url=http://nginx:8090/stub_status 42 | load-generator: 43 | image: golang:1.25 44 | networks: 45 | access-log-exporter: 46 | depends_on: 47 | - nginx 48 | environment: 49 | BASE_URL: http://nginx:8090 50 | command: 51 | - go 52 | - run 53 | - /main.go 54 | volumes: 55 | - ./load-generator/main.go:/main.go 56 | prometheus: 57 | image: prom/prometheus 58 | depends_on: 59 | - access-log-exporter 60 | networks: 61 | access-log-exporter: 62 | aliases: 63 | - prometheus 64 | ports: 65 | - "9090:9090/tcp" 66 | command: 67 | - --config.file=/etc/prometheus/prometheus.yml 68 | - --web.listen-address=:9090 69 | volumes: 70 | - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml 71 | - prometheus-data:/prometheus 72 | grafana: 73 | image: grafana/grafana 74 | networks: 75 | access-log-exporter: 76 | aliases: 77 | - grafana 78 | ports: 79 | - "3000:3000/tcp" 80 | environment: 81 | - GF_AUTH_ANONYMOUS_ENABLED=true 82 | - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin 83 | volumes: 84 | - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml 85 | - ./grafana/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml 86 | - ./../../contrib/grafana-dashboard.json:/var/lib/grafana/dashboards/access-log-exporter.json 87 | networks: 88 | access-log-exporter: 89 | driver: bridge 90 | 91 | volumes: 92 | prometheus-data: 93 | -------------------------------------------------------------------------------- /docs/demo/grafana/dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: Default 5 | type: file 6 | options: 7 | path: /var/lib/grafana/dashboards 8 | -------------------------------------------------------------------------------- /docs/demo/grafana/datasource.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jkroepke/access-log-exporter/21ffa5960507045b0054b7616a4238d9dea2d789/docs/demo/grafana/datasource.yaml -------------------------------------------------------------------------------- /docs/demo/nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | pid /run/nginx.pid; 5 | 6 | events { 7 | worker_connections 1024; 8 | } 9 | 10 | http { 11 | log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent\t$upstream_addr\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time\t$request_uri'; 12 | access_log syslog:server=access-log-exporter:8514,nohostname accesslog_exporter; 13 | 14 | server { 15 | listen 8090; 16 | server_name localhost; 17 | 18 | location = /direct/200 { 19 | return 200 "OK"; 20 | } 21 | location = /direct/204 { 22 | return 204 "No Content"; 23 | } 24 | location = /direct/404 { 25 | return 404 "Not Found"; 26 | } 27 | location = /direct/500 { 28 | return 500 "Internal Server Error"; 29 | } 30 | 31 | location = /stub_status { 32 | stub_status; 33 | 34 | server_tokens on; 35 | access_log off; 36 | } 37 | 38 | location /httpbin/ { 39 | proxy_pass http://httpbin:8080/; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /docs/demo/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | 5 | scrape_configs: 6 | - job_name: 'access-log-exporter' 7 | static_configs: 8 | - targets: ['access-log-exporter:4040'] 9 | scrape_interval: 15s 10 | metrics_path: /metrics 11 | scheme: http 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jkroepke/access-log-exporter 2 | 3 | go 1.25 4 | 5 | require ( 6 | github.com/KimMachineGun/automemlimit v0.7.4 7 | github.com/docker/docker v28.5.0+incompatible 8 | github.com/prometheus/client_golang v1.23.2 9 | github.com/prometheus/common v0.66.1 10 | github.com/stretchr/testify v1.11.1 11 | github.com/testcontainers/testcontainers-go v0.39.0 12 | github.com/ua-parser/uap-go v0.0.0-20250917011043-9c86a9b0f8f0 13 | go.yaml.in/yaml/v4 v4.0.0-rc.2 14 | golang.org/x/net v0.44.0 15 | ) 16 | 17 | require ( 18 | dario.cat/mergo v1.0.2 // indirect 19 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 20 | github.com/Microsoft/go-winio v0.6.2 // indirect 21 | github.com/beorn7/perks v1.0.1 // indirect 22 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 23 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 24 | github.com/containerd/errdefs v1.0.0 // indirect 25 | github.com/containerd/errdefs/pkg v0.3.0 // indirect 26 | github.com/containerd/log v0.1.0 // indirect 27 | github.com/containerd/platforms v0.2.1 // indirect 28 | github.com/cpuguy83/dockercfg v0.3.2 // indirect 29 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 30 | github.com/distribution/reference v0.6.0 // indirect 31 | github.com/docker/go-connections v0.6.0 // indirect 32 | github.com/docker/go-units v0.5.0 // indirect 33 | github.com/ebitengine/purego v0.9.0 // indirect 34 | github.com/felixge/httpsnoop v1.0.4 // indirect 35 | github.com/go-logr/logr v1.4.3 // indirect 36 | github.com/go-logr/stdr v1.2.2 // indirect 37 | github.com/go-ole/go-ole v1.3.0 // indirect 38 | github.com/google/uuid v1.6.0 // indirect 39 | github.com/hashicorp/golang-lru v1.0.2 // indirect 40 | github.com/klauspost/compress v1.18.0 // indirect 41 | github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 // indirect 42 | github.com/magiconair/properties v1.8.10 // indirect 43 | github.com/moby/docker-image-spec v1.3.1 // indirect 44 | github.com/moby/go-archive v0.1.0 // indirect 45 | github.com/moby/patternmatcher v0.6.0 // indirect 46 | github.com/moby/sys/sequential v0.6.0 // indirect 47 | github.com/moby/sys/user v0.4.0 // indirect 48 | github.com/moby/sys/userns v0.1.0 // indirect 49 | github.com/moby/term v0.5.2 // indirect 50 | github.com/morikuni/aec v1.0.0 // indirect 51 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 52 | github.com/opencontainers/go-digest v1.0.0 // indirect 53 | github.com/opencontainers/image-spec v1.1.1 // indirect 54 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 55 | github.com/pkg/errors v0.9.1 // indirect 56 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 57 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect 58 | github.com/prometheus/client_model v0.6.2 // indirect 59 | github.com/prometheus/procfs v0.17.0 // indirect 60 | github.com/shirou/gopsutil/v4 v4.25.9 // indirect 61 | github.com/sirupsen/logrus v1.9.3 // indirect 62 | github.com/tklauser/go-sysconf v0.3.15 // indirect 63 | github.com/tklauser/numcpus v0.10.0 // indirect 64 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 65 | go.opentelemetry.io/auto/sdk v1.2.1 // indirect 66 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect 67 | go.opentelemetry.io/otel v1.38.0 // indirect 68 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect 69 | go.opentelemetry.io/otel/metric v1.38.0 // indirect 70 | go.opentelemetry.io/otel/trace v1.38.0 // indirect 71 | go.opentelemetry.io/proto/otlp v1.8.0 // indirect 72 | go.yaml.in/yaml/v2 v2.4.3 // indirect 73 | golang.org/x/crypto v0.42.0 // indirect 74 | golang.org/x/sys v0.36.0 // indirect 75 | google.golang.org/protobuf v1.36.10 // indirect 76 | gopkg.in/yaml.v3 v3.0.1 // indirect 77 | ) 78 | -------------------------------------------------------------------------------- /internal/collector/collector.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log/slog" 7 | "sync" 8 | 9 | "github.com/jkroepke/access-log-exporter/internal/config" 10 | "github.com/jkroepke/access-log-exporter/internal/metric" 11 | "github.com/prometheus/client_golang/prometheus" 12 | ) 13 | 14 | func New(ctx context.Context, logger *slog.Logger, preset config.Preset, workerCount int, messageCh <-chan string) (*Collector, error) { 15 | var ( 16 | err error 17 | userAgent bool 18 | ) 19 | 20 | metrics := make([]*metric.Metric, len(preset.Metrics)) 21 | for i, metricConfig := range preset.Metrics { 22 | metrics[i], err = metric.New(metricConfig) 23 | if err != nil { 24 | return nil, fmt.Errorf("could not create metric '%s': %w", metricConfig.Name, err) 25 | } 26 | 27 | for _, label := range metricConfig.Labels { 28 | if label.UserAgent { 29 | userAgent = true 30 | } 31 | } 32 | } 33 | 34 | if userAgent { 35 | logger.WarnContext(ctx, "The user agent parser is currently experimental and changed in the future or may not work as expected. "+ 36 | "Please report any issues you encounter.") 37 | } 38 | 39 | collector := &Collector{ 40 | wg: &sync.WaitGroup{}, 41 | metrics: metrics, 42 | metricLogParseError: prometheus.NewCounter(prometheus.CounterOpts{ 43 | Name: "log_parse_errors_total", 44 | Help: "Total number of parse errors", 45 | }), 46 | metricLogLastReceived: prometheus.NewGauge(prometheus.GaugeOpts{ 47 | Name: "log_last_received_timestamp_seconds", 48 | Help: "Timestamp of the last received log message in seconds since epoch", 49 | }), 50 | } 51 | 52 | collector.lineHandlerWorkers(ctx, logger, workerCount, messageCh) 53 | 54 | return collector, nil 55 | } 56 | 57 | // Describe implements the prometheus.Collector interface. 58 | func (c *Collector) Describe(ch chan<- *prometheus.Desc) { 59 | c.metricLogParseError.Describe(ch) 60 | 61 | for _, met := range c.metrics { 62 | met.Describe(ch) 63 | } 64 | } 65 | 66 | // Collect implements the prometheus.Collector interface. 67 | func (c *Collector) Collect(ch chan<- prometheus.Metric) { 68 | c.metricLogParseError.Collect(ch) 69 | 70 | for _, met := range c.metrics { 71 | met.Collect(ch) 72 | } 73 | } 74 | 75 | // Close stops the collector and waits for all workers to finish. 76 | func (c *Collector) Close() { 77 | c.wg.Wait() 78 | } 79 | -------------------------------------------------------------------------------- /internal/collector/line.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log/slog" 8 | "runtime" 9 | "strings" 10 | ) 11 | 12 | // lineHandlerWorkers starts several workers that will handle incoming 13 | // messages from the message channel. 14 | // Each worker will parse the incoming message and call the lineHandler method to process it. 15 | // The amount workers can be specified, and if less than or equal to zero, it defaults to the amount CPU cores available. 16 | func (c *Collector) lineHandlerWorkers(ctx context.Context, logger *slog.Logger, workerCount int, messageCh <-chan string) { 17 | if workerCount <= 0 { 18 | workerCount = runtime.NumCPU() 19 | } 20 | 21 | for range workerCount { 22 | c.wg.Add(1) 23 | 24 | go func() { 25 | defer c.wg.Done() 26 | 27 | c.lineHandlerWorker(ctx, logger, messageCh) 28 | }() 29 | } 30 | 31 | logger.InfoContext(ctx, "line handler started", slog.Int("workers", runtime.NumCPU())) 32 | } 33 | 34 | // lineHandlerWorker is a worker that will read messages from the message channel 35 | // and call the lineHandler method to process them. 36 | // It will log any errors that occur during parsing and increment the metricLogParseError. 37 | // The worker will stop when the context is done or when the message channel is closed. 38 | func (c *Collector) lineHandlerWorker(ctx context.Context, logger *slog.Logger, messageCh <-chan string) { 39 | var err error 40 | 41 | for { 42 | select { 43 | case <-ctx.Done(): 44 | return 45 | case msg, ok := <-messageCh: 46 | if !ok { 47 | return 48 | } 49 | 50 | c.metricLogLastReceived.SetToCurrentTime() 51 | 52 | err = c.lineHandler(strings.Split(msg, "\t")) 53 | if err != nil { 54 | logger.LogAttrs(ctx, slog.LevelDebug, "error parsing metric", 55 | slog.Any("err", err), 56 | slog.String("line", msg), 57 | ) 58 | 59 | c.metricLogParseError.Inc() 60 | } 61 | } 62 | } 63 | } 64 | 65 | // lineHandler processes a single line of log data. 66 | func (c *Collector) lineHandler(line []string) error { 67 | errs := make([]error, 0) 68 | 69 | for _, met := range c.metrics { 70 | err := met.Parse(line) 71 | if err != nil { 72 | errs = append(errs, fmt.Errorf("metric %s: %w", met.Name(), err)) 73 | } 74 | } 75 | 76 | if len(errs) != 0 { 77 | return errors.Join(errs...) 78 | } 79 | 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /internal/collector/types.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/jkroepke/access-log-exporter/internal/metric" 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | type Collector struct { 11 | metricLogParseError prometheus.Counter 12 | metricLogLastReceived prometheus.Gauge 13 | wg *sync.WaitGroup 14 | metrics []*metric.Metric 15 | } 16 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "os" 10 | "reflect" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | "go.yaml.in/yaml/v4" 16 | ) 17 | 18 | var ErrVersion = errors.New("flag: version requested") 19 | 20 | // New loads the configuration from configuration files, command line arguments and environment variables in that order. 21 | // 22 | //goland:noinspection GoMixedReceiverTypes 23 | func New(args []string, writer io.Writer) (Config, error) { 24 | config := Defaults 25 | 26 | if !lookupVersionOrHelpArgument(args) { 27 | configFilePath := lookupConfigArgument(args) 28 | if err := config.ReadFromConfigFile(configFilePath); err != nil { 29 | if errors.Is(err, io.EOF) { 30 | err = ErrEmptyConfigFile 31 | } 32 | 33 | return Config{}, err 34 | } 35 | } 36 | 37 | if err := config.ReadFromFlagAndEnvironment(args, writer); err != nil { 38 | return Config{}, err 39 | } 40 | 41 | return config, nil 42 | } 43 | 44 | // ReadFromConfigFile reads the configuration from a configuration file and command line arguments. 45 | // 46 | //goland:noinspection GoMixedReceiverTypes 47 | func (c *Config) ReadFromConfigFile(configFilePath string) error { 48 | configFile, err := os.Open(configFilePath) 49 | if err != nil { 50 | return fmt.Errorf("error opening config file %s: %w", configFilePath, err) 51 | } 52 | 53 | defer func() { 54 | _ = configFile.Close() 55 | }() 56 | 57 | decoder := yaml.NewDecoder(configFile) 58 | decoder.KnownFields(true) 59 | 60 | // Load the config file 61 | if err = decoder.Decode(c); err != nil { 62 | return fmt.Errorf("error decoding config file %s: %w", configFilePath, err) 63 | } 64 | 65 | return nil 66 | } 67 | 68 | // ReadFromFlagAndEnvironment reads the configuration from command line arguments and environment variables. 69 | // 70 | //goland:noinspection GoMixedReceiverTypes 71 | func (c *Config) ReadFromFlagAndEnvironment(args []string, writer io.Writer) error { 72 | // Load the c from command line arguments 73 | flagSet := flag.NewFlagSet("access-log-exporter", flag.ContinueOnError) 74 | flagSet.SetOutput(writer) 75 | flagSet.Usage = func() { 76 | _, _ = fmt.Fprint(flagSet.Output(), "Documentation available at https://github.com/jkroepke/access-log-exporter/wiki\r\n\r\n") 77 | _, _ = fmt.Fprint(flagSet.Output(), "Usage of access-log-exporter:\r\n\r\n") 78 | // --help should display options with double dash 79 | flagSet.VisitAll(func(flag *flag.Flag) { 80 | flag.Name = "-" + flag.Name 81 | }) 82 | flagSet.PrintDefaults() 83 | } 84 | 85 | c.flagSet(flagSet) 86 | 87 | flagSet.VisitAll(func(flag *flag.Flag) { 88 | if flag.Name == "version" { 89 | return 90 | } 91 | 92 | flag.Usage += fmt.Sprintf(" (env: %s)", getEnvironmentVariableByFlagName(flag.Name)) 93 | }) 94 | 95 | if err := flagSet.Parse(args[1:]); err != nil { 96 | return fmt.Errorf("error parsing command line arguments: %w", err) 97 | } 98 | 99 | if flagSet.Lookup("version").Value.String() == "true" { 100 | return ErrVersion 101 | } 102 | 103 | return nil 104 | } 105 | 106 | func lookupConfigArgument(args []string) string { 107 | for i, arg := range args { 108 | if !strings.HasPrefix(arg, "--config") { 109 | continue 110 | } 111 | 112 | if strings.HasPrefix(arg, "--config=") { 113 | return strings.TrimPrefix(arg, "--config=") 114 | } 115 | 116 | // check if the argument is --config without value and look for the next argument 117 | if len(args) > i+1 { 118 | return args[i+1] 119 | } 120 | } 121 | 122 | defaultConfigFilePath := "config.yaml" 123 | 124 | if koDataPath, ok := os.LookupEnv("KO_DATA_PATH"); ok { 125 | // If KO_DATA_PATH is set, use it as the config file path 126 | defaultConfigFilePath = koDataPath + "/config.yaml" 127 | } 128 | 129 | return lookupEnvOrDefault("config", defaultConfigFilePath) 130 | } 131 | 132 | func lookupVersionOrHelpArgument(args []string) bool { 133 | for _, arg := range args { 134 | switch arg { 135 | case "-h", "--help", "-help": 136 | return true 137 | case "-v", "--version": 138 | return true 139 | } 140 | } 141 | 142 | return false 143 | } 144 | 145 | // lookupEnvOrDefault looks up the environment variable by the flag name and returns the value. 146 | // If the environment variable is not set, it returns the default value. 147 | // It supports the following types: string, bool, int, uint, time.Duration and types implementing [encoding.TextUnmarshaler]. 148 | // If the type is not supported, it panics. 149 | // 150 | //nolint:cyclop 151 | func lookupEnvOrDefault[T any](key string, defaultValue T) T { 152 | envValue, ok := os.LookupEnv(getEnvironmentVariableByFlagName(key)) 153 | if !ok { 154 | return defaultValue 155 | } 156 | 157 | ok = false 158 | 159 | var value T 160 | 161 | switch any(defaultValue).(type) { 162 | case string: 163 | value, ok = any(envValue).(T) 164 | case bool: 165 | boolVal, err := strconv.ParseBool(envValue) 166 | if err != nil { 167 | return defaultValue 168 | } 169 | 170 | value, ok = any(boolVal).(T) 171 | case int: 172 | intValue, err := strconv.Atoi(envValue) 173 | if err != nil { 174 | return defaultValue 175 | } 176 | 177 | value, ok = any(intValue).(T) 178 | case uint: 179 | intValue, err := strconv.ParseUint(envValue, 10, 0) 180 | if err != nil { 181 | return defaultValue 182 | } 183 | 184 | value, ok = any(uint(intValue)).(T) 185 | case float64: 186 | floatValue, err := strconv.ParseFloat(envValue, 64) 187 | if err != nil { 188 | return defaultValue 189 | } 190 | 191 | value, ok = any(floatValue).(T) 192 | case time.Duration: 193 | durationValue, err := time.ParseDuration(envValue) 194 | if err != nil { 195 | return defaultValue 196 | } 197 | 198 | value, ok = any(durationValue).(T) 199 | default: 200 | // Handle types implementing encoding.TextUnmarshaler via reflection 201 | t := reflect.TypeOf(defaultValue) 202 | 203 | var valPtr reflect.Value 204 | 205 | if t.Kind() == reflect.Pointer { 206 | valPtr = reflect.New(t.Elem()) 207 | } else { 208 | valPtr = reflect.New(t) 209 | } 210 | 211 | if unmarshaler, okUnmarshal := valPtr.Interface().(encoding.TextUnmarshaler); okUnmarshal { 212 | if err := unmarshaler.UnmarshalText([]byte(envValue)); err != nil { 213 | return defaultValue 214 | } 215 | 216 | if t.Kind() == reflect.Pointer { 217 | value, ok = valPtr.Convert(t).Interface().(T) 218 | } else { 219 | value, ok = valPtr.Elem().Interface().(T) 220 | } 221 | } 222 | } 223 | 224 | if !ok { 225 | panic(fmt.Sprintf("failed to convert environment variable %s to type %T", key, defaultValue)) 226 | } 227 | 228 | return value 229 | } 230 | 231 | // getEnvironmentVariableByFlagName converts a flag name to an environment variable name. 232 | // It replaces all dots with underscores and all dashes with double underscores. 233 | // It also converts the flag name to uppercase. 234 | func getEnvironmentVariableByFlagName(flagName string) string { 235 | if flagName == "config" { 236 | return "CONFIG_FILE" 237 | } 238 | 239 | return "CONFIG_" + strings.ReplaceAll(strings.ReplaceAll(strings.ToUpper(flagName), ".", "_"), "-", "__") 240 | } 241 | -------------------------------------------------------------------------------- /internal/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/jkroepke/access-log-exporter/internal/config" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestConfig(t *testing.T) { 16 | t.Parallel() 17 | 18 | for _, tc := range []struct { 19 | name string 20 | configFile string 21 | conf config.Config 22 | err error 23 | }{ 24 | { 25 | "empty file", 26 | "", 27 | config.Defaults, 28 | config.ErrEmptyConfigFile, 29 | }, 30 | { 31 | "minimal file", 32 | // language=yaml 33 | ` 34 | web: 35 | listenAddress: ":9000" 36 | `, 37 | func() config.Config { 38 | conf := config.Defaults 39 | conf.Web.ListenAddress = ":9000" 40 | 41 | return conf 42 | }(), 43 | nil, 44 | }, 45 | } { 46 | t.Run(tc.name, func(t *testing.T) { 47 | t.Parallel() 48 | 49 | var buf bytes.Buffer 50 | 51 | _ = io.Writer(&buf) 52 | 53 | file, err := os.CreateTemp(t.TempDir(), "access-log-exporter-*") 54 | require.NoError(t, err) 55 | 56 | // close and remove the temporary file at the end of the program. 57 | t.Cleanup(func() { 58 | require.NoError(t, file.Close()) 59 | require.NoError(t, os.Remove(file.Name())) 60 | }) 61 | 62 | _, err = file.WriteString(tc.configFile) 63 | require.NoError(t, err) 64 | 65 | conf, err := config.New([]string{"access-log-exporter", "--config", file.Name()}, &buf) 66 | if tc.err != nil { 67 | require.Error(t, err) 68 | assert.Equal(t, tc.err.Error(), err.Error()) 69 | } else { 70 | require.NoError(t, err) 71 | assert.Equal(t, tc.conf, conf) 72 | } 73 | }) 74 | } 75 | } 76 | 77 | func TestConfigHelpFlag(t *testing.T) { 78 | t.Parallel() 79 | 80 | var buf bytes.Buffer 81 | 82 | _ = io.Writer(&buf) 83 | 84 | _, err := config.New([]string{"access-log-exporter", "--help"}, &buf) 85 | 86 | require.ErrorIs(t, err, flag.ErrHelp) 87 | } 88 | 89 | func TestConfigVersionFlag(t *testing.T) { 90 | t.Parallel() 91 | 92 | var buf bytes.Buffer 93 | 94 | _ = io.Writer(&buf) 95 | 96 | _, err := config.New([]string{"access-log-exporter", "--version"}, &buf) 97 | 98 | require.ErrorIs(t, err, config.ErrVersion) 99 | } 100 | -------------------------------------------------------------------------------- /internal/config/defaults.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "log/slog" 5 | ) 6 | 7 | //nolint:gochecknoglobals 8 | var Defaults = Config{ 9 | ConfigFile: "config.yaml", 10 | BufferSize: 1000, 11 | WorkerCount: 0, 12 | Preset: "simple", 13 | Debug: Debug{}, 14 | Log: Log{ 15 | Format: "console", 16 | Level: slog.LevelInfo, 17 | }, 18 | Web: Web{ 19 | ListenAddress: ":4040", 20 | }, 21 | Syslog: Syslog{ 22 | ListenAddress: "udp://[::]:8514", 23 | }, 24 | } 25 | -------------------------------------------------------------------------------- /internal/config/env_test.go: -------------------------------------------------------------------------------- 1 | package config //nolint:testpackage 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestLookupEnvOrDefault(t *testing.T) { 10 | for _, tc := range []struct { 11 | name string 12 | input string 13 | badInput string 14 | defaultValue any 15 | expected any 16 | panic bool 17 | }{ 18 | { 19 | name: "string", 20 | defaultValue: "test", 21 | input: "test2", 22 | expected: "test2", 23 | }, 24 | { 25 | name: "bool", 26 | defaultValue: false, 27 | input: "true", 28 | badInput: "A", 29 | expected: true, 30 | }, 31 | { 32 | name: "int", 33 | defaultValue: 1336, 34 | input: "1337", 35 | badInput: "A", 36 | expected: 1337, 37 | }, 38 | { 39 | name: "uint", 40 | defaultValue: uint(1336), 41 | input: "1337", 42 | badInput: "A", 43 | expected: uint(1337), 44 | }, 45 | { 46 | name: "float64", 47 | defaultValue: float64(1336), 48 | input: "1337", 49 | expected: float64(1337), 50 | }, 51 | { 52 | name: "float32", 53 | defaultValue: float32(1336), 54 | input: "1337", 55 | expected: float32(1337), 56 | panic: true, 57 | }, 58 | } { 59 | t.Run(tc.name, func(t *testing.T) { 60 | testFn := func() { 61 | require.Equal(t, tc.defaultValue, lookupEnvOrDefault("unset", tc.defaultValue)) 62 | 63 | t.Setenv("CONFIG_SET", tc.input) 64 | require.Equal(t, tc.expected, lookupEnvOrDefault("set", tc.defaultValue)) 65 | 66 | if tc.badInput != "" { 67 | t.Setenv("CONFIG_BAD", tc.badInput) 68 | require.Equal(t, tc.defaultValue, lookupEnvOrDefault("bad", tc.defaultValue)) 69 | } 70 | } 71 | 72 | if tc.panic { 73 | require.Panics(t, testFn) 74 | } else { 75 | require.NotPanics(t, testFn) 76 | } 77 | }) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/config/errors.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "errors" 4 | 5 | var ErrRequired = errors.New("required") 6 | -------------------------------------------------------------------------------- /internal/config/flags.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "flag" 5 | ) 6 | 7 | //goland:noinspection GoMixedReceiverTypes 8 | func (c *Config) flagSet(flagSet *flag.FlagSet) { 9 | flagSet.String( 10 | "config", 11 | lookupEnvOrDefault("config", "config.yaml"), 12 | "path to one .yaml config file", 13 | ) 14 | 15 | flagSet.Bool( 16 | "version", 17 | false, 18 | "show version", 19 | ) 20 | 21 | flagSet.BoolVar( 22 | &c.VerifyConfig, 23 | "verify-config", 24 | c.VerifyConfig, 25 | "Enable this flag to check config file loads, then exit", 26 | ) 27 | 28 | flagSet.UintVar( 29 | &c.BufferSize, 30 | "buffer-size", 31 | lookupEnvOrDefault("buffer_size", c.BufferSize), 32 | "Size of the buffer for syslog messages. Default is 1000. Set to 0 to disable buffering.", 33 | ) 34 | 35 | flagSet.IntVar( 36 | &c.WorkerCount, 37 | "worker", 38 | lookupEnvOrDefault("worker", c.WorkerCount), 39 | "Number of workers to process syslog messages. 0 or below means number of available CPU cores.", 40 | ) 41 | 42 | flagSet.StringVar( 43 | &c.Preset, 44 | "preset", 45 | lookupEnvOrDefault("preset", c.Preset), 46 | "Preset configuration to use. "+ 47 | "Available presets: simple, simple_upstream, simple_uri_upstream. "+ 48 | "Custom presets can be defined via config file.", 49 | ) 50 | 51 | c.flagSetLog(flagSet) 52 | c.flagSetNginx(flagSet) 53 | c.flagSetDebug(flagSet) 54 | c.flagSetWeb(flagSet) 55 | c.flagSetSyslog(flagSet) 56 | } 57 | 58 | //goland:noinspection GoMixedReceiverTypes 59 | func (c *Config) flagSetLog(flagSet *flag.FlagSet) { 60 | flagSet.StringVar( 61 | &c.Log.Format, 62 | "log.format", 63 | lookupEnvOrDefault("log.format", c.Log.Format), 64 | "log format. json or console", 65 | ) 66 | flagSet.TextVar( 67 | &c.Log.Level, 68 | "log.level", 69 | lookupEnvOrDefault("log.level", c.Log.Level), 70 | "log level. Can be one of: debug, info, warn, error", 71 | ) 72 | } 73 | 74 | //goland:noinspection GoMixedReceiverTypes 75 | func (c *Config) flagSetNginx(flagSet *flag.FlagSet) { 76 | flagSet.TextVar( 77 | &c.Nginx.ScrapeURL, 78 | "nginx.scrape-url", 79 | lookupEnvOrDefault("nginx.scrape-url", c.Nginx.ScrapeURL), 80 | "A URI or unix domain socket path for scraping NGINX metrics. "+ 81 | "For NGINX, the stub_status page must be available through the URI. Examples: http://127.0.0.1/stub_status", 82 | ) 83 | } 84 | 85 | //goland:noinspection GoMixedReceiverTypes 86 | func (c *Config) flagSetDebug(flagSet *flag.FlagSet) { 87 | flagSet.BoolVar( 88 | &c.Debug.Enable, 89 | "debug.enable", 90 | lookupEnvOrDefault("debug.enable", c.Debug.Enable), 91 | "Enables go profiling endpoint. This should be never exposed.", 92 | ) 93 | } 94 | 95 | //goland:noinspection GoMixedReceiverTypes 96 | func (c *Config) flagSetWeb(flagSet *flag.FlagSet) { 97 | flagSet.StringVar( 98 | &c.Web.ListenAddress, 99 | "web.listen-address", 100 | lookupEnvOrDefault("web.listen-address", c.Web.ListenAddress), 101 | "Addresses on which to expose metrics. Examples: `:4041` or `[::1]:4041` for http", 102 | ) 103 | } 104 | 105 | //goland:noinspection GoMixedReceiverTypes 106 | func (c *Config) flagSetSyslog(flagSet *flag.FlagSet) { 107 | flagSet.StringVar( 108 | &c.Syslog.ListenAddress, 109 | "syslog.listen-address", 110 | lookupEnvOrDefault("syslog.listen-address", c.Syslog.ListenAddress), 111 | "Addresses on which to expose syslog. Examples: udp://0.0.0.0:8514, tcp://0.0.0.0:8514, unix:///path/to/socket.", 112 | ) 113 | } 114 | -------------------------------------------------------------------------------- /internal/config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "log/slog" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/jkroepke/access-log-exporter/internal/config/types" 11 | "go.yaml.in/yaml/v4" 12 | ) 13 | 14 | var ErrEmptyConfigFile = errors.New("configuration file is empty") 15 | 16 | type Config struct { 17 | Presets Presets `json:"presets" yaml:"presets"` 18 | Nginx Nginx `json:"nginx" yaml:"nginx"` 19 | Web Web `json:"web" yaml:"web"` 20 | ConfigFile string `json:"config" yaml:"config"` 21 | Syslog Syslog `json:"syslog" yaml:"syslog"` 22 | Preset string `json:"preset" yaml:"preset"` 23 | Log Log `json:"log" yaml:"log"` 24 | WorkerCount int `json:"workerCount" yaml:"workerCount"` 25 | BufferSize uint `json:"bufferSize" yaml:"bufferSize"` 26 | Debug Debug `json:"debug" yaml:"debug"` 27 | VerifyConfig bool `json:"-"` 28 | } 29 | 30 | type Log struct { 31 | Format string `json:"format" yaml:"format"` 32 | Level slog.Level `json:"level" yaml:"level"` 33 | } 34 | 35 | type Syslog struct { 36 | ListenAddress string `json:"listenAddress" yaml:"listenAddress"` 37 | } 38 | 39 | type Debug struct { 40 | Enable bool `json:"enable" yaml:"enable"` 41 | } 42 | 43 | type Web struct { 44 | ListenAddress string `json:"listenAddress" yaml:"listenAddress"` 45 | } 46 | 47 | type Presets map[string]Preset 48 | 49 | type Preset struct { 50 | Metrics []Metric `json:"metrics" yaml:"metrics"` 51 | } 52 | 53 | type Metric struct { 54 | ConstLabels map[string]string `json:"constLabels" yaml:"constLabels"` 55 | ValueIndex *uint `json:"valueIndex,omitempty" yaml:"valueIndex,omitempty"` 56 | Name string `json:"name" yaml:"name"` 57 | Type string `json:"type" yaml:"type"` 58 | Help string `json:"help" yaml:"help"` 59 | Buckets types.Float64Slice `json:"buckets,omitempty" yaml:"buckets,omitempty"` 60 | Labels []Label `json:"labels" yaml:"labels"` 61 | Replacements []Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"` 62 | Upstream Upstream `json:"upstream" yaml:"upstream"` 63 | Math Math `json:"math" yaml:"math"` 64 | } 65 | 66 | type Math struct { 67 | Enabled bool `json:"enabled" yaml:"enabled"` 68 | Mul float64 `json:"mul" yaml:"mul"` 69 | Div float64 `json:"div" yaml:"div"` 70 | } 71 | 72 | type Upstream struct { 73 | Excludes []string `json:"excludes" yaml:"excludes"` 74 | AddrLineIndex uint `json:"addrLineIndex" yaml:"addrLineIndex"` 75 | Enabled bool `json:"enabled" yaml:"enabled"` 76 | Label bool `json:"label" yaml:"label"` 77 | } 78 | 79 | type Label struct { 80 | Name string `json:"name" yaml:"name"` 81 | Replacements []Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"` 82 | LineIndex uint `json:"lineIndex" yaml:"lineIndex"` 83 | UserAgent bool `json:"userAgent" yaml:"userAgent"` 84 | } 85 | 86 | type Replacement struct { 87 | String *string `json:"string,omitempty" yaml:"string,omitempty"` 88 | Regexp *regexp.Regexp `json:"regexp,omitempty" yaml:"regexp,omitempty"` 89 | StringReplacer *strings.Replacer `json:"-" yaml:"-"` 90 | Replacement string `json:"replacement" yaml:"replacement"` 91 | } 92 | 93 | type Nginx struct { 94 | ScrapeURL types.URL `json:"scrapeUri" yaml:"scrapeUri"` 95 | } 96 | 97 | //goland:noinspection GoMixedReceiverTypes 98 | func (c Config) String() string { 99 | jsonString, err := json.Marshal(c) 100 | if err != nil { 101 | panic(err) 102 | } 103 | 104 | return string(jsonString) 105 | } 106 | 107 | func (r *Replacement) UnmarshalYAML(data *yaml.Node) error { 108 | type Alias Replacement 109 | 110 | aux := Alias(*r) 111 | 112 | if err := data.Decode(&aux); err != nil { 113 | return err //nolint:wrapcheck 114 | } 115 | 116 | *r = Replacement(aux) 117 | 118 | if r.Regexp != nil && r.String != nil { 119 | return errors.New("replacement can not have both regexp and string") 120 | } 121 | 122 | if r.String != nil { 123 | r.StringReplacer = strings.NewReplacer(*r.String, r.Replacement) 124 | } 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /internal/config/types/slice.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "go.yaml.in/yaml/v4" 11 | ) 12 | 13 | type StringSlice []string 14 | 15 | // String returns the string representation of the URL. 16 | // 17 | //goland:noinspection GoMixedReceiverTypes 18 | func (s StringSlice) String() string { 19 | return strings.Join(s, ",") 20 | } 21 | 22 | // MarshalText implements [encoding.TextMarshaler] interface. 23 | // 24 | //goland:noinspection GoMixedReceiverTypes 25 | func (s StringSlice) MarshalText() ([]byte, error) { 26 | return []byte(s.String()), nil 27 | } 28 | 29 | // UnmarshalText implements the [encoding.TextUnmarshaler] interface. 30 | // 31 | //goland:noinspection GoMixedReceiverTypes 32 | func (s *StringSlice) UnmarshalText(text []byte) error { 33 | *s = strings.Split(string(text), ",") 34 | 35 | return nil 36 | } 37 | 38 | // UnmarshalJSON implements the [json.Unmarshaler] interface. 39 | // 40 | //goland:noinspection GoMixedReceiverTypes 41 | func (s *StringSlice) UnmarshalJSON(jsonBytes []byte) error { 42 | var slice []string 43 | 44 | err := json.NewDecoder(bytes.NewReader(jsonBytes)).Decode(&slice) 45 | 46 | *s = slice 47 | 48 | //nolint:wrapcheck 49 | return err 50 | } 51 | 52 | // UnmarshalYAML implements the [yaml.Unmarshaler] interface. 53 | // 54 | //goland:noinspection GoMixedReceiverTypes 55 | func (s *StringSlice) UnmarshalYAML(data *yaml.Node) error { 56 | var slice []string 57 | 58 | err := data.Decode(&slice) 59 | 60 | *s = slice 61 | 62 | //nolint:wrapcheck 63 | return err 64 | } 65 | 66 | type Float64Slice []float64 67 | 68 | // String returns the string representation of the URL. 69 | // 70 | //goland:noinspection GoMixedReceiverTypes 71 | func (s Float64Slice) String() string { 72 | stringSlice := make([]string, len(s)) 73 | 74 | for i, floatValue := range s { 75 | stringSlice[i] = strconv.FormatFloat(floatValue, 'g', -1, 64) 76 | } 77 | 78 | return strings.Join(stringSlice, ",") 79 | } 80 | 81 | // MarshalText implements [encoding.TextMarshaler] interface. 82 | // 83 | //goland:noinspection GoMixedReceiverTypes 84 | func (s Float64Slice) MarshalText() ([]byte, error) { 85 | return []byte(s.String()), nil 86 | } 87 | 88 | // UnmarshalText implements the [encoding.TextUnmarshaler] interface. 89 | // 90 | //goland:noinspection GoMixedReceiverTypes 91 | func (s *Float64Slice) UnmarshalText(text []byte) error { 92 | stringSlice := strings.Split(string(text), ",") 93 | floatSlice := make(Float64Slice, len(stringSlice)) 94 | 95 | var err error 96 | 97 | for i, stringFloat := range stringSlice { 98 | floatSlice[i], err = strconv.ParseFloat(stringFloat, 64) 99 | if err != nil { 100 | return fmt.Errorf("failed to parse float64 from string '%s': %w", stringFloat, err) 101 | } 102 | } 103 | 104 | *s = floatSlice 105 | 106 | return nil 107 | } 108 | 109 | // UnmarshalJSON implements the [json.Unmarshaler] interface. 110 | // 111 | //goland:noinspection GoMixedReceiverTypes 112 | func (s *Float64Slice) UnmarshalJSON(jsonBytes []byte) error { 113 | var slice []float64 114 | 115 | err := json.NewDecoder(bytes.NewReader(jsonBytes)).Decode(&slice) 116 | 117 | *s = slice 118 | 119 | //nolint:wrapcheck 120 | return err 121 | } 122 | 123 | // UnmarshalYAML implements the [yaml.Unmarshaler] interface. 124 | // 125 | //goland:noinspection GoMixedReceiverTypes 126 | func (s *Float64Slice) UnmarshalYAML(data *yaml.Node) error { 127 | var slice []float64 128 | 129 | err := data.Decode(&slice) 130 | 131 | *s = slice 132 | 133 | //nolint:wrapcheck 134 | return err 135 | } 136 | -------------------------------------------------------------------------------- /internal/config/types/slice_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/jkroepke/access-log-exporter/internal/config/types" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | "go.yaml.in/yaml/v4" 12 | ) 13 | 14 | func TestSliceUnmarshalText(t *testing.T) { 15 | t.Parallel() 16 | 17 | slice := types.StringSlice{} 18 | 19 | require.NoError(t, slice.UnmarshalText([]byte("a,b,c,d"))) 20 | 21 | assert.Equal(t, types.StringSlice{"a", "b", "c", "d"}, slice) 22 | } 23 | 24 | func TestSliceMarshalText(t *testing.T) { 25 | t.Parallel() 26 | 27 | slice, err := types.StringSlice{"a", "b", "c", "d"}.MarshalText() 28 | 29 | require.NoError(t, err) 30 | 31 | assert.Equal(t, []byte("a,b,c,d"), slice) 32 | } 33 | 34 | func TestSliceUnmarshalJSON(t *testing.T) { 35 | t.Parallel() 36 | 37 | slice := types.StringSlice{} 38 | 39 | require.NoError(t, json.NewDecoder(strings.NewReader(`["a","b","c","d"]`)).Decode(&slice)) 40 | 41 | assert.Equal(t, types.StringSlice{"a", "b", "c", "d"}, slice) 42 | } 43 | 44 | func TestSliceUnmarshalYAML(t *testing.T) { 45 | t.Parallel() 46 | 47 | slice := types.StringSlice{} 48 | 49 | require.NoError(t, yaml.NewDecoder(strings.NewReader("- a\n- b\n- c\n- d\n")).Decode(&slice)) 50 | 51 | assert.Equal(t, types.StringSlice{"a", "b", "c", "d"}, slice) 52 | } 53 | 54 | func TestFloat64SliceUnmarshalText(t *testing.T) { 55 | t.Parallel() 56 | 57 | slice := types.Float64Slice{} 58 | 59 | require.NoError(t, slice.UnmarshalText([]byte("0.5,0.6,0.7,0.8"))) 60 | 61 | assert.Equal(t, types.Float64Slice{0.5, 0.6, 0.7, 0.8}, slice) 62 | } 63 | 64 | func TestFloat64SliceMarshalText(t *testing.T) { 65 | t.Parallel() 66 | 67 | slice, err := types.Float64Slice{0.5, 0.6, 0.7, 0.8}.MarshalText() 68 | 69 | require.NoError(t, err) 70 | 71 | assert.Equal(t, []byte("0.5,0.6,0.7,0.8"), slice) 72 | } 73 | 74 | func TestFloat64SliceUnmarshalJSON(t *testing.T) { 75 | t.Parallel() 76 | 77 | var slice types.Float64Slice 78 | 79 | require.NoError(t, json.NewDecoder(strings.NewReader(`[0.5,0.6,0.7,0.8]`)).Decode(&slice)) 80 | 81 | assert.Equal(t, types.Float64Slice{0.5, 0.6, 0.7, 0.8}, slice) 82 | } 83 | 84 | func TestFloat64SliceUnmarshalYAML(t *testing.T) { 85 | t.Parallel() 86 | 87 | var slice types.Float64Slice 88 | 89 | require.NoError(t, yaml.NewDecoder(strings.NewReader("- 0.5\n- 0.6\n- 0.7\n- 0.8\n")).Decode(&slice)) 90 | 91 | assert.Equal(t, types.Float64Slice{0.5, 0.6, 0.7, 0.8}, slice) 92 | } 93 | -------------------------------------------------------------------------------- /internal/config/types/url.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net/url" 8 | ) 9 | 10 | type URL struct { 11 | *url.URL 12 | } 13 | 14 | func NewURL(u string) (URL, error) { 15 | if u == "" { 16 | return URL{}, errors.New("empty URL") 17 | } 18 | 19 | stdURL, err := url.Parse(u) 20 | if err != nil { 21 | return URL{}, fmt.Errorf("failed to parse URL: %w", err) 22 | } 23 | 24 | return URL{stdURL}, nil 25 | } 26 | 27 | // IsEmpty checks if the URL is empty. 28 | // 29 | //goland:noinspection GoMixedReceiverTypes 30 | func (u *URL) IsEmpty() bool { 31 | return u == nil || u.URL == nil || u.URL.String() == "" 32 | } 33 | 34 | // String returns the string representation of the URL. 35 | // 36 | //goland:noinspection GoMixedReceiverTypes 37 | func (u URL) String() string { 38 | if u.IsEmpty() { 39 | return "" 40 | } 41 | 42 | return u.URL.String() 43 | } 44 | 45 | // MarshalText implements [encoding.TextMarshaler] interface. 46 | // 47 | //goland:noinspection GoMixedReceiverTypes 48 | func (u URL) MarshalText() ([]byte, error) { 49 | return []byte(u.String()), nil 50 | } 51 | 52 | // UnmarshalText implements the [encoding.TextUnmarshaler] interface. 53 | // 54 | //goland:noinspection GoMixedReceiverTypes 55 | func (u *URL) UnmarshalText(text []byte) error { 56 | parsedURL, err := NewURL(string(text)) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | *u = parsedURL 62 | 63 | return nil 64 | } 65 | 66 | // MarshalJSON implements the [json.Marshaler] interface. 67 | // 68 | //goland:noinspection GoMixedReceiverTypes 69 | func (u *URL) MarshalJSON() ([]byte, error) { 70 | return json.Marshal(u.String()) //nolint:wrapcheck 71 | } 72 | -------------------------------------------------------------------------------- /internal/config/types/url_test.go: -------------------------------------------------------------------------------- 1 | package types_test 2 | 3 | import ( 4 | "encoding/json" 5 | "net/url" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/jkroepke/access-log-exporter/internal/config/types" 10 | "github.com/stretchr/testify/require" 11 | "go.yaml.in/yaml/v4" 12 | ) 13 | 14 | func TestURL(t *testing.T) { 15 | t.Parallel() 16 | 17 | for _, tc := range []struct { 18 | name string 19 | input string 20 | err string 21 | }{ 22 | { 23 | "empty", 24 | "", 25 | "empty URL", 26 | }, 27 | { 28 | "invalid", 29 | "://", 30 | "missing protocol scheme", 31 | }, 32 | { 33 | "valid", 34 | "https://example.com", 35 | "", 36 | }, 37 | } { 38 | t.Run(tc.name, func(t *testing.T) { 39 | t.Parallel() 40 | 41 | _, err := types.NewURL(tc.input) 42 | if tc.err == "" { 43 | require.NoError(t, err) 44 | } else { 45 | require.ErrorContains(t, err, tc.err) 46 | } 47 | }) 48 | } 49 | } 50 | 51 | //nolint:exhaustruct 52 | func TestURLIsEmpty(t *testing.T) { 53 | t.Parallel() 54 | 55 | for _, tc := range []struct { 56 | name string 57 | url *types.URL 58 | expect bool 59 | }{ 60 | { 61 | "nil", 62 | nil, 63 | true, 64 | }, 65 | { 66 | "empty", 67 | &types.URL{}, 68 | true, 69 | }, 70 | { 71 | "non-empty", 72 | &types.URL{URL: &url.URL{Scheme: "http", Host: "localhost"}}, 73 | false, 74 | }, 75 | } { 76 | t.Run(tc.name, func(t *testing.T) { 77 | t.Parallel() 78 | require.Equal(t, tc.expect, tc.url.IsEmpty()) 79 | }) 80 | } 81 | } 82 | 83 | func TestURLUnmarshalText(t *testing.T) { 84 | t.Parallel() 85 | 86 | for _, tc := range []struct { 87 | name string 88 | input string 89 | err string 90 | }{ 91 | { 92 | "empty", 93 | "", 94 | "empty URL", 95 | }, 96 | { 97 | "invalid", 98 | "://", 99 | "missing protocol scheme", 100 | }, 101 | { 102 | "valid", 103 | "https://example.com", 104 | "", 105 | }, 106 | } { 107 | t.Run(tc.name, func(t *testing.T) { 108 | t.Parallel() 109 | 110 | actualURL := types.URL{} 111 | 112 | err := actualURL.UnmarshalText([]byte(tc.input)) 113 | if tc.err != "" { 114 | require.ErrorContains(t, err, tc.err) 115 | 116 | return 117 | } 118 | 119 | require.NoError(t, err) 120 | 121 | expectedURL, err := types.NewURL(tc.input) 122 | require.NoError(t, err) 123 | 124 | require.Equal(t, expectedURL, actualURL) 125 | }) 126 | } 127 | } 128 | 129 | func TestURLMarshalText(t *testing.T) { 130 | t.Parallel() 131 | 132 | actualURL, err := types.NewURL("https://example.com") 133 | require.NoError(t, err) 134 | 135 | urlBytes, err := actualURL.MarshalText() 136 | require.NoError(t, err) 137 | 138 | require.Equal(t, []byte("https://example.com"), urlBytes) 139 | } 140 | 141 | func TestURLUnmarshalJSON(t *testing.T) { 142 | t.Parallel() 143 | 144 | actualURL := types.URL{} 145 | require.NoError(t, json.NewDecoder(strings.NewReader(`"https://example.com"`)).Decode(&actualURL)) 146 | 147 | expectedURL, err := types.NewURL("https://example.com") 148 | require.NoError(t, err) 149 | 150 | require.Equal(t, expectedURL, actualURL) 151 | } 152 | 153 | func TestURLUnmarshalYAML(t *testing.T) { 154 | t.Parallel() 155 | 156 | actualURL := types.URL{} 157 | require.NoError(t, yaml.NewDecoder(strings.NewReader(`"https://example.com"`)).Decode(&actualURL)) 158 | 159 | expectedURL, err := types.NewURL("https://example.com") 160 | require.NoError(t, err) 161 | 162 | require.Equal(t, expectedURL, actualURL) 163 | } 164 | -------------------------------------------------------------------------------- /internal/config/validate.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Validate validates the config. 8 | func Validate(conf Config) error { 9 | _, ok := conf.Presets[conf.Preset] 10 | if !ok { 11 | return fmt.Errorf("preset '%s' not found in configuration", conf.Preset) 12 | } 13 | 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /internal/config/validate_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/jkroepke/access-log-exporter/internal/config" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestValidate(t *testing.T) { 12 | t.Parallel() 13 | 14 | for _, tc := range []struct { 15 | conf config.Config 16 | err string 17 | }{ 18 | { 19 | config.Config{}, 20 | "preset '' not found in configuration", 21 | }, 22 | } { 23 | t.Run(tc.err, func(t *testing.T) { 24 | t.Parallel() 25 | 26 | err := config.Validate(tc.conf) 27 | if tc.err == "" { 28 | require.NoError(t, err) 29 | } else { 30 | require.Error(t, err) 31 | 32 | if tc.err != "-" { 33 | assert.EqualError(t, err, tc.err) 34 | } 35 | } 36 | }) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/metric/metric_bench_test.go: -------------------------------------------------------------------------------- 1 | package metric_test 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/jkroepke/access-log-exporter/internal/config" 9 | "github.com/jkroepke/access-log-exporter/internal/config/types" 10 | "github.com/jkroepke/access-log-exporter/internal/metric" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func BenchmarkMetricParseSimple(b *testing.B) { 15 | met, err := metric.New(config.Metric{ 16 | Name: "http_requests_total", 17 | Type: "counter", 18 | Help: "The total number of client requests.", 19 | Labels: []config.Label{ 20 | { 21 | Name: "host", 22 | LineIndex: 0, 23 | }, 24 | { 25 | Name: "method", 26 | LineIndex: 1, 27 | }, 28 | { 29 | Name: "status", 30 | LineIndex: 2, 31 | }, 32 | }, 33 | }) 34 | 35 | require.NoError(b, err) 36 | 37 | logLine := strings.Split("example.com\tGET\t200", "\t") 38 | 39 | for b.Loop() { 40 | _ = met.Parse(logLine) 41 | } 42 | 43 | b.ReportAllocs() 44 | } 45 | 46 | func BenchmarkMetricParseUserAgent(b *testing.B) { 47 | met, err := metric.New(config.Metric{ 48 | Name: "http_requests_total", 49 | Type: "counter", 50 | Help: "The total number of client requests.", 51 | Labels: []config.Label{ 52 | { 53 | Name: "host", 54 | LineIndex: 0, 55 | }, 56 | { 57 | Name: "method", 58 | LineIndex: 1, 59 | }, 60 | { 61 | Name: "status", 62 | LineIndex: 2, 63 | }, 64 | { 65 | Name: "user_agent", 66 | LineIndex: 3, 67 | UserAgent: true, 68 | }, 69 | }, 70 | }) 71 | 72 | require.NoError(b, err) 73 | 74 | logLine := strings.Split("example.com\tGET\t200\tMozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15", "\t") 75 | 76 | for b.Loop() { 77 | _ = met.Parse(logLine) 78 | } 79 | 80 | b.ReportAllocs() 81 | } 82 | 83 | func BenchmarkMetricParseReplacement(b *testing.B) { 84 | met, err := metric.New(config.Metric{ 85 | Name: "http_requests_total", 86 | Type: "counter", 87 | Help: "The total number of client requests.", 88 | Labels: []config.Label{ 89 | { 90 | Name: "host", 91 | LineIndex: 0, 92 | }, 93 | { 94 | Name: "method", 95 | LineIndex: 1, 96 | }, 97 | { 98 | Name: "status", 99 | LineIndex: 2, 100 | }, 101 | { 102 | Name: "path", 103 | LineIndex: 3, 104 | Replacements: []config.Replacement{ 105 | { 106 | Regexp: regexp.MustCompile(`/api/v1/resource\?id=\d+.+`), 107 | Replacement: "/api/v1/resource?id=:id", 108 | }, 109 | }, 110 | }, 111 | }, 112 | }) 113 | 114 | require.NoError(b, err) 115 | 116 | logLine := strings.Split("example.com\tGET\t200\t/api/v1/resource?id=12345&name=test", "\t") 117 | 118 | for b.Loop() { 119 | _ = met.Parse(logLine) 120 | } 121 | 122 | b.ReportAllocs() 123 | } 124 | 125 | func BenchmarkMetricParseUpstream(b *testing.B) { 126 | met, err := metric.New(config.Metric{ 127 | Name: "http_upstream_connect_duration_seconds", 128 | Type: "histogram", 129 | Help: "The time spent on establishing a connection with the upstream server", 130 | ValueIndex: ptr(uint(7)), 131 | Buckets: types.Float64Slice{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, 132 | Math: config.Math{ 133 | Enabled: true, 134 | Div: 1000, 135 | Mul: 0, // default value 136 | }, 137 | Upstream: config.Upstream{ 138 | Enabled: true, 139 | AddrLineIndex: 6, 140 | Excludes: make([]string, 0), 141 | Label: false, // default value 142 | }, 143 | Labels: []config.Label{ 144 | { 145 | Name: "host", 146 | LineIndex: 0, 147 | }, 148 | { 149 | Name: "method", 150 | LineIndex: 1, 151 | }, 152 | { 153 | Name: "status", 154 | LineIndex: 2, 155 | }, 156 | }, 157 | }) 158 | 159 | require.NoError(b, err) 160 | 161 | logLine := strings.Split("web.example.org\tPOST\t502\t2.150\t2048\t512\t10.0.1.10:8080, 10.0.1.11:8080, 10.0.1.12:8080\t0.005, 0.004, -\t0.120, 0.115, -\t0.800, 0.900, -", "\t") 162 | 163 | for b.Loop() { 164 | _ = met.Parse(logLine) 165 | } 166 | 167 | b.ReportAllocs() 168 | } 169 | -------------------------------------------------------------------------------- /internal/metric/types.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/jkroepke/access-log-exporter/internal/config" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/ua-parser/uap-go/uaparser" 9 | ) 10 | 11 | type Metric struct { 12 | metric prometheus.Collector 13 | ua *uaparser.Parser 14 | labelsPool *sync.Pool // Pool for reusing label maps in a thread-safe way 15 | 16 | cfg config.Metric 17 | } 18 | -------------------------------------------------------------------------------- /internal/nginx/collector.go: -------------------------------------------------------------------------------- 1 | package nginx 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "log/slog" 8 | "net/http" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | const templateMetrics string = `Active connections: %d 17 | server accepts handled requests 18 | %d %d %d 19 | Reading: %d Writing: %d Waiting: %d 20 | ` 21 | 22 | type Collector struct { 23 | upMetric *prometheus.Desc 24 | connectionsAccepted *prometheus.Desc 25 | connectionsActive *prometheus.Desc 26 | connectionsHandled *prometheus.Desc 27 | connectionsReading *prometheus.Desc 28 | connectionsWaiting *prometheus.Desc 29 | connectionsWriting *prometheus.Desc 30 | logger *slog.Logger 31 | scrapeURL string 32 | mu sync.Mutex 33 | } 34 | 35 | // StubStats represents NGINX stub_status metrics. 36 | type StubStats struct { 37 | Connections StubConnections 38 | Requests int64 39 | } 40 | 41 | // StubConnections represents connections related metrics. 42 | type StubConnections struct { 43 | Active int64 44 | Accepted int64 45 | Handled int64 46 | Reading int64 47 | Writing int64 48 | Waiting int64 49 | } 50 | 51 | func New(logger *slog.Logger, scrapeURL string) *Collector { 52 | return &Collector{ 53 | scrapeURL: scrapeURL, 54 | logger: logger.With(slog.String("component", "nginx_collector")), 55 | upMetric: prometheus.NewDesc( 56 | "nginx_up", 57 | "Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable.", 58 | []string{"version"}, nil, 59 | ), 60 | connectionsAccepted: prometheus.NewDesc( 61 | "nginx_connections_accepted_total", 62 | "Accepted client connections.", 63 | nil, nil, 64 | ), 65 | connectionsActive: prometheus.NewDesc( 66 | "nginx_connections_active", 67 | "Active client connections.", 68 | nil, nil, 69 | ), 70 | connectionsHandled: prometheus.NewDesc( 71 | "nginx_connections_handled_total", 72 | "Handled client connections.", 73 | nil, nil, 74 | ), 75 | connectionsReading: prometheus.NewDesc( 76 | "nginx_connections_reading", 77 | "Connections where NGINX is reading the request header.", 78 | nil, nil, 79 | ), 80 | connectionsWaiting: prometheus.NewDesc( 81 | "nginx_connections_waiting", 82 | "Idle client connections.", 83 | nil, nil, 84 | ), 85 | connectionsWriting: prometheus.NewDesc( 86 | "nginx_connections_writing", 87 | "Connections where NGINX is writing the response back to the client.", 88 | nil, nil, 89 | ), 90 | } 91 | } 92 | 93 | func (c *Collector) Describe(ch chan<- *prometheus.Desc) { 94 | ch <- c.upMetric 95 | 96 | ch <- c.connectionsAccepted 97 | 98 | ch <- c.connectionsActive 99 | 100 | ch <- c.connectionsHandled 101 | 102 | ch <- c.connectionsReading 103 | 104 | ch <- c.connectionsWaiting 105 | 106 | ch <- c.connectionsWriting 107 | } 108 | 109 | func (c *Collector) Collect(ch chan<- prometheus.Metric) { 110 | c.mu.Lock() // To protect metrics from concurrent collects 111 | defer c.mu.Unlock() 112 | 113 | serverVersion := "N/A" 114 | 115 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 116 | defer cancel() 117 | 118 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.scrapeURL, nil) 119 | if err != nil { 120 | c.logger.Error("Failed to create HTTP request for NGINX metrics", 121 | slog.String("url", c.scrapeURL), 122 | slog.Any("error", err), 123 | ) 124 | 125 | ch <- prometheus.MustNewConstMetric(c.upMetric, 126 | prometheus.GaugeValue, 0, serverVersion) 127 | 128 | return 129 | } 130 | 131 | req.Header.Set("User-Agent", "jkroepke/access-log-exporter") 132 | 133 | resp, err := http.DefaultClient.Do(req) 134 | if err != nil { 135 | c.logger.Error("Failed to scrape NGINX metrics", 136 | slog.String("url", c.scrapeURL), 137 | slog.Any("error", err), 138 | ) 139 | 140 | ch <- prometheus.MustNewConstMetric(c.upMetric, 141 | prometheus.GaugeValue, 0, serverVersion) 142 | 143 | return 144 | } 145 | 146 | defer func() { 147 | _ = resp.Body.Close() 148 | }() 149 | 150 | if resp.StatusCode != http.StatusOK { 151 | c.logger.Error("NGINX metrics endpoint returned non-200 status code", 152 | slog.String("url", c.scrapeURL), 153 | slog.Int("status_code", resp.StatusCode), 154 | ) 155 | 156 | ch <- prometheus.MustNewConstMetric(c.upMetric, 157 | prometheus.GaugeValue, 0, serverVersion) 158 | 159 | return 160 | } 161 | 162 | // Attempt to read the server version from the response header 163 | if version := resp.Header.Get("Server"); strings.HasPrefix(version, "nginx/") { 164 | serverVersion = strings.TrimPrefix(version, "nginx/") 165 | } 166 | 167 | stats, err := parseStubStats(resp.Body) 168 | if err != nil { 169 | c.logger.Error("Failed to parse NGINX metrics", 170 | slog.String("url", c.scrapeURL), 171 | slog.Any("error", err), 172 | ) 173 | 174 | ch <- prometheus.MustNewConstMetric(c.upMetric, 175 | prometheus.GaugeValue, 0, serverVersion) 176 | 177 | return 178 | } 179 | 180 | ch <- prometheus.MustNewConstMetric(c.upMetric, 181 | prometheus.GaugeValue, 1, serverVersion) 182 | 183 | ch <- prometheus.MustNewConstMetric(c.connectionsActive, 184 | prometheus.GaugeValue, float64(stats.Connections.Active)) 185 | 186 | ch <- prometheus.MustNewConstMetric(c.connectionsAccepted, 187 | prometheus.CounterValue, float64(stats.Connections.Accepted)) 188 | 189 | ch <- prometheus.MustNewConstMetric(c.connectionsHandled, 190 | prometheus.CounterValue, float64(stats.Connections.Handled)) 191 | 192 | ch <- prometheus.MustNewConstMetric(c.connectionsReading, 193 | prometheus.GaugeValue, float64(stats.Connections.Reading)) 194 | 195 | ch <- prometheus.MustNewConstMetric(c.connectionsWriting, 196 | prometheus.GaugeValue, float64(stats.Connections.Writing)) 197 | 198 | ch <- prometheus.MustNewConstMetric(c.connectionsWaiting, 199 | prometheus.GaugeValue, float64(stats.Connections.Waiting)) 200 | } 201 | 202 | func parseStubStats(r io.Reader) (*StubStats, error) { 203 | var stubStats StubStats 204 | if _, err := fmt.Fscanf(r, templateMetrics, 205 | &stubStats.Connections.Active, 206 | &stubStats.Connections.Accepted, 207 | &stubStats.Connections.Handled, 208 | &stubStats.Requests, 209 | &stubStats.Connections.Reading, 210 | &stubStats.Connections.Writing, 211 | &stubStats.Connections.Waiting); err != nil { 212 | return nil, fmt.Errorf("failed to scan template metrics: %w", err) 213 | } 214 | 215 | return &stubStats, nil 216 | } 217 | -------------------------------------------------------------------------------- /internal/nginx/collector_test.go: -------------------------------------------------------------------------------- 1 | package nginx_test 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log/slog" 7 | "net/http" 8 | "net/http/httptest" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/jkroepke/access-log-exporter/internal/nginx" 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestCollector(t *testing.T) { 19 | t.Parallel() 20 | 21 | for _, tc := range []struct { 22 | name string 23 | handler http.HandlerFunc 24 | metrics string 25 | }{ 26 | { 27 | name: "valid URL", 28 | handler: func(w http.ResponseWriter, _ *http.Request) { 29 | w.Header().Add("Server", "nginx") 30 | w.WriteHeader(http.StatusOK) 31 | _, err := w.Write([]byte("Active connections: 1\nserver accepts handled requests\n10 10 10\nReading: 0 Writing: 1 Waiting: 0\n")) 32 | if err != nil { 33 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 34 | } 35 | }, 36 | metrics: `# HELP nginx_connections_accepted_total Accepted client connections. 37 | # TYPE nginx_connections_accepted_total counter 38 | nginx_connections_accepted_total 10 39 | # HELP nginx_connections_active Active client connections. 40 | # TYPE nginx_connections_active gauge 41 | nginx_connections_active 1 42 | # HELP nginx_connections_handled_total Handled client connections. 43 | # TYPE nginx_connections_handled_total counter 44 | nginx_connections_handled_total 10 45 | # HELP nginx_connections_reading Connections where NGINX is reading the request header. 46 | # TYPE nginx_connections_reading gauge 47 | nginx_connections_reading 0 48 | # HELP nginx_connections_waiting Idle client connections. 49 | # TYPE nginx_connections_waiting gauge 50 | nginx_connections_waiting 0 51 | # HELP nginx_connections_writing Connections where NGINX is writing the response back to the client. 52 | # TYPE nginx_connections_writing gauge 53 | nginx_connections_writing 1 54 | # HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 55 | # TYPE nginx_up gauge 56 | nginx_up{version="N/A"} 1`, 57 | }, 58 | { 59 | name: "valid URL with version", 60 | handler: func(w http.ResponseWriter, _ *http.Request) { 61 | w.Header().Add("Server", "nginx/1.23.1") 62 | w.WriteHeader(http.StatusOK) 63 | _, err := w.Write([]byte("Active connections: 1\nserver accepts handled requests\n10 10 10\nReading: 0 Writing: 1 Waiting: 0\n")) 64 | if err != nil { 65 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 66 | } 67 | }, 68 | metrics: `# HELP nginx_connections_accepted_total Accepted client connections. 69 | # TYPE nginx_connections_accepted_total counter 70 | nginx_connections_accepted_total 10 71 | # HELP nginx_connections_active Active client connections. 72 | # TYPE nginx_connections_active gauge 73 | nginx_connections_active 1 74 | # HELP nginx_connections_handled_total Handled client connections. 75 | # TYPE nginx_connections_handled_total counter 76 | nginx_connections_handled_total 10 77 | # HELP nginx_connections_reading Connections where NGINX is reading the request header. 78 | # TYPE nginx_connections_reading gauge 79 | nginx_connections_reading 0 80 | # HELP nginx_connections_waiting Idle client connections. 81 | # TYPE nginx_connections_waiting gauge 82 | nginx_connections_waiting 0 83 | # HELP nginx_connections_writing Connections where NGINX is writing the response back to the client. 84 | # TYPE nginx_connections_writing gauge 85 | nginx_connections_writing 1 86 | # HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 87 | # TYPE nginx_up gauge 88 | nginx_up{version="1.23.1"} 1`, 89 | }, 90 | { 91 | name: "invalid format", 92 | handler: func(w http.ResponseWriter, _ *http.Request) { 93 | w.WriteHeader(http.StatusOK) 94 | _, err := w.Write([]byte("Invalid format")) 95 | if err != nil { 96 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 97 | } 98 | }, 99 | metrics: `# HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 100 | # TYPE nginx_up gauge 101 | nginx_up{version="N/A"} 0`, 102 | }, 103 | { 104 | name: "empty response", 105 | handler: func(w http.ResponseWriter, _ *http.Request) { 106 | w.WriteHeader(http.StatusOK) 107 | _, err := w.Write([]byte("")) 108 | if err != nil { 109 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 110 | } 111 | }, 112 | metrics: `# HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 113 | # TYPE nginx_up gauge 114 | nginx_up{version="N/A"} 0`, 115 | }, 116 | { 117 | name: "access denied", 118 | handler: func(w http.ResponseWriter, _ *http.Request) { 119 | w.WriteHeader(http.StatusForbidden) 120 | }, 121 | metrics: `# HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 122 | # TYPE nginx_up gauge 123 | nginx_up{version="N/A"} 0`, 124 | }, 125 | } { 126 | t.Run(tc.name, func(t *testing.T) { 127 | t.Parallel() 128 | 129 | stubServer := httptest.NewServer(tc.handler) 130 | t.Cleanup(stubServer.Close) 131 | 132 | col := nginx.New(slog.New(slog.DiscardHandler), stubServer.URL) 133 | 134 | metrics, err := MetricsToText(t, col) 135 | require.NoError(t, err) 136 | require.Equal(t, tc.metrics, metrics) 137 | }) 138 | } 139 | } 140 | 141 | func TestCollector_NoServer(t *testing.T) { 142 | t.Parallel() 143 | 144 | col := nginx.New(slog.New(slog.DiscardHandler), "http://nonexistent-server") 145 | 146 | metrics, err := MetricsToText(t, col) 147 | require.NoError(t, err) 148 | 149 | expected := `# HELP nginx_up Whether the NGINX server is up (1) or down (0). 1 means the server is up and metrics are being collected, 0 means the server is down or unreachable. 150 | # TYPE nginx_up gauge 151 | nginx_up{version="N/A"} 0` 152 | 153 | require.Equal(t, expected, metrics) 154 | } 155 | 156 | func MetricsToText(tb testing.TB, met prometheus.Collector) (string, error) { 157 | tb.Helper() 158 | 159 | reg := prometheus.NewRegistry() 160 | err := reg.Register(met) 161 | require.NoError(tb, err) 162 | 163 | request, err := http.NewRequestWithContext(tb.Context(), http.MethodGet, "/", nil) 164 | require.NoError(tb, err) 165 | 166 | request.Header.Add("Accept", "text/plain") 167 | 168 | writer := httptest.NewRecorder() 169 | 170 | regHandler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) 171 | regHandler.ServeHTTP(writer, request) 172 | 173 | require.Equal(tb, http.StatusOK, writer.Code) 174 | 175 | allMetrics, err := io.ReadAll(writer.Body) 176 | if err != nil { 177 | return "", fmt.Errorf("error reading writer body: %w", err) 178 | } 179 | 180 | return strings.TrimSpace(string(allMetrics)), nil 181 | } 182 | -------------------------------------------------------------------------------- /internal/syslog/syslog.go: -------------------------------------------------------------------------------- 1 | package syslog 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "log/slog" 9 | "net" 10 | "net/url" 11 | "os" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | type Syslog struct { 18 | con net.PacketConn 19 | logger *slog.Logger 20 | msgCh chan<- string 21 | poolBuffer *sync.Pool 22 | listenAddr string 23 | } 24 | 25 | func New(ctx context.Context, logger *slog.Logger, listenAddr string, msgCh chan<- string) (Syslog, error) { 26 | syslogServer := Syslog{ 27 | listenAddr: listenAddr, 28 | logger: logger.With(slog.String("component", "syslog")), 29 | msgCh: msgCh, 30 | poolBuffer: &sync.Pool{ 31 | New: func() any { 32 | buf := make([]byte, 4096) 33 | 34 | return &buf 35 | }, 36 | }, 37 | } 38 | 39 | uri, err := url.Parse(listenAddr) 40 | if err != nil { 41 | return Syslog{}, fmt.Errorf("could not parse syslog listen address '%s': %w", listenAddr, err) 42 | } 43 | 44 | var listenConf net.ListenConfig 45 | 46 | switch uri.Scheme { 47 | case "udp": 48 | syslogServer.con, err = listenConf.ListenPacket(ctx, "udp", uri.Host) 49 | case "unix": 50 | syslogServer.con, err = listenConf.ListenPacket(ctx, "unixgram", uri.Host+uri.Path) 51 | default: 52 | err = errors.New("syslog listen address must be start with udp:// or unix://") 53 | } 54 | 55 | if err != nil { 56 | return Syslog{}, fmt.Errorf("could not listen syslog server on '%s': %w", listenAddr, err) 57 | } 58 | 59 | return syslogServer, nil 60 | } 61 | 62 | //nolint:gocognit,cyclop 63 | func (s *Syslog) Start() error { 64 | for { 65 | buf, _ := s.poolBuffer.Get().(*[]byte) 66 | msg := *buf 67 | 68 | n, _, err := s.con.ReadFrom(msg) 69 | if err != nil { 70 | // there has been an error. Either the server has been killed 71 | // or may be getting a transitory error due to (e.g.) the 72 | // interface being shutdown in which case sleep() to avoid busy wait. 73 | var opError *net.OpError 74 | 75 | ok := errors.As(err, &opError) 76 | if (ok) && !opError.Temporary() && !opError.Timeout() { 77 | return fmt.Errorf("syslog server stopped: %w", err) 78 | } 79 | 80 | time.Sleep(10 * time.Millisecond) 81 | 82 | s.poolBuffer.Put(buf) 83 | 84 | continue 85 | } 86 | 87 | if n <= 0 { 88 | // Ignore empty messages 89 | s.poolBuffer.Put(buf) 90 | 91 | continue 92 | } 93 | 94 | // Ignore messages not starting with '<' 95 | if !bytes.HasPrefix(msg, []byte("<")) { 96 | s.poolBuffer.Put(buf) 97 | 98 | continue 99 | } 100 | 101 | // Ignore trailing control characters and NULs 102 | //nolint:revive 103 | for ; (n > 0) && (msg[n-1] < 32); n-- { 104 | } 105 | 106 | // buf may contain a syslog message with a header like "<34>Oct 11 22:14:15 nginx: " 107 | // We need to find the first occurrence of ": " to extract the actual message. 108 | // Find the index after the 3th occurrence of ':' (optionally followed by a space) 109 | colonCount := 0 110 | idx := -1 111 | 112 | for i := range n { 113 | if msg[i] == ':' { 114 | colonCount++ 115 | if colonCount == 3 { 116 | idx = i 117 | // Optionally, check for a space after the colon 118 | if i+1 < n && msg[i+1] == ' ' { 119 | idx = i + 1 // include the space 120 | } 121 | 122 | break 123 | } 124 | } 125 | } 126 | 127 | if idx == -1 { 128 | s.poolBuffer.Put(buf) 129 | 130 | continue // fewer than 4 colons found 131 | } 132 | 133 | // Now buf[idx+1:n] contains the message after the 3th colon (and space, if present) 134 | s.msgCh <- string(msg[idx+1 : n]) 135 | 136 | s.poolBuffer.Put(buf) 137 | } 138 | } 139 | 140 | func (s *Syslog) Close(ctx context.Context) error { 141 | if s.con == nil { 142 | return errors.New("syslog server is not initialized") 143 | } 144 | 145 | err := s.con.Close() 146 | if err != nil { 147 | return fmt.Errorf("could not stop syslog server: %w", err) 148 | } 149 | 150 | if strings.HasPrefix(s.listenAddr, "unix://") { 151 | _ = os.Remove(strings.TrimPrefix(s.listenAddr, "unix://")) 152 | } 153 | 154 | s.logger.InfoContext(ctx, "syslog server shutdown complete") 155 | 156 | return nil 157 | } 158 | -------------------------------------------------------------------------------- /internal/syslog/syslog_bench_test.go: -------------------------------------------------------------------------------- 1 | package syslog_test 2 | 3 | import ( 4 | "log/slog" 5 | "net" 6 | "testing" 7 | 8 | "github.com/jkroepke/access-log-exporter/internal/syslog" 9 | "github.com/stretchr/testify/require" 10 | "golang.org/x/net/nettest" 11 | ) 12 | 13 | func Benchmark_Syslog(b *testing.B) { 14 | unixSocket, err := nettest.LocalPath() 15 | require.NoError(b, err) 16 | 17 | logBuffer := make(chan string, 1) 18 | 19 | server, err := syslog.New(b.Context(), slog.New(slog.DiscardHandler), "unix://"+unixSocket, logBuffer) 20 | require.NoError(b, err) 21 | 22 | b.Cleanup(func() { 23 | require.NoError(b, server.Close(b.Context())) 24 | }) 25 | 26 | var dial net.Dialer 27 | 28 | syslogClient, err := dial.DialContext(b.Context(), "unixgram", unixSocket) 29 | require.NoError(b, err) 30 | 31 | logMessageBytes := []byte("<190>Aug 15 20:16:01 nginx: localhost:8080\tGET\t404\t0.000\t767\t710") 32 | 33 | var serverErr error 34 | 35 | go func() { 36 | serverErr = server.Start() 37 | }() 38 | 39 | b.Cleanup(func() { 40 | require.NoError(b, serverErr) 41 | }) 42 | 43 | // Benchmark the syslog server by sending a message 44 | b.ResetTimer() 45 | 46 | for b.Loop() { 47 | _, _ = syslogClient.Write(logMessageBytes) 48 | 49 | <-logBuffer 50 | } 51 | 52 | b.ReportAllocs() 53 | } 54 | -------------------------------------------------------------------------------- /internal/syslog/syslog_test.go: -------------------------------------------------------------------------------- 1 | package syslog_test 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | syslogclient "log/syslog" 7 | "net" 8 | "testing" 9 | 10 | "github.com/jkroepke/access-log-exporter/internal/syslog" 11 | "github.com/stretchr/testify/require" 12 | "golang.org/x/net/nettest" 13 | ) 14 | 15 | func TestSyslogServer(t *testing.T) { 16 | t.Parallel() 17 | 18 | unixSocket, err := nettest.LocalPath() 19 | require.NoError(t, err) 20 | 21 | logBuffer := make(chan string, 1) 22 | 23 | server, err := syslog.New(t.Context(), slog.New(slog.DiscardHandler), "unix://"+unixSocket, logBuffer) 24 | require.NoError(t, err) 25 | 26 | t.Cleanup(func() { 27 | require.NoError(t, server.Close(t.Context())) 28 | }) 29 | 30 | var serverErr error 31 | 32 | go func() { 33 | serverErr = server.Start() 34 | }() 35 | 36 | t.Cleanup(func() { 37 | require.NoError(t, serverErr) 38 | }) 39 | 40 | syslogClient, err := syslogclient.Dial("unixgram", unixSocket, syslogclient.LOG_LOCAL7, "") 41 | require.NoError(t, err) 42 | 43 | logMessage := "Test message" 44 | 45 | _, err = fmt.Fprint(syslogClient, logMessage) 46 | require.NoError(t, err) 47 | 48 | require.Equal(t, logMessage, <-logBuffer) 49 | } 50 | 51 | func TestSyslogServerRawMessage(t *testing.T) { 52 | t.Parallel() 53 | 54 | unixSocket, err := nettest.LocalPath() 55 | require.NoError(t, err) 56 | 57 | logBuffer := make(chan string, 1) 58 | 59 | server, err := syslog.New(t.Context(), slog.New(slog.DiscardHandler), "unix://"+unixSocket, logBuffer) 60 | require.NoError(t, err) 61 | 62 | t.Cleanup(func() { 63 | require.NoError(t, server.Close(t.Context())) 64 | }) 65 | 66 | var serverErr error 67 | 68 | go func() { 69 | serverErr = server.Start() 70 | }() 71 | 72 | t.Cleanup(func() { 73 | require.NoError(t, serverErr) 74 | }) 75 | 76 | var dial net.Dialer 77 | 78 | syslogClient, err := dial.DialContext(t.Context(), "unixgram", unixSocket) 79 | require.NoError(t, err) 80 | 81 | _, err = syslogClient.Write([]byte("<190>Aug 15 20:16:01 nginx: localhost:8080\tGET\t404\t0.000\t767\t710")) 82 | require.NoError(t, err) 83 | 84 | logMessage := "localhost:8080\tGET\t404\t0.000\t767\t710" 85 | 86 | _, err = fmt.Fprint(syslogClient, logMessage) 87 | require.NoError(t, err) 88 | 89 | require.Equal(t, logMessage, <-logBuffer) 90 | } 91 | 92 | func TestSyslogServerWithInvalidMessages(t *testing.T) { 93 | t.Parallel() 94 | 95 | for _, tc := range []struct { 96 | name string 97 | message string 98 | }{ 99 | { 100 | name: "Empty message", 101 | message: "", 102 | }, 103 | { 104 | name: "without message", 105 | message: "<34>Oct 11 22:14:15", 106 | }, 107 | { 108 | name: "with partial time", 109 | message: "<34>Oct 22:14", 110 | }, 111 | { 112 | name: "without time", 113 | message: "<34>Oct 11", 114 | }, 115 | { 116 | name: "without priority", 117 | message: "Oct 11 22:14:15", 118 | }, 119 | } { 120 | t.Run(tc.name, func(t *testing.T) { 121 | t.Parallel() 122 | 123 | unixSocket, err := nettest.LocalPath() 124 | require.NoError(t, err) 125 | 126 | logBuffer := make(chan string, 1) 127 | 128 | server, err := syslog.New(t.Context(), slog.New(slog.DiscardHandler), "unix://"+unixSocket, logBuffer) 129 | require.NoError(t, err) 130 | 131 | t.Cleanup(func() { 132 | require.NoError(t, server.Close(t.Context())) 133 | }) 134 | 135 | var serverErr error 136 | 137 | go func() { 138 | serverErr = server.Start() 139 | }() 140 | 141 | t.Cleanup(func() { 142 | require.NoError(t, serverErr) 143 | }) 144 | 145 | var dial net.Dialer 146 | 147 | syslogClient, err := dial.DialContext(t.Context(), "unixgram", unixSocket) 148 | require.NoError(t, err) 149 | 150 | _, err = fmt.Fprint(syslogClient, tc.message) 151 | require.NoError(t, err) 152 | 153 | require.Empty(t, logBuffer) 154 | }) 155 | } 156 | } 157 | 158 | func TestSyslogInvalidListenAddr(t *testing.T) { 159 | t.Parallel() 160 | 161 | for _, tc := range []string{ 162 | "://address", 163 | "invalid://address", 164 | "tcp://invalid:1234", 165 | "udp://invalid:1234", 166 | "udp://0.0.0.1:1000000", 167 | } { 168 | t.Run(tc, func(t *testing.T) { 169 | t.Parallel() 170 | 171 | _, err := syslog.New(t.Context(), slog.New(slog.DiscardHandler), tc, nil) 172 | require.Error(t, err) 173 | }) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /internal/useragent/useragent.go: -------------------------------------------------------------------------------- 1 | package useragent 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/ua-parser/uap-go/uaparser" 7 | ) 8 | 9 | //nolint:gochecknoglobals // user agent parser is a global singleton 10 | var parser = sync.OnceValue(uaparser.NewFromSaved) 11 | 12 | func New() *uaparser.Parser { 13 | return parser() 14 | } 15 | -------------------------------------------------------------------------------- /packaging/apt/access-log-exporter.sources: -------------------------------------------------------------------------------- 1 | Types: deb 2 | URIs: https://github.com/jkroepke/access-log-exporter/releases/latest/download 3 | Suites: ./ 4 | Signed-By: 5 | -----BEGIN PGP PUBLIC KEY BLOCK----- 6 | . 7 | mQINBFwK+LwBEAC/ruyzeQ3LqDF2D61FaAH9tJhe6zuLp6xNwfBZydO9FbKyecu3 8 | z6fEtw7B2rQ1kMpmFeGZYhra0bmsSdTyU/Z/V43HYQgpY1PsY7KTTRv7d/UXbiwG 9 | sbgwjoLNc6Ggt5WnrSgjq5gjA5cPUWYv2gyZOFzzgyGu+i/KD25yjBVVNkR+AZFS 10 | JgaRL6ryV6GUtwbu3yenoiy2EFpC84s4obudWpvhp7M2PppAx5NZtJqgrKzvAMSQ 11 | qWKO76RvP660uuJ2TBDrhuBmzwVi+0rpPqbruVcWLh284RHK9R1rGUkjAwwb2VCw 12 | h7iE9CC0R87aN9R9XPENoxyEKPoy/xb9qOMVOBcOXm/AgtrfVW1oTn5idHF4rBjx 13 | iR+N47rC1NjgTMaDahoCgm6a+BvexYbVAC7MBu1OYrWiFuo2nqxhQpcLPgBbpjea 14 | DFf/BfjGoRkDt3VG08cJOWEo4lXzBHaPW32jk3COKVXNQe3c9OCxkYg3S/tSnpIC 15 | UPrJzxXeHivskANZ4rAzyMXd1azwyVsnN1OxWPwn7x6kHu9k8VRrHMhSpl9yL+kQ 16 | TYf6r78dd2pcoJqbsjukaDHgOhXdrcovM3B7MpFfld7ErIjz7gM/NN/PN7A/xArU 17 | K+Zdzl3A+QQuG8Kz6Ms4Mj/u1bKsh1cRDNKbF0YtnsGisiwSbVIjX19euQARAQAB 18 | tCNKYW4tT3R0byBLcsO2cGtlIDxtYWlsQGprcm9lcGtlLmRlPokCTgQTAQgAOAIb 19 | AwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgBYhBB80+VtPMLxbBuDXzD9hnxcAJ5DY 20 | BQJcCvjhAAoJED9hnxcAJ5DYbVYP/Rb8Umq+Z5oDpfNxdUZ5wpVBOHpWFuhM3WoB 21 | xqes/u/B4g7xAMhToVOMVgmeUsthHUDZ/mFRaHQJri1BoFILWiby077RUJ8e9RH7 22 | M7zZ+Lw/Svf3RpFNaUJX8yRbdaGq1Poc0wOGSRnYGQYHmRrUvbmNHIxxoMV//A6l 23 | Pmtwlnmw8vHA6XpFxRuyNSEcyKx67ewk8BIrAMngjP9+K86+rbzGVoMy7xeGb+34 24 | 3pgZCkxPuWQu46IU7BH/HQcfD8zbAk2kWHD6QgaCNRMQY4m3W8SKpLwfdTYaMIRF 25 | qhMb1Q3FaYBdWQe4xDxPJ978682Dk/gzuaS8Y9jVbfXgyPoDB3TDt6dtVgpo1ZgC 26 | mmIt/9f2zjNY7PyHpW9lM1wL84whHUR37XpsPFkFz8mNhrYNx5vt74ytjTb8+KEr 27 | kZPVvhrMCLDq38JttlLq9+7KV/3shKdILsG5L3m4O6W4gj+WCrhefNgYclxGlk8D 28 | Skl4ZMCYZonvoXnkU4wu+QbcEdgufUxz2IDdkqAtp9tCq42rfTQw8EzU/K0U3Hv+ 29 | nN+YgIKGK+30jJdJoiNnAAIc+mKIKOWtVXLij5Z68NzXNGigcpWIirhFGG9kvwFg 30 | xy+t9zQqb2xUSD0ut6tdwd3gTrxC/PNCh9KH1ns0iBJiUT0dgtDJ5PM85a+Ym1Hh 31 | Cb5skD2FuQINBFwK+LwBEAD8OH0j7rP6zpA7noUUNoaGHYOEIRIRDRClbyS88S/m 32 | MdJK2XTQjyzvUdEmBQspX7A/3SeJIZYQbQyIxEOvJVBC62ZupFTwpmnLDr+79K74 33 | lQEzRHNeZeHFbGS3F7PoLnkB65QIIGRlqzzUJsdnWIHs8HXKaCJ5hzH5zVlDi+pH 34 | N8cLNV6tt6tRNd7IFEV0dJy2Kk2GsTGdsaDsrrF5Ge8mWaEh9tixzBORa2ThWUSm 35 | 5TYBWrehvzb8eipu0WUBt3MlWWwXeAykf5byt+HHDmYPj/TD59hKHCzHdci0qjIe 36 | H6cQzBCSRu0pKAN75+Z/V5acarYVAE6bbX7130M7MuqqzTMioJ+93XnW13M42bSz 37 | Um+D/Drayv2OW7wfDrrhJ1EcBklsbY3Uq/ZeosahXsoHbJbvGGoqY2Ijhk3SRVNK 38 | 7TRwrxVu6Wzi3wF7tbn+AqOQECkCtFGTR16Q9heUtfw7UUB1+bNT/s0egDGNWHvh 39 | 1AGtKsarhvmqk2j2MTqfJQglrScZy7JJ3xq63kqykhTKDndjBYqBE2pdsnhHemU7 40 | hgGe74xIdxqZZrQssDv80Pl1OU70WLo5DPciVmNLlg3YeTvXLL6/qy5+h/mCZmPU 41 | 3ds3AO/emqDSqXEJg5zESBitQsqAMxus8DrZY1anK2/cKEoFIchdOn6e3AFBTFND 42 | gwARAQABiQI2BBgBCAAgAhsMFiEEHzT5W08wvFsG4NfMP2GfFwAnkNgFAl+gQSAA 43 | CgkQP2GfFwAnkNjxARAAvyybWLltzm/gagcxOUWPwHhge3MPWRUQEU5bwJ7xla4q 44 | 4yfR9ECo6e/XR+oN/CzKMpRsShAuilvdDox4zYDLNAz0GOCj9PGX1XzCEBadzPzU 45 | XcmgmKsvfHfhkWx7hr88oPFuNTQm7a2TmTTqLTRZMODBlAU6pM50BaKKkJa4MvIA 46 | H56dBPHd5fseCn+jemNLSn3wG+1IOEZ1TaDrPiTVTaCpGvZ2IMu7B8X2urqA8uLh 47 | wyszzxwJZNBsfk4+wRqblwcuRCoDNuj5X6gBHH8YBsxr+lx79RbdgqSWWUvB1XfI 48 | z3rQL0+eTC1pE4tB3yrulF+wJh4qMbCuVzEWb9/HWTewIIWskFRC6Kg5APVhaVyK 49 | M82NysBLj0vFMg6BF9eNK4CUiJxgDqZ15ZIX2AXTlNk1GJUf3Dq94uy+H06H6ThV 50 | khkaOOVMlJY8B+zY2t+EWk8WDzyLTeXtF7R0pndC/QcAm6ygtclZ9zdHyqcJ/tSP 51 | TDGY348/X77DMzW3sQJzGCnjYngUjQ/+R1kLQCCZNfB6Bn8VSgELyssEty+L6y4B 52 | lIHD+zb+71Qcmo+B3k1/Ji7TC8PqLR8LtreRxKydgNJgsl0BSDrDrQCaTuALUq9X 53 | 6G1PeLMpBP83IuSHDvgmumDbPFFwiWyofmqGMiVFb+r6LLB4td8fakmuguea4zA= 54 | =tQ9i 55 | -----END PGP PUBLIC KEY BLOCK----- 56 | -------------------------------------------------------------------------------- /packaging/etc/access-log-exporter/config.yaml: -------------------------------------------------------------------------------- 1 | # bufferSize: 1000 2 | # syslog: 3 | # listenAddress: "udp://[::]:8514" 4 | # web: 5 | # listenAddress: ":4040" 6 | # config: "" 7 | # workerCount: 0 8 | # preset: "simple" 9 | # log: 10 | # level: "info" 11 | # format: "console" 12 | # debug: 13 | # enabled: false 14 | presets: 15 | # apache 16 | # LogFormat "%v\t%m\t%>s\tOK\t%{ms}T\t%I\t%O" accesslog_exporter 17 | # CustomLog "|/usr/bin/logger --rfc3164 --server 127.0.0.1 --port 8514 --udp" accesslog_exporter 18 | # nginx 19 | # log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent'; 20 | # access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 21 | simple: 22 | metrics: 23 | - name: "http_requests_total" 24 | type: "counter" 25 | help: "The total number of client requests." 26 | labels: 27 | - name: "host" 28 | lineIndex: 0 29 | - name: "method" 30 | lineIndex: 1 31 | - name: "status" 32 | lineIndex: 2 33 | 34 | - name: "http_requests_completed_total" 35 | type: "counter" 36 | help: "The total number of completed requests." 37 | valueIndex: 3 38 | replacements: 39 | - string: "OK" 40 | replacement: "1" 41 | labels: 42 | - name: "host" 43 | lineIndex: 0 44 | - name: "method" 45 | lineIndex: 1 46 | - name: "status" 47 | lineIndex: 2 48 | 49 | - name: "http_request_size_bytes" 50 | type: "histogram" 51 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 52 | help: "The request length (including request line, header, and request body)" 53 | valueIndex: 5 54 | labels: 55 | - name: "host" 56 | lineIndex: 0 57 | - name: "method" 58 | lineIndex: 1 59 | - name: "status" 60 | lineIndex: 2 61 | 62 | - name: "http_response_size_bytes" 63 | type: "histogram" 64 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 65 | help: "The response length (including request line, header, and request body)" 66 | valueIndex: 6 67 | labels: 68 | - name: "host" 69 | lineIndex: 0 70 | - name: "method" 71 | lineIndex: 1 72 | - name: "status" 73 | lineIndex: 2 74 | 75 | - name: "http_request_duration_seconds" 76 | type: "histogram" 77 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 78 | help: "The time spent on receiving the response from the upstream server" 79 | valueIndex: 4 80 | math: 81 | enabled: true 82 | div: 1000 83 | labels: 84 | - name: "host" 85 | lineIndex: 0 86 | - name: "method" 87 | lineIndex: 1 88 | - name: "status" 89 | lineIndex: 2 90 | 91 | # apache 92 | # not supported 93 | # nginx 94 | # log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent\t$upstream_addr\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time'; 95 | # access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 96 | simple_upstream: 97 | metrics: 98 | - name: "http_requests_total" 99 | type: "counter" 100 | help: "The total number of client requests." 101 | labels: 102 | - name: "host" 103 | lineIndex: 0 104 | - name: "method" 105 | lineIndex: 1 106 | - name: "status" 107 | lineIndex: 2 108 | 109 | - name: "http_requests_completed_total" 110 | type: "counter" 111 | help: "The total number of completed requests." 112 | valueIndex: 3 113 | replacements: 114 | - string: "OK" 115 | replacement: "1" 116 | labels: 117 | - name: "host" 118 | lineIndex: 0 119 | - name: "method" 120 | lineIndex: 1 121 | - name: "status" 122 | lineIndex: 2 123 | 124 | - name: "http_request_size_bytes" 125 | type: "histogram" 126 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 127 | help: "The request length (including request line, header, and request body)" 128 | valueIndex: 5 129 | labels: 130 | - name: "host" 131 | lineIndex: 0 132 | - name: "method" 133 | lineIndex: 1 134 | - name: "status" 135 | lineIndex: 2 136 | 137 | - name: "http_response_size_bytes" 138 | type: "histogram" 139 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 140 | help: "The response length (including request line, header, and request body)" 141 | valueIndex: 6 142 | labels: 143 | - name: "host" 144 | lineIndex: 0 145 | - name: "method" 146 | lineIndex: 1 147 | - name: "status" 148 | lineIndex: 2 149 | 150 | - name: "http_request_duration_seconds" 151 | type: "histogram" 152 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 153 | help: "The time spent on receiving and response the response to the client" 154 | valueIndex: 4 155 | math: 156 | enabled: true 157 | div: 1000 158 | labels: 159 | - name: "host" 160 | lineIndex: 0 161 | - name: "method" 162 | lineIndex: 1 163 | - name: "status" 164 | lineIndex: 2 165 | 166 | - name: "http_upstream_connect_duration_seconds" 167 | type: "histogram" 168 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 169 | help: "The time spent on establishing a connection with the upstream server" 170 | valueIndex: 8 171 | math: 172 | enabled: true 173 | div: 1000 174 | upstream: 175 | enabled: true 176 | addrLineIndex: 7 177 | excludes: [] 178 | labels: 179 | - name: "host" 180 | lineIndex: 0 181 | - name: "method" 182 | lineIndex: 1 183 | - name: "status" 184 | lineIndex: 2 185 | 186 | - name: "http_upstream_header_duration_seconds" 187 | type: "histogram" 188 | help: "The time spent on receiving the response header from the upstream server" 189 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 190 | valueIndex: 9 191 | math: 192 | enabled: true 193 | div: 1000 194 | upstream: 195 | enabled: true 196 | addrLineIndex: 7 197 | excludes: [] 198 | labels: 199 | - name: "host" 200 | lineIndex: 0 201 | - name: "method" 202 | lineIndex: 1 203 | - name: "status" 204 | lineIndex: 2 205 | 206 | - name: "http_upstream_request_duration_seconds" 207 | type: "histogram" 208 | help: "The time spent on receiving the response from the upstream server" 209 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 210 | valueIndex: 10 211 | math: 212 | enabled: true 213 | div: 1000 214 | upstream: 215 | enabled: true 216 | addrLineIndex: 7 217 | excludes: [] 218 | labels: 219 | - name: "host" 220 | lineIndex: 0 221 | - name: "method" 222 | lineIndex: 1 223 | - name: "status" 224 | lineIndex: 2 225 | 226 | # apache 227 | # not supported 228 | # nginx 229 | # log_format accesslog_exporter '$http_host\t$request_method\t$status\t$request_completion\t$request_time\t$request_length\t$bytes_sent\t$upstream_addr\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time\t$request_uri'; 230 | # access_log syslog:server=127.0.0.1:8514 accesslog_exporter,nohostname; 231 | simple_uri_upstream: 232 | metrics: 233 | - name: "http_requests_total" 234 | type: "counter" 235 | help: "The total number of client requests." 236 | labels: 237 | - name: "host" 238 | lineIndex: 0 239 | - name: "method" 240 | lineIndex: 1 241 | - name: "status" 242 | lineIndex: 2 243 | - name: "path" 244 | lineIndex: 11 245 | replacements: 246 | - regexp: "^$" 247 | replacement: "/" 248 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 249 | replacement: "$1.+" 250 | - regexp: "^(^/[^/]+/[^/]+/).+" 251 | replacement: "$1.+" 252 | 253 | - name: "http_requests_completed_total" 254 | type: "counter" 255 | help: "The total number of completed requests." 256 | valueIndex: 3 257 | replacements: 258 | - string: "OK" 259 | replacement: "1" 260 | labels: 261 | - name: "host" 262 | lineIndex: 0 263 | - name: "method" 264 | lineIndex: 1 265 | - name: "status" 266 | lineIndex: 2 267 | - name: "path" 268 | lineIndex: 11 269 | replacements: 270 | - regexp: "^$" 271 | replacement: "/" 272 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 273 | replacement: "$1.+" 274 | - regexp: "^(^/[^/]+/[^/]+/).+" 275 | replacement: "$1.+" 276 | 277 | - name: "http_request_size_bytes" 278 | type: "histogram" 279 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 280 | help: "The request length (including request line, header, and request body)" 281 | valueIndex: 5 282 | labels: 283 | - name: "host" 284 | lineIndex: 0 285 | - name: "method" 286 | lineIndex: 1 287 | - name: "status" 288 | lineIndex: 2 289 | - name: "path" 290 | lineIndex: 11 291 | replacements: 292 | - regexp: "^$" 293 | replacement: "/" 294 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 295 | replacement: "$1.+" 296 | - regexp: "^(^/[^/]+/[^/]+/).+" 297 | replacement: "$1.+" 298 | 299 | - name: "http_response_size_bytes" 300 | type: "histogram" 301 | buckets: [ 10,1000,100000,1000000,5000000,50000000,200000000 ] 302 | help: "The response length (including request line, header, and request body)" 303 | valueIndex: 6 304 | labels: 305 | - name: "host" 306 | lineIndex: 0 307 | - name: "method" 308 | lineIndex: 1 309 | - name: "status" 310 | lineIndex: 2 311 | - name: "path" 312 | lineIndex: 11 313 | replacements: 314 | - regexp: "^$" 315 | replacement: "/" 316 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 317 | replacement: "$1.+" 318 | - regexp: "^(^/[^/]+/[^/]+/).+" 319 | replacement: "$1.+" 320 | 321 | - name: "http_request_duration_seconds" 322 | type: "histogram" 323 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 324 | help: "The time spent on receiving and response the response to the client" 325 | valueIndex: 4 326 | math: 327 | enabled: true 328 | div: 1000 329 | labels: 330 | - name: "host" 331 | lineIndex: 0 332 | - name: "method" 333 | lineIndex: 1 334 | - name: "status" 335 | lineIndex: 2 336 | - name: "path" 337 | lineIndex: 11 338 | replacements: 339 | - regexp: "^$" 340 | replacement: "/" 341 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 342 | replacement: "$1.+" 343 | - regexp: "^(^/[^/]+/[^/]+/).+" 344 | replacement: "$1.+" 345 | 346 | - name: "http_upstream_connect_duration_seconds" 347 | type: "histogram" 348 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 349 | help: "The time spent on establishing a connection with the upstream server" 350 | valueIndex: 8 351 | math: 352 | enabled: true 353 | div: 1000 354 | upstream: 355 | enabled: true 356 | addrLineIndex: 7 357 | excludes: [] 358 | labels: 359 | - name: "host" 360 | lineIndex: 0 361 | - name: "method" 362 | lineIndex: 1 363 | - name: "status" 364 | lineIndex: 2 365 | - name: "path" 366 | lineIndex: 11 367 | replacements: 368 | - regexp: "^$" 369 | replacement: "/" 370 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 371 | replacement: "$1.+" 372 | - regexp: "^(^/[^/]+/[^/]+/).+" 373 | replacement: "$1.+" 374 | 375 | - name: "http_upstream_header_duration_seconds" 376 | type: "histogram" 377 | help: "The time spent on receiving the response header from the upstream server" 378 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 379 | valueIndex: 9 380 | math: 381 | enabled: true 382 | div: 1000 383 | upstream: 384 | enabled: true 385 | addrLineIndex: 7 386 | excludes: [] 387 | labels: 388 | - name: "host" 389 | lineIndex: 0 390 | - name: "method" 391 | lineIndex: 1 392 | - name: "status" 393 | lineIndex: 2 394 | - name: "path" 395 | lineIndex: 11 396 | replacements: 397 | - regexp: "^$" 398 | replacement: "/" 399 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 400 | replacement: "$1.+" 401 | - regexp: "^(^/[^/]+/[^/]+/).+" 402 | replacement: "$1.+" 403 | 404 | - name: "http_upstream_request_duration_seconds" 405 | type: "histogram" 406 | help: "The time spent on receiving the response from the upstream server" 407 | buckets: [ .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10 ] 408 | valueIndex: 10 409 | math: 410 | enabled: true 411 | div: 1000 412 | upstream: 413 | enabled: true 414 | addrLineIndex: 7 415 | excludes: [] 416 | labels: 417 | - name: "host" 418 | lineIndex: 0 419 | - name: "method" 420 | lineIndex: 1 421 | - name: "status" 422 | lineIndex: 2 423 | - name: "path" 424 | lineIndex: 11 425 | replacements: 426 | - regexp: "^$" 427 | replacement: "/" 428 | - regexp: "^(^/api/[^/]+/[^/]+/).+" 429 | replacement: "$1.+" 430 | - regexp: "^(^/[^/]+/[^/]+/).+" 431 | replacement: "$1.+" 432 | -------------------------------------------------------------------------------- /packaging/scripts/postinst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | if ! command -v systemctl >/dev/null 2>&1; then 6 | exit 0 7 | fi 8 | 9 | systemctl --system daemon-reload >/dev/null || true 10 | 11 | if systemctl is-active --quiet access-log-exporter; then 12 | systemctl restart access-log-exporter >/dev/null || true 13 | fi 14 | 15 | systemd-sysusers /usr/lib/sysusers.d/access-log-exporter.conf >/dev/null || true 16 | 17 | if [ -d /etc/access-log-exporter ]; then 18 | chown -R root:access-log-exporter /etc/access-log-exporter/ >/dev/null || true 19 | fi 20 | -------------------------------------------------------------------------------- /packaging/scripts/postremove.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | case $1 in 6 | 1 | upgrade | failed-upgrade) 7 | # RPM: If the first argument is 1, it means the package is being upgraded. https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax 8 | # DEB: If the first argument is upgrade or failed-upgrade, it means the package is being upgraded. https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#summary-of-ways-maintainer-scripts-are-called 9 | ;; 10 | 11 | *) 12 | if id -g access-log-exporter >/dev/null 2>&1; then 13 | groupdel access-log-exporter >/dev/null || true 14 | fi 15 | 16 | if ! command -v systemctl >/dev/null 2>&1; then 17 | exit 0 18 | fi 19 | 20 | systemctl daemon-reload 21 | systemctl reset-failed 22 | ;; 23 | esac 24 | -------------------------------------------------------------------------------- /packaging/scripts/preinst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if ! command -v systemctl >/dev/null 2>&1; then 4 | exit 0 5 | fi 6 | -------------------------------------------------------------------------------- /packaging/scripts/preremove.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | case $1 in 6 | 1 | upgrade | failed-upgrade) 7 | # RPM: If the first argument is 1, it means the package is being upgraded. https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax 8 | # DEB: If the first argument is upgrade or failed-upgrade, it means the package is being upgraded. https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#summary-of-ways-maintainer-scripts-are-called 9 | ;; 10 | 11 | *) 12 | if ! command -v systemctl >/dev/null 2>&1; then 13 | exit 0 14 | fi 15 | 16 | systemctl stop access-log-exporter.service || true 17 | systemctl daemon-reload 18 | systemctl reset-failed 19 | ;; 20 | esac 21 | -------------------------------------------------------------------------------- /packaging/usr/lib/systemd/system/access-log-exporter.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Access log exporter service 3 | Documentation=https://github.com/jkroepke/access-log-exporter 4 | Wants=network-online.target 5 | After=network-online.target 6 | 7 | [Service] 8 | # Run as an unprivileged user with random user id. 9 | DynamicUser=true 10 | SupplementaryGroups=access-log-exporter 11 | 12 | ExecStart=/usr/bin/access-log-exporter -config-file ${CONFIG_FILE} 13 | NoExecPaths=/ 14 | ExecPaths=/usr/bin/access-log-exporter /usr/bin/env /usr/bin/kill 15 | ExecReload=/usr/bin/env kill -USR1 $MAINPID 16 | 17 | Environment=CONFIG_FILE=/etc/access-log-exporter/config.yaml 18 | 19 | RestartSec=5s 20 | Restart=always 21 | 22 | # We don't need write access anywhere. 23 | AmbientCapabilities= 24 | CapabilityBoundingSet= 25 | MemoryDenyWriteExecute=true 26 | NoNewPrivileges=true 27 | LockPersonality=true 28 | PrivateDevices=true 29 | PrivateTmp=true 30 | PrivateUsers=true 31 | ProcSubset=pid 32 | ProtectClock=true 33 | ProtectControlGroups=true 34 | ProtectHome=true 35 | ProtectHostname=true 36 | ProtectKernelLogs=true 37 | ProtectKernelModules=true 38 | ProtectKernelTunables=true 39 | ProtectProc=noaccess 40 | ProtectSystem=strict 41 | RemoveIPC=true 42 | RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 43 | RestrictNamespaces=true 44 | RestrictRealtime=true 45 | RestrictSUIDSGID=true 46 | SystemCallArchitectures=native 47 | SystemCallFilter=@io-event @file-system @basic-io @system-service 48 | SystemCallFilter=~@clock @debug @module @mount @obsolete @privileged @reboot @resources @setuid @swap 49 | SystemCallErrorNumber=EPERM 50 | UMask=0027 51 | 52 | [Install] 53 | WantedBy=multi-user.target 54 | -------------------------------------------------------------------------------- /packaging/usr/lib/sysusers.d/access-log-exporter.conf: -------------------------------------------------------------------------------- 1 | #Type Name ID GECOS Home directory Shell 2 | 3 | g access-log-exporter - - 4 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["github>jkroepke/renovate-config"] 4 | } 5 | --------------------------------------------------------------------------------