├── .github ├── CODEOWNERS ├── workflows │ ├── mark-as-done.yaml │ ├── validate-renovate.yml │ ├── integration.yml │ ├── codeql.yml │ ├── release.yml │ └── test.yml ├── dependabot.yml └── zizmor.yml ├── mypy.ini ├── .gitignore ├── sigma ├── backends │ └── loki │ │ ├── __init__.py │ │ └── deferred.py ├── pipelines │ └── loki │ │ ├── __init__.py │ │ └── loki.py └── shared.py ├── tests ├── test_sigma_rule.yml ├── test_pysigma_integration.py ├── test_sigma_cli_integration.sh ├── test_backend_loki_value_count_correlation.py ├── test_backend_loki_field_modifiers.py ├── test_backend_loki_fieldref.py ├── test_backend_loki_event_count_correlation.py ├── sigma_backend_tester.py ├── test_backend_negation_loki.py ├── test_backend_loki_add_line_filters_case_insensitive.py ├── test_backend_loki_add_line_filters.py └── test_pipelines_loki.py ├── renovate.json ├── LICENSE ├── pyproject.toml ├── .githooks └── pre-commit ├── getting_started.md └── README.md /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @grafana/security-operations 2 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | ignore_missing_imports = True 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage* 2 | .vscode/ 3 | .idea/ 4 | **/__pycache__ 5 | .pytest_cache/ 6 | cov.xml 7 | dist/ 8 | docs/_build 9 | -------------------------------------------------------------------------------- /sigma/backends/loki/__init__.py: -------------------------------------------------------------------------------- 1 | from .loki import LogQLBackend 2 | 3 | __all__ = ("LogQLBackend",) 4 | 5 | backends = { 6 | "loki": LogQLBackend, 7 | } 8 | -------------------------------------------------------------------------------- /tests/test_sigma_rule.yml: -------------------------------------------------------------------------------- 1 | title: Test Sigma File 2 | id: b9030a45-f041-49b5-a948-841c097a6dc7 3 | status: test 4 | description: A testing file to check basic sigma conversion functionality 5 | references: 6 | - https://github.com/SigmaHQ/sigma-specification/blob/main/Sigma_specification.md 7 | author: kelnage 8 | date: 2023/09/04 9 | logsource: 10 | category: testing 11 | detection: 12 | test: 13 | file_name: field_value 14 | condition: test 15 | -------------------------------------------------------------------------------- /.github/workflows/mark-as-done.yaml: -------------------------------------------------------------------------------- 1 | name: Mark as Done 2 | permissions: {} 3 | 4 | on: 5 | push: 6 | tags: 7 | - v* 8 | 9 | jobs: 10 | mark-as-done: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Move closed issues and merged PRs to the "✅ Done" column 14 | uses: grafana/auto-mate@fec4390aee45a9b143209fc465315b82a7032e00 # v1 15 | with: 16 | token: ${{ secrets.AUTO_MATE_TOKEN }} 17 | projectNo: 346 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["github>grafana/grafana-renovate-config//presets/base"], 4 | "packageRules": [ 5 | { 6 | "matchManagers": ["poetry"], 7 | "rangeStrategy": "replace" 8 | }, 9 | { 10 | "description": "Group together all GitHub Action updates in a single weekly commit (Tuesday, midnight GMT)", 11 | "matchDepTypes": [ 12 | "action" 13 | ], 14 | "groupName": "GH Actions", 15 | "schedule": ["* 0-2 * * 2"] 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2022 Nick Moore 2 | 3 | This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3. 4 | 5 | This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. 6 | 7 | You should have received a copy of the GNU Affero General Public License along with this program. If not, see https://www.gnu.org/licenses 8 | -------------------------------------------------------------------------------- /.github/zizmor.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | unpinned-uses: 3 | config: 4 | policies: 5 | "*": hash-pin 6 | actions/*: any 7 | github/*: any 8 | grafana/*: any 9 | forbidden-uses: 10 | config: 11 | deny: 12 | # Policy-banned by our security team due to CVE-2025-30066 & CVE-2025-30154. 13 | # https://www.cisa.gov/news-events/alerts/2025/03/18/supply-chain-compromise-third-party-tj-actionschanged-files-cve-2025-30066-and-reviewdogaction 14 | # https://nvd.nist.gov/vuln/detail/cve-2025-30066 15 | # https://nvd.nist.gov/vuln/detail/cve-2025-30154 16 | - reviewdog/* 17 | -------------------------------------------------------------------------------- /.github/workflows/validate-renovate.yml: -------------------------------------------------------------------------------- 1 | name: Validate Renovate Config 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "renovate.json" 7 | push: 8 | branches: 9 | - main 10 | paths: 11 | - "renovate.json" 12 | 13 | jobs: 14 | validate: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 19 | with: 20 | persist-credentials: false 21 | - name: Validate Renovate Config 22 | uses: grafana/shared-workflows/actions/validate-renovate-config@4178c5d9a0403e8b8e123baebc66e224d9c4f3cd # v0.1.0 23 | 24 | -------------------------------------------------------------------------------- /sigma/pipelines/loki/__init__.py: -------------------------------------------------------------------------------- 1 | from .loki import ( 2 | LokiCustomAttributes, 3 | SetCustomAttributeTransformation, 4 | CustomLogSourceTransformation, 5 | loki_grafana_logfmt, 6 | loki_promtail_sysmon, 7 | loki_okta_system_log, 8 | ) 9 | 10 | __all__ = ( 11 | "LokiCustomAttributes", 12 | "SetCustomAttributeTransformation", 13 | "CustomLogSourceTransformation", 14 | "loki_grafana_logfmt", 15 | "loki_promtail_sysmon", 16 | "loki_okta_system_log", 17 | ) 18 | 19 | pipelines = { 20 | "loki_grafana_logfmt": loki_grafana_logfmt, 21 | "loki_promtail_sysmon": loki_promtail_sysmon, 22 | "loki_okta_system_log": loki_okta_system_log, 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: Integration 2 | permissions: {} 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | pull_request: 7 | branches: [ "*" ] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | test: 12 | runs-on: 'ubuntu-latest' 13 | steps: 14 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 15 | with: 16 | persist-credentials: false 17 | - name: Set up Python 18 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6 19 | with: 20 | python-version: '3.10' 21 | - name: Install Poetry 22 | uses: abatilo/actions-poetry@3765cf608f2d4a72178a9fc5b918668e542b89b1 # v4.0.0 23 | with: 24 | poetry-version: '1.8' 25 | - name: Install dependencies 26 | run: poetry install 27 | - name: Run integration tests 28 | run: sh tests/test_sigma_cli_integration.sh 29 | 30 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pySigma-backend-loki" 3 | version = "0.13.0" 4 | description = "pySigma Loki backend" 5 | readme = "README.md" 6 | authors = [ 7 | "Nick Moore ", 8 | ] 9 | license = "AGPL-3.0-only" 10 | repository = "https://github.com/grafana/pySigma-backend-loki" 11 | packages = [{ include = "sigma" }] 12 | 13 | [tool.poetry.dependencies] 14 | python = "^3.10" 15 | pysigma = "^1.0.0" 16 | 17 | [tool.poetry.group.dev.dependencies] 18 | coverage = "^7.12.0" 19 | mypy = { extras = ["faster-cache"], version = "^1.19.0" } 20 | pytest = "^9.0.0" 21 | pytest-cov = "^7.0.0" 22 | pytest-mypy = "^1.0.0" 23 | ruff = "^0.14.0" 24 | types-pyyaml = "^6.0.12" 25 | 26 | [tool.ruff] 27 | exclude = ["dist", "build", "env", "venv", ".env", ".venv"] 28 | line-length = 100 29 | 30 | [build-system] 31 | requires = ["poetry-core>=1.0.0"] 32 | build-backend = "poetry.core.masonry.api" 33 | -------------------------------------------------------------------------------- /tests/test_pysigma_integration.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sigma.plugins import InstalledSigmaPlugins 3 | from sigma.backends.loki import LogQLBackend 4 | from sigma.pipelines.loki import ( 5 | loki_grafana_logfmt, 6 | loki_promtail_sysmon, 7 | loki_okta_system_log, 8 | ) 9 | 10 | 11 | @pytest.fixture 12 | def installed() -> InstalledSigmaPlugins: 13 | return InstalledSigmaPlugins.autodiscover() 14 | 15 | 16 | def test_auto_discover_loki_backend(installed: InstalledSigmaPlugins): 17 | assert "loki" in installed.backends 18 | assert installed.backends["loki"] is LogQLBackend 19 | 20 | 21 | def test_auto_discover_loki_pipelines(installed: InstalledSigmaPlugins): 22 | loki_pipelines = [ 23 | loki_grafana_logfmt, 24 | loki_promtail_sysmon, 25 | loki_okta_system_log, 26 | ] 27 | for pipeline in loki_pipelines: 28 | assert pipeline.__name__ in installed.pipelines 29 | assert installed.pipelines[pipeline.__name__] == pipeline 30 | -------------------------------------------------------------------------------- /.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | RESULT=0 4 | 5 | check() { 6 | output=$($1 2>&1) 7 | result=$? 8 | if [ "$result" -eq "0" ]; then 9 | echo "✅ PASS: $1" 10 | else 11 | echo "❌ FAIL: $1" 12 | echo "$output" 13 | RESULT=1 14 | fi 15 | } 16 | 17 | CHANGED_FILES=$(git diff --name-only --cached --diff-filter=ACMR) 18 | if [ -z "$CHANGED_FILES" ]; then 19 | echo "No relevant files added/changed, no checks run" 20 | exit 0 21 | fi 22 | 23 | POETRY_CHANGED_FILES=$(echo "$CHANGED_FILES" | grep --extended-regexp "^(pyproject\.toml|poetry\.lock)$") 24 | if [ -n "$POETRY_CHANGED_FILES" ]; then 25 | check "poetry check" 26 | check "poetry lock --check" # deprecated, remove when upgrading to poetry v2 27 | check "poetry install" 28 | fi 29 | 30 | PY_CHANGED_FILES=$(echo "$CHANGED_FILES" | grep "\.py$") 31 | if [ -n "$PY_CHANGED_FILES" ]; then 32 | PY_CHANGED_FILES_ARGS=$(echo "$PY_CHANGED_FILES" | tr '\n' ' ') 33 | check "poetry run mypy --explicit-package-bases $PY_CHANGED_FILES_ARGS" 34 | check "poetry run ruff check $PY_CHANGED_FILES_ARGS" 35 | fi 36 | 37 | exit $RESULT 38 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | name: "CodeQL" 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | # The branches below must be a subset of the branches above 11 | branches: [ "main" ] 12 | schedule: 13 | - cron: '37 21 * * 2' 14 | 15 | jobs: 16 | analyze: 17 | name: Analyze 18 | runs-on: ubuntu-latest 19 | permissions: 20 | actions: read 21 | contents: read 22 | security-events: write 23 | 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | language: [ 'python' ] 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 32 | with: 33 | persist-credentials: false 34 | 35 | # Initializes the CodeQL tools for scanning. 36 | - name: Initialize CodeQL 37 | uses: github/codeql-action/init@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v4 38 | with: 39 | languages: ${{ matrix.language }} 40 | # If you wish to specify custom queries, you can do so here or in a config file. 41 | # By default, queries listed here will override any specified in a config file. 42 | # Prefix the list here with "+" to use these queries and those in the config file. 43 | 44 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 45 | # queries: security-extended,security-and-quality 46 | 47 | - name: Perform CodeQL Analysis 48 | uses: github/codeql-action/analyze@cf1bb45a277cb3c205638b2cd5c984db1c46a412 # v4 49 | with: 50 | category: "/language:${{matrix.language}}" 51 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release to PyPI 2 | permissions: {} 3 | on: 4 | release: 5 | types: [published] 6 | push: 7 | tags: 8 | - v*.*.* 9 | 10 | jobs: 11 | build-and-publish: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: read 15 | id-token: write 16 | steps: 17 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 18 | with: 19 | persist-credentials: false 20 | - name: Set up Python 21 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6 22 | with: 23 | python-version: '3.10' # quoted as otherwise yaml treats as a number 24 | - name: Install Poetry 25 | uses: abatilo/actions-poetry@3765cf608f2d4a72178a9fc5b918668e542b89b1 # v4.0.0 26 | with: 27 | poetry-version: '1.8' 28 | - name: Verify versioning 29 | run: | 30 | [ "$(poetry version -s)" == "${GITHUB_REF#refs/tags/v}" ] 31 | - name: Install dependencies 32 | run: poetry install 33 | - name: Run tests 34 | run: poetry run pytest 35 | - name: Build packages 36 | run: poetry build 37 | - name: Get service account token for PyPI 38 | uses: grafana/shared-workflows/actions/get-vault-secrets@main 39 | with: 40 | repo_secrets: | 41 | TEST_PYPI_API_TOKEN=pypi-deploy:pypi_test 42 | PYPI_API_TOKEN=pypi-deploy:pypi_main 43 | - name: Configure Poetry 44 | run: | 45 | poetry config repositories.testpypi https://test.pypi.org/legacy/ 46 | poetry config pypi-token.testpypi "${{ env.TEST_PYPI_API_TOKEN }}" 47 | poetry config pypi-token.pypi "${{ env.PYPI_API_TOKEN }}" 48 | - name: Publish to test PyPI 49 | if: ${{ github.event_name == 'push' }} 50 | run: poetry publish -r testpypi 51 | - name: Publish to PyPI 52 | if: ${{ github.event_name == 'release' }} 53 | run: poetry publish 54 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | permissions: {} 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | pull_request: 7 | branches: [ "*" ] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | test: 12 | strategy: 13 | matrix: 14 | os: [ 'ubuntu-latest' ] 15 | python-version: [ '3.10', '3.11', '3.12', '3.13', '3.14' ] 16 | poetry-version: [ '2.2' ] 17 | runs-on: ${{ matrix.os }} 18 | steps: 19 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 20 | with: 21 | persist-credentials: false 22 | - name: Set up Python 23 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install Poetry 27 | uses: abatilo/actions-poetry@3765cf608f2d4a72178a9fc5b918668e542b89b1 # v4.0.0 28 | with: 29 | poetry-version: ${{ matrix.poetry-version }} 30 | - name: Install dependencies 31 | run: poetry install 32 | - name: Run formatter, linter and type checker 33 | run: | 34 | poetry run mypy --explicit-package-bases . 35 | poetry run ruff check . 36 | - name: Run unit tests 37 | run: poetry run pytest --cov=sigma --cov-report term --cov-report lcov:coverage.lcov -vv 38 | - name: Submit coverage report to Coveralls 39 | if: ${{ success() }} 40 | uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 41 | with: 42 | github-token: ${{ secrets.GITHUB_TOKEN }} 43 | path-to-lcov: ./coverage.lcov 44 | parallel: true 45 | flag-name: python-${{ matrix.python-version }} 46 | finish: 47 | needs: test 48 | runs-on: ubuntu-latest 49 | steps: 50 | - name: Finish coveralls 51 | uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 52 | with: 53 | github-token: ${{ secrets.GITHUB_TOKEN }} 54 | parallel-finished: true 55 | carryforward: "python-3.10,python-3.11,python-3.12,python-3.13,python-3.14" 56 | -------------------------------------------------------------------------------- /tests/test_sigma_cli_integration.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | plugin="loki" 5 | 6 | # Ensure current environment is up-to-date 7 | poetry install 8 | poetry run pip install sigma-cli==1.1.0rc1 9 | 10 | sigma=$(poetry run sigma version) 11 | 12 | # Fetch plugin metadata from sigma-cli 13 | # Relies on structured output from sigma-cli 14 | meta=$(poetry run sigma plugin list | grep "^| $plugin" | cat) # Avoid termination due to grep not matching 15 | if [ "$meta" = "" ]; then 16 | echo "❌ FAIL: Could not find metadata for pySigma-backend-$plugin - check it appears in the plugin directory: https://github.com/SigmaHQ/pySigma-plugin-directory/" 17 | exit 1 18 | fi 19 | echo "✅ PASS: pySigma-backend-$plugin was found in the plugin list" 20 | 21 | # Check that sigma-cli believes the plugin is compatible 22 | compatible=$(echo "$meta" | awk -F "|" '{print $6}' | awk '{$1=$1};1') 23 | force_arg="" 24 | if [ "$compatible" = "no" ]; then 25 | echo "⚠️ WARN: This version of pySigma-backend-$plugin is not compatible with the latest version of sigma-cli ($sigma) - the plugin directory may require updating: https://github.com/SigmaHQ/pySigma-plugin-directory/" 26 | force_arg=" --force-install" 27 | else 28 | echo "✅ PASS: pySigma-backend-$plugin is compatible with the latest version of sigma-cli ($sigma)" 29 | fi 30 | 31 | # Check the plugin can be successfully installed 32 | install_logs=$(poetry run sigma plugin install $force_arg "$plugin") 33 | install_status=$(echo "$install_logs" | grep "Successfully installed plugin '$plugin'" | cat) 34 | if [ "$install_status" = "" ]; then 35 | echo "❌ FAIL: Installing this version of pySigma-backend-$plugin was not successful. The install logs were:" 36 | echo "$install_logs" 37 | exit 3 38 | fi 39 | echo "✅ PASS: pySigma-backend-$plugin was successfully installed as a plugin" 40 | 41 | # Check the plugin can be used successfully to convert a simple rule 42 | # May need tweaking if the rule is not supported or really requires a pipeline 43 | convert_err=$(poetry run sigma convert -t $plugin --without-pipeline tests/test_sigma_rule.yml 2>&1 | grep "^Error: " | cat) 44 | if [ "$convert_err" != "" ]; then 45 | echo "❌ FAIL: pySigma-backend-$plugin was not able to convert a simple test rule. The error message was:" 46 | echo "$convert_err" 47 | exit 4 48 | fi 49 | 50 | echo "✅ PASS: pySigma-backend-$plugin was able to convert a simple test rule" 51 | exit 0 52 | 53 | -------------------------------------------------------------------------------- /sigma/backends/loki/deferred.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import auto 3 | from typing import List, Union 4 | 5 | from sigma.conversion.deferred import DeferredQueryExpression 6 | from sigma.types import SigmaRegularExpression, SigmaString 7 | 8 | from sigma.shared import ( 9 | join_or_values_re, 10 | negated_line_filter_operator, 11 | negated_label_filter_operator, 12 | ) 13 | 14 | 15 | class LogQLDeferredType: 16 | """The different types of deferred expressions that can be created by this backend""" 17 | 18 | STR = auto() 19 | CIDR = auto() 20 | REGEXP = auto() 21 | OR_STR = auto() 22 | FIELD_REF = auto() 23 | 24 | 25 | @dataclass 26 | class LogQLDeferredUnboundStrExpression(DeferredQueryExpression): 27 | """'Defer' unbounded matching to pipelined command **BEFORE** main search expression.""" 28 | 29 | value: str 30 | op: str = "|=" # default to matching 31 | 32 | def negate(self) -> DeferredQueryExpression: 33 | self.op = negated_line_filter_operator[self.op] 34 | return self 35 | 36 | def finalize_expression(self) -> str: 37 | return f"{self.op} {self.value}" 38 | 39 | 40 | @dataclass 41 | class LogQLDeferredUnboundCIDRExpression(DeferredQueryExpression): 42 | """'Defer' unbounded matching of CIDR to pipelined command **BEFORE** main search expression.""" 43 | 44 | ip: str 45 | op: str = "|=" # default to matching 46 | 47 | def negate(self) -> DeferredQueryExpression: 48 | self.op = negated_line_filter_operator[self.op] 49 | return self 50 | 51 | def finalize_expression(self) -> str: 52 | return f'{self.op} ip("{self.ip}")' 53 | 54 | 55 | @dataclass 56 | class LogQLDeferredUnboundRegexpExpression(DeferredQueryExpression): 57 | """'Defer' unbounded matching of regex to pipelined command **BEFORE** main search 58 | expression.""" 59 | 60 | regexp: str 61 | op: str = "|~" # default to matching 62 | 63 | def negate(self) -> DeferredQueryExpression: 64 | self.op = negated_line_filter_operator[self.op] 65 | return self 66 | 67 | def finalize_expression(self) -> str: 68 | if "`" in self.regexp: 69 | value = '"' + SigmaRegularExpression(self.regexp).escape(['"']) + '"' 70 | else: 71 | value = f"`{self.regexp}`" 72 | return f"{self.op} {value}" 73 | 74 | 75 | @dataclass 76 | class LogQLDeferredOrUnboundExpression(DeferredQueryExpression): 77 | """'Defer' unbounded OR matching to pipelined command **BEFORE** main search expression.""" 78 | 79 | exprs: List[Union[SigmaString, SigmaRegularExpression]] 80 | op: str = "|~" # default to matching 81 | case_insensitive: bool = True 82 | 83 | def negate(self) -> DeferredQueryExpression: 84 | self.op = negated_line_filter_operator[self.op] 85 | return self 86 | 87 | def finalize_expression(self) -> str: 88 | return f"{self.op} {join_or_values_re(self.exprs, self.case_insensitive)}" 89 | 90 | 91 | @dataclass 92 | class LogQLDeferredLabelFormatExpression(DeferredQueryExpression): 93 | """'Defer' field reference matching to pipelined command **AFTER** main search expression.""" 94 | 95 | label: str 96 | template: str 97 | 98 | def finalize_expression(self) -> str: 99 | return f"{self.label}=`{self.template}`" 100 | 101 | 102 | @dataclass 103 | class LogQLDeferredLabelFilterExpression(DeferredQueryExpression): 104 | """ 105 | 'Defer' generated label matching to after the label_format expressions 106 | """ 107 | 108 | field: str 109 | op: str = "=" 110 | value: str = "true" 111 | 112 | def negate(self) -> DeferredQueryExpression: 113 | self.op = negated_label_filter_operator[self.op] 114 | return self 115 | 116 | def finalize_expression(self) -> str: 117 | return f"{self.field}{self.op}`{self.value}`" 118 | -------------------------------------------------------------------------------- /getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | This guide assumes you have: 4 | * One or more systems that are generating **log data** 5 | * One or more [Sigma rules](https://github.com/SigmaHQ/sigma/tree/master/rules) that you wish identify in that log data through **queries** 6 | * (Optionally) One of more Sigma rules that you want to receive **alerts** for when it matches incoming log entries (WIP) 7 | 8 | ## Grafana Loki set-up 9 | 10 | 1. Install, configure and start [Grafana](https://grafana.com/docs/grafana/latest/#installing-grafana) and [Grafana Loki](https://grafana.com/docs/loki/latest/installation/) 11 | * Ensure that your Grafana instance and Loki instances are connected, and that Loki is configured as a data source 12 | * Don't want to host these yourself? Try [Grafana Cloud](https://grafana.com/docs/grafana-cloud/quickstart/) 13 | 2. Install [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/installation/) and [configure it](https://grafana.com/docs/loki/latest/clients/promtail/configuration/) to scrape the log data from the target system and send it on to your Loki instance 14 | * If you are using Grafana Cloud, you can automatically generate a [Promtail configuration](https://grafana.com/docs/grafana-cloud/data-configuration/logs/collect-logs-with-promtail/), adjusting the `scrape_configs` stanza to reflect the target system 15 | 3. Start Promtail, wait a minute or two, and validate that the expected log data is being received 16 | 1. In Grafana, go to the **Explore** page (the compass icon on the left-hand menu) 17 | 2. Ensure your Loki instance is selected in the top-left corner 18 | 3. Use the Label filters pull-downs to see the relevant labels that are being sent to Loki and their respective values 19 | 4. Select a relevant label and value, and click on the **Run query** button in the top-right corner 20 | 5. Check that any logs come back and they match the format you expected 21 | 22 | ## Sigma set-up 23 | 24 | 1. Ensure you have the following installed: 25 | * Git 26 | * [Python 3](https://wiki.python.org/moin/BeginnersGuide/Download) (3.9 or newer, check with `python --version`) 27 | 2. Install the Sigma command line tool, e.g., by [following these instructions](https://sigmahq.io/docs/guide/getting-started.html) 28 | 3. Once installed, install the Loki backend: 29 | ``` 30 | sigma plugin install loki 31 | ``` 32 | 33 | ## Rule conversion - queries 34 | 35 | With both Loki and Sigma setup, you can start converting Sigma rules into Loki queries. Use git to clone the [Sigma rules repository](https://github.com/SigmaHQ/sigma/): 36 | ``` 37 | git clone https://github.com/SigmaHQ/sigma.git 38 | ``` 39 | 40 | To convert a specific rule into a Loki query, you use the `sigma convert` command, with arguments telling it that you want to produce a Loki query, what file(s) to convert, and (optionally) providing one or more pipelines to adjust the rule to make sure it works correctly for your data. For example: 41 | ``` 42 | sigma convert -t loki sigma/rules/web/web_cve_2021_43798_grafana.yml # this generated query will likely not work! 43 | ``` 44 | 45 | The above converts a rule designed to detect an old vulnerability in Grafana into a Loki query, using the field names defined in the rule. However, the Grafana logs stored within Loki will likely not match the fields used by Sigma rules. Hence you need to use the `loki_grafana_logfmt` pipeline to make the query work: 46 | ``` 47 | sigma convert -t loki sigma/rules/web/web_cve_2021_43798_grafana.yml -p loki_grafana_logfmt 48 | ``` 49 | 50 | A similar process is used when querying Windows System Monitor (sysmon) event data (such as the rules in sigma/rules/windows/sysmon/). Assuming you are [using Promtail](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#windows_events) to collect the sysmon logs, you will need to combine two pipelines; `sysmon` and `loki_promtail_sysmon`. This command will convert all those sysmon rules into queries: 51 | ``` 52 | sigma convert -t loki sigma/rules/windows/sysmon/ -p sysmon -p loki_promtail_sysmon 53 | ``` 54 | 55 | The sigma-cli tool does not support rules that include deprecated Sigma functionality - use the `-s` flag to ignore those rules when converting multiple rule files. 56 | 57 | You will likely need to ingest a wider range of log data than the two examples shown above - [contributions of or suggestions for new pipelines](https://github.com/grafana/pySigma-backend-loki/issues) are more than welcome. 58 | 59 | ## Rule conversion - alerts 60 | 61 | Coming soon! 62 | -------------------------------------------------------------------------------- /sigma/shared.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Dict, List, Union 3 | 4 | from sigma.types import ( 5 | SigmaCasedString, 6 | SigmaString, 7 | SigmaRegularExpression, 8 | SpecialChars, 9 | ) 10 | 11 | negated_line_filter_operator: Dict[str, str] = { 12 | "|=": "!=", 13 | "!=": "|=", 14 | "|~": "!~", 15 | "!~": "|~", 16 | } 17 | 18 | negated_label_filter_operator: Dict[str, str] = { 19 | "=": "!=", 20 | "==": "!=", 21 | "!=": "=", 22 | ">": "<=", 23 | ">=": "<", 24 | "<": ">=", 25 | "<=": ">", 26 | } 27 | 28 | 29 | def sanitize_label_key(key: str, isprefix: bool = True) -> str: 30 | """Implements the logic used by Loki to sanitize labels. 31 | 32 | See: https://github.com/grafana/loki/blob/main/pkg/logql/log/util.go#L21""" 33 | # pySigma treats null or empty fields as unbound expressions, rather than keys 34 | if key is None or len(key) == 0: # pragma: no cover 35 | return "" 36 | key = key.strip() 37 | if len(key) == 0: 38 | return key 39 | if isprefix and key[0] >= "0" and key[0] <= "9": 40 | key = "_" + key 41 | return "".join( 42 | ( 43 | ( 44 | r 45 | if (r >= "a" and r <= "z") 46 | or (r >= "A" and r <= "Z") 47 | or r == "_" 48 | or (r >= "0" and r <= "9") 49 | else "_" 50 | ) 51 | for r in key 52 | ) 53 | ) 54 | 55 | 56 | def quote_string_value(s: SigmaString) -> str: 57 | """By default, use the tilde character to quote fields, which needs limited escaping. 58 | If the value contains a tilde character, use double quotes and apply more rigourous 59 | escaping.""" 60 | quote = "`" 61 | if any([c == quote for c in str(s)]): 62 | quote = '"' 63 | # If our string doesn't contain any tilde characters 64 | if quote == "`": 65 | converted = s.convert() 66 | else: 67 | converted = s.convert(escape_char="\\", add_escaped='"\\') 68 | return f"{quote}{converted}{quote}" 69 | 70 | 71 | def convert_str_to_re( 72 | value: SigmaString, 73 | case_insensitive: bool = True, 74 | field_filter: bool = False, 75 | ) -> SigmaRegularExpression: 76 | """Convert a SigmaString into a regular expression, replacing any 77 | wildcards with equivalent regular expression operators, and enforcing 78 | case-insensitive matching""" 79 | return SigmaRegularExpression( 80 | ("(?i)" if case_insensitive else "") 81 | + ("^" if field_filter and not value.startswith(SpecialChars.WILDCARD_MULTI) else "") 82 | + re.escape(str(value)).replace("\\?", ".").replace("\\*", ".*") 83 | + ("$" if field_filter and not value.endswith(SpecialChars.WILDCARD_MULTI) else "") 84 | ) 85 | 86 | 87 | def escape_and_quote_re(r: SigmaRegularExpression, flag_prefix=True) -> str: 88 | """LogQL does not require any additional escaping for regular expressions if we 89 | can use the tilde character""" 90 | if "`" in str(r.regexp): 91 | return ( 92 | '"' 93 | + r.escape( 94 | [ 95 | '"', 96 | ], 97 | flag_prefix=flag_prefix, 98 | ) 99 | + '"' 100 | ) 101 | return "`" + r.escape([], "", False, flag_prefix) + "`" 102 | 103 | 104 | def join_or_values_re( 105 | exprs: List[Union[SigmaString, SigmaRegularExpression]], case_insensitive: bool 106 | ) -> str: 107 | # This makes the regex case insensitive if any values are SigmaStrings 108 | # or if any of the regexes are case insensitive 109 | # TODO: can we make this more precise? 110 | case_insensitive = any( 111 | ( 112 | isinstance(val, SigmaString) 113 | and case_insensitive 114 | and not isinstance(val, SigmaCasedString) 115 | ) 116 | or (isinstance(val, SigmaRegularExpression) and str(val.regexp).startswith("(?i)")) 117 | for val in exprs 118 | ) 119 | vals = [ 120 | convert_str_to_re(val) if isinstance(val, SigmaString) and val.contains_special() else val 121 | for val in exprs 122 | ] 123 | or_value = "|".join( 124 | ( 125 | ( 126 | re.escape(str(val)) 127 | if isinstance(val, SigmaString) 128 | else re.sub("^\\(\\?i\\)", "", str(val.regexp)) 129 | ) 130 | for val in vals 131 | ) 132 | ) 133 | if case_insensitive: 134 | or_value = f"(?i){or_value}" 135 | if "`" in or_value: 136 | or_value = '"' + SigmaRegularExpression(or_value).escape(['"']) + '"' 137 | else: 138 | or_value = f"`{or_value}`" 139 | return or_value 140 | -------------------------------------------------------------------------------- /tests/test_backend_loki_value_count_correlation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from sigma.backends.loki import LogQLBackend 4 | from sigma.collection import SigmaCollection 5 | 6 | from sigma.pipelines.loki import loki_okta_system_log 7 | 8 | 9 | @pytest.fixture 10 | def loki_backend(): 11 | return LogQLBackend() 12 | 13 | 14 | def test_loki_default_value_count_no_group(loki_backend: LogQLBackend): 15 | rules = SigmaCollection.from_yaml( 16 | """ 17 | title: Test Rule 18 | name: test_rule 19 | status: test 20 | logsource: 21 | category: test_category 22 | product: test_product 23 | detection: 24 | sel: 25 | fieldA: valueA 26 | condition: sel 27 | --- 28 | title: Test Correlation 29 | status: test 30 | correlation: 31 | type: value_count 32 | rules: 33 | - test_rule 34 | timespan: 30s 35 | condition: 36 | field: fieldB 37 | eq: 42 38 | """ 39 | ) 40 | queries = loki_backend.convert(rules) 41 | assert queries == [ 42 | 'count without (fieldB) (sum by (fieldB) (count_over_time({job=~".+"} | ' 43 | "logfmt | fieldA=~`(?i)^valueA$` [30s]))) == 42" 44 | ] 45 | 46 | 47 | def test_loki_default_value_count_single_group(loki_backend: LogQLBackend): 48 | rules = SigmaCollection.from_yaml( 49 | """ 50 | title: Test Rule 51 | name: test_rule 52 | status: test 53 | logsource: 54 | category: test_category 55 | product: test_product 56 | detection: 57 | sel: 58 | fieldA: valueA 59 | condition: sel 60 | --- 61 | title: Test Correlation 62 | status: test 63 | correlation: 64 | type: value_count 65 | rules: 66 | - test_rule 67 | group-by: 68 | - fieldB 69 | timespan: 5m 70 | condition: 71 | field: fieldC 72 | gte: 1 73 | """ 74 | ) 75 | queries = loki_backend.convert(rules) 76 | assert queries == [ 77 | "count without (fieldC) (sum by (fieldB, fieldC) (count_over_time(" 78 | '{job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` [5m]))) >= 1' 79 | ] 80 | 81 | 82 | def test_loki_default_value_count_multiple_fields(loki_backend: LogQLBackend): 83 | rules = SigmaCollection.from_yaml( 84 | """ 85 | title: Test Rule 86 | name: test_rule 87 | status: test 88 | logsource: 89 | category: test_category 90 | product: test_product 91 | detection: 92 | sel: 93 | fieldA: valueA 94 | condition: sel 95 | --- 96 | title: Test Correlation 97 | status: test 98 | correlation: 99 | type: value_count 100 | rules: 101 | - test_rule 102 | group-by: 103 | - fieldB 104 | - fieldC 105 | timespan: 1d 106 | condition: 107 | field: fieldD 108 | lt: 100 109 | """ 110 | ) 111 | queries = loki_backend.convert(rules) 112 | assert queries == [ 113 | "count without (fieldD) (sum by (fieldB, fieldC, fieldD) (count_over_time(" 114 | '{job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` [1d]))) < 100' 115 | ] 116 | 117 | 118 | def test_loki_okta_country_count(): 119 | pipeline = loki_okta_system_log() 120 | # Note: using 121 | rules = SigmaCollection.from_yaml( 122 | """ 123 | title: Okta User Activity With Country Defined 124 | id: 79bbc335-7ab0-4316-a17b-30c85f7f0595 125 | status: experimental 126 | description: Detects any Okta activity that includes a country 127 | references: 128 | - https://developer.okta.com/docs/reference/api/system-log/ 129 | author: kelnage 130 | date: 2024-08-01 131 | logsource: 132 | product: okta 133 | service: okta 134 | detection: 135 | selection: 136 | actor.alternateid|exists: true 137 | client.geographicalcontext.country|exists: true 138 | condition: selection 139 | falsepositives: 140 | - If a user requires an anonymising proxy due to valid justifications. 141 | level: high 142 | --- 143 | title: Okta User Activity Across Multiple Countries 144 | id: a8c75573-8513-40c6-85a6-818b7c58a601 145 | author: kelnage 146 | date: 2024-08-01 147 | status: experimental 148 | correlation: 149 | type: value_count 150 | rules: 151 | - 79bbc335-7ab0-4316-a17b-30c85f7f0595 152 | group-by: 153 | - actor.alternateid 154 | timespan: 1h 155 | condition: 156 | field: client.geographicalcontext.country 157 | gt: 1 158 | level: high 159 | """ 160 | ) 161 | loki_backend = LogQLBackend(processing_pipeline=pipeline) 162 | queries = loki_backend.convert(rules) 163 | assert queries == [ 164 | "count without (event_client_geographicalContext_country) " 165 | "(sum by (event_actor_alternateId, " 166 | "event_client_geographicalContext_country) " 167 | '(count_over_time({job=~".+"} | json | event_actor_alternateId!="" and ' 168 | 'event_client_geographicalContext_country!="" [1h]))) ' 169 | "> 1" 170 | ] 171 | -------------------------------------------------------------------------------- /tests/test_backend_loki_field_modifiers.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Tuple 2 | import pytest 3 | from sigma.backends.loki import LogQLBackend 4 | from sigma.collection import SigmaCollection 5 | from sigma.exceptions import SigmaFeatureNotSupportedByBackendError, SigmaTypeError 6 | from sigma.modifiers import modifier_mapping 7 | from sigma.processing.pipeline import ProcessingPipeline 8 | 9 | 10 | @pytest.fixture 11 | def loki_backend() -> LogQLBackend: 12 | # Add a processing pipeline to the backend to test the expand modifier. 13 | pipeline = ProcessingPipeline.from_yaml( 14 | """ 15 | name: Test Pipeline 16 | priority: 20 17 | vars: 18 | test: valueA 19 | transformations: 20 | - type: value_placeholders 21 | """ 22 | ) 23 | return LogQLBackend(processing_pipeline=pipeline) 24 | 25 | 26 | # Mapping from modifier identifier strings to modifier classes 27 | modifier_sample_data: Dict[str, Tuple[Any, str]] = { 28 | # "modifier": (value, expected_output) 29 | "contains": ("valueA", "fieldA=~`(?i).*valueA.*`"), 30 | "startswith": ("valueA", "fieldA=~`(?i)^valueA.*`"), 31 | "endswith": ("valueA", "fieldA=~`(?i).*valueA$`"), 32 | "exists": ("yes", 'fieldA!=""'), 33 | "base64": ("valueA", "fieldA=~`(?i)^dmFsdWVB$`"), 34 | "base64offset": ( 35 | "valueA", 36 | "fieldA=~`(?i)^dmFsdWVB$` or fieldA=~`(?i)^ZhbHVlQ$` or fieldA=~`(?i)^2YWx1ZU$`", 37 | ), 38 | "wide": ("valueA", "fieldA=~`(?i)^v\x00a\x00l\x00u\x00e\x00A\x00$`"), 39 | "windash": ("-foo", "fieldA=~`(?i)^\\-foo$` or fieldA=~`(?i)^/foo$`"), 40 | "re": (".*valueA$", "fieldA=~`.*valueA$`"), 41 | "i": ("valueA", "fieldA=~`(?i)valueA`"), 42 | "ignorecase": ("valueA", "fieldA=~`(?i)valueA`"), 43 | # Multiline modifier is not supported by LogQL, and the recommended way to handle 44 | # multiline logs can be found below: 45 | # https://grafana.com/docs/loki/latest/send-data/promtail/stages/multiline/ 46 | "m": (["valueA", "valueB"], "---"), 47 | "multiline": (["valueA", "valueB"], "---"), 48 | "s": ("valueA", "---"), 49 | "dotall": ("valueA", "---"), 50 | "cased": ("valueA", "fieldA=`valueA`"), 51 | "cidr": ("192.0.0.0/8", 'fieldA=ip("192.0.0.0/8")'), 52 | "all": (["valueA", "valueB"], "fieldA=~`(?i)^valueA$` and fieldA=~`(?i)^valueB$`"), 53 | "lt": (1, "fieldA<1"), 54 | "lte": (1, "fieldA<=1"), 55 | "gt": (1, "fieldA>1"), 56 | "gte": (1, "fieldA>=1"), 57 | "fieldref": ( 58 | "fieldB", 59 | "label_format match_0=`{{ if eq .fieldB .fieldA }}true{{ else }}false{{ end }}`," 60 | "match_1=`{{ if eq .fieldB .fieldA }}true{{ else }}false{{ end }}`" 61 | " | match_0=`true` and match_1!=`true`", 62 | ), 63 | "expand": ('"%test%"', "fieldA=~`(?i)^valueA$`"), 64 | "minute": ( 65 | 1, 66 | 'label_format date_0=`{{ date "04" (unixToTime .fieldA) }}`,' 67 | 'date_1=`{{ date "04" (unixToTime .fieldA) }}`' 68 | " | date_0=`1` and date_1!=`1`", 69 | ), 70 | "hour": ( 71 | 1, 72 | 'label_format date_0=`{{ date "15" (unixToTime .fieldA) }}`,' 73 | 'date_1=`{{ date "15" (unixToTime .fieldA) }}`' 74 | " | date_0=`1` and date_1!=`1`", 75 | ), 76 | "day": ( 77 | 1, 78 | 'label_format date_0=`{{ date "02" (unixToTime .fieldA) }}`,' 79 | 'date_1=`{{ date "02" (unixToTime .fieldA) }}`' 80 | " | date_0=`1` and date_1!=`1`", 81 | ), 82 | "week": (1, "---"), # Unsupported by the datetime layout 83 | "month": ( 84 | 1, 85 | 'label_format date_0=`{{ date "01" (unixToTime .fieldA) }}`,' 86 | 'date_1=`{{ date "01" (unixToTime .fieldA) }}`' 87 | " | date_0=`1` and date_1!=`1`", 88 | ), 89 | "year": ( 90 | 1, 91 | 'label_format date_0=`{{ date "2006" (unixToTime .fieldA) }}`,' 92 | 'date_1=`{{ date "2006" (unixToTime .fieldA) }}`' 93 | " | date_0=`1` and date_1!=`1`", 94 | ), 95 | } 96 | 97 | 98 | def generate_rule_with_field_modifier(modifier: str, value: Tuple[Any, str]) -> Tuple[str, str]: 99 | """Generate a Sigma rule with a field modifier.""" 100 | rule = """ 101 | title: Test 102 | status: test 103 | logsource: 104 | category: test_category 105 | product: test_product 106 | detection: 107 | selection: 108 | fieldA|{modifier}: {value[0]} 109 | neg: 110 | fieldA|{modifier}: {value[0]} 111 | condition: selection or not neg 112 | """ 113 | # Ignorecase modifier is a special case of the regex (re) modifier. 114 | if modifier in ["i", "ignorecase"]: 115 | modifier = "re|" + modifier 116 | return (rule.format(modifier=modifier, value=value), value[1]) 117 | 118 | 119 | def test_modifiers(loki_backend: LogQLBackend): 120 | # Check if all modifiers are tested. 121 | # This is to ensure that the test suite is updated when new modifiers are added. 122 | if len(modifier_sample_data) != len(modifier_mapping): 123 | diff = set(sorted(modifier_sample_data.keys())).symmetric_difference( 124 | set(sorted(modifier_mapping.keys())) 125 | ) 126 | pytest.fail( 127 | "Not all modifiers are tested, please update the sample data: modifier_sample_data.\n" 128 | f"Missing modifiers: {diff}" 129 | ) 130 | 131 | 132 | @pytest.mark.parametrize("label", modifier_mapping.keys()) 133 | def test_loki_field_modifiers(loki_backend: LogQLBackend, label: str): 134 | input_rule, output_expr = generate_rule_with_field_modifier(label, modifier_sample_data[label]) 135 | try: 136 | query = loki_backend.convert(SigmaCollection.from_yaml(input_rule)) 137 | assert output_expr in query[0] 138 | except (SigmaFeatureNotSupportedByBackendError, SigmaTypeError): 139 | pytest.skip(f"Backend does not support {modifier_mapping[label].__name__} modifier") 140 | except Exception as e: 141 | pytest.fail(f"Unexpected exception: {e}") 142 | -------------------------------------------------------------------------------- /tests/test_backend_loki_fieldref.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sigma.backends.loki import LogQLBackend 3 | from sigma.collection import SigmaCollection 4 | from sigma.processing.pipeline import ProcessingPipeline 5 | 6 | 7 | @pytest.fixture 8 | def loki_backend(): 9 | return LogQLBackend(add_line_filters=True) 10 | 11 | 12 | # Testing line filters introduction 13 | def test_loki_field_ref_single(loki_backend: LogQLBackend): 14 | assert loki_backend.convert( 15 | SigmaCollection.from_yaml( 16 | """ 17 | title: Test 18 | status: test 19 | logsource: 20 | category: test_category 21 | product: test_product 22 | detection: 23 | sel: 24 | field|fieldref: fieldA 25 | condition: sel 26 | """ 27 | ) 28 | ) == [ 29 | '{job=~".+"} | logfmt | label_format match_0=`{{ if eq .fieldA .field }}true{{ else }}false{{ end }}` | match_0=`true`' 30 | ] 31 | 32 | 33 | def test_loki_field_ref_multi(loki_backend: LogQLBackend): 34 | assert loki_backend.convert( 35 | SigmaCollection.from_yaml( 36 | """ 37 | title: Test 38 | status: test 39 | logsource: 40 | category: test_category 41 | product: test_product 42 | detection: 43 | sel: 44 | field1|fieldref: fieldA 45 | field2|fieldref: fieldB 46 | condition: sel 47 | """ 48 | ) 49 | ) == [ 50 | '{job=~".+"} | logfmt | label_format match_0=`{{ if eq .fieldA .field1 }}true{{ else }}false{{ end }}`,match_1=`{{ if eq .fieldB .field2 }}true{{ else }}false{{ end }}` | match_0=`true` and match_1=`true`' 51 | ] 52 | 53 | 54 | def test_loki_field_ref_json(loki_backend: LogQLBackend): 55 | assert loki_backend.convert( 56 | SigmaCollection.from_yaml( 57 | """ 58 | title: Test 59 | status: test 60 | logsource: 61 | category: test_category 62 | product: windows 63 | detection: 64 | sel: 65 | field|fieldref: fieldA 66 | condition: sel 67 | """ 68 | ) 69 | ) == [ 70 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .fieldA .field }}true{{ else }}false{{ end }}` | match_0=`true`' 71 | ] 72 | 73 | 74 | def test_loki_field_ref_json_multi_selection(loki_backend: LogQLBackend): 75 | assert loki_backend.convert( 76 | SigmaCollection.from_yaml( 77 | """ 78 | title: Test 79 | status: test 80 | logsource: 81 | category: test_category 82 | product: windows 83 | detection: 84 | sel: 85 | field1|fieldref: fieldA 86 | field2: Something 87 | condition: sel 88 | """ 89 | ) 90 | ) == [ 91 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | field2=~`(?i)^Something$`' 92 | "| label_format match_0=`{{ if eq .fieldA .field1 }}true{{ else }}false{{ end }}` " 93 | "| match_0=`true`" 94 | ] 95 | 96 | 97 | def test_loki_field_ref_negated(loki_backend: LogQLBackend): 98 | assert loki_backend.convert( 99 | SigmaCollection.from_yaml( 100 | """ 101 | title: Test 102 | status: test 103 | logsource: 104 | category: test_category 105 | product: windows 106 | detection: 107 | sel: 108 | field|fieldref: fieldA 109 | sel2: 110 | field2|fieldref: fieldB 111 | condition: sel and not sel2 112 | """ 113 | ) 114 | ) == [ 115 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .fieldA .field }}true{{ else }}false{{ end }}`,match_1=`{{ if eq .fieldB .field2 }}true{{ else }}false{{ end }}` | match_0=`true` and match_1!=`true`' 116 | ] 117 | 118 | 119 | def test_loki_field_ref_with_pipeline(loki_backend: LogQLBackend): 120 | pipeline = ProcessingPipeline.from_yaml( 121 | """ 122 | name: Test Pipeline 123 | priority: 20 124 | transformations: 125 | - id: field_prefix 126 | type: field_name_prefix 127 | prefix: "event_" 128 | """ 129 | ) 130 | loki_backend.processing_pipeline = pipeline 131 | 132 | assert loki_backend.convert( 133 | SigmaCollection.from_yaml( 134 | """ 135 | title: Test 136 | status: test 137 | logsource: 138 | category: test_category 139 | product: windows 140 | detection: 141 | sel: 142 | field|fieldref: fieldA 143 | condition: sel 144 | """ 145 | ) 146 | ) == [ 147 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | label_format match_0=`{{ if eq .event_fieldA .event_field }}true{{ else }}false{{ end }}` | match_0=`true`' 148 | ] 149 | 150 | 151 | def test_loki_field_ref_substring_matching(loki_backend: LogQLBackend): 152 | assert loki_backend.convert( 153 | SigmaCollection.from_yaml( 154 | """ 155 | title: Test 156 | status: test 157 | logsource: 158 | category: test_category 159 | product: windows 160 | detection: 161 | sel: 162 | field1|fieldref|contains: fieldA 163 | sel2: 164 | field2|fieldref|endswith: fieldB 165 | sel3: 166 | field3|fieldref|startswith: fieldC 167 | condition: sel and not sel2 or sel3 168 | """ 169 | ) 170 | ) == [ 171 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | ' 172 | + "label_format match_0=`{{ if contains .fieldA .field1 }}true{{ else }}false{{ end }}`," 173 | + "match_1=`{{ if hasSuffix .fieldB .field2 }}true{{ else }}false{{ end }}`," 174 | + "match_2=`{{ if hasPrefix .fieldC .field3 }}true{{ else }}false{{ end }}` " 175 | + "| match_0=`true` and match_1!=`true` and match_2=`true`" 176 | ] 177 | -------------------------------------------------------------------------------- /tests/test_backend_loki_event_count_correlation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sigma.processing.pipeline import ProcessingPipeline, ProcessingItem 3 | from sigma.processing.transformations import FieldMappingTransformation 4 | 5 | from sigma.backends.loki import LogQLBackend 6 | from sigma.collection import SigmaCollection 7 | 8 | 9 | @pytest.fixture 10 | def loki_backend(): 11 | return LogQLBackend() 12 | 13 | 14 | def test_loki_default_event_count_no_field(loki_backend: LogQLBackend): 15 | rules = SigmaCollection.from_yaml( 16 | """ 17 | title: Test Rule 18 | name: test_rule 19 | status: test 20 | logsource: 21 | category: test_category 22 | product: test_product 23 | detection: 24 | sel: 25 | fieldA: valueA 26 | condition: sel 27 | --- 28 | title: Test Correlation 29 | status: test 30 | correlation: 31 | type: event_count 32 | rules: 33 | - test_rule 34 | timespan: 30s 35 | condition: 36 | eq: 42 37 | """ 38 | ) 39 | queries = loki_backend.convert(rules) 40 | assert queries == [ 41 | 'sum(count_over_time({job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` [30s])) == 42' 42 | ] 43 | 44 | 45 | def test_loki_default_event_count_single_field(loki_backend: LogQLBackend): 46 | rules = SigmaCollection.from_yaml( 47 | """ 48 | title: Test Rule 49 | name: test_rule 50 | status: test 51 | logsource: 52 | category: test_category 53 | product: test_product 54 | detection: 55 | sel: 56 | fieldA: valueA 57 | condition: sel 58 | --- 59 | title: Test Correlation 60 | status: test 61 | correlation: 62 | type: event_count 63 | rules: 64 | - test_rule 65 | group-by: 66 | - fieldB 67 | timespan: 5m 68 | condition: 69 | gte: 1 70 | """ 71 | ) 72 | queries = loki_backend.convert(rules) 73 | assert queries == [ 74 | 'sum by (fieldB) (count_over_time({job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` [5m])) >= 1' 75 | ] 76 | 77 | 78 | def test_loki_default_event_count_multiple_fields(loki_backend: LogQLBackend): 79 | rules = SigmaCollection.from_yaml( 80 | """ 81 | title: Test Rule 82 | name: test_rule 83 | status: test 84 | logsource: 85 | category: test_category 86 | product: test_product 87 | detection: 88 | sel: 89 | fieldA: valueA 90 | condition: sel 91 | --- 92 | title: Test Correlation 93 | status: test 94 | correlation: 95 | type: event_count 96 | rules: 97 | - test_rule 98 | group-by: 99 | - fieldB 100 | - fieldC 101 | timespan: 1d 102 | condition: 103 | lt: 100 104 | """ 105 | ) 106 | queries = loki_backend.convert(rules) 107 | assert queries == [ 108 | 'sum by (fieldB, fieldC) (count_over_time({job=~".+"} | logfmt | ' 109 | "fieldA=~`(?i)^valueA$` [1d])) < 100" 110 | ] 111 | 112 | 113 | def test_loki_default_event_count_field_mapping(loki_backend: LogQLBackend): 114 | pipeline = ProcessingPipeline( 115 | name="Test mapping fields in correlations", 116 | priority=20, 117 | items=[ 118 | ProcessingItem( 119 | identifier="update_field_B_to_C", 120 | transformation=FieldMappingTransformation( 121 | mapping={ 122 | "fieldB": "fieldC", 123 | } 124 | ), 125 | ), 126 | ], 127 | ) 128 | rules = SigmaCollection.from_yaml( 129 | """ 130 | title: Test Rule 131 | name: test_rule 132 | status: test 133 | logsource: 134 | category: test_category 135 | product: test_product 136 | detection: 137 | sel: 138 | fieldA: valueA 139 | fieldB|contains: valueB 140 | condition: sel 141 | --- 142 | title: Test Correlation 143 | status: test 144 | correlation: 145 | type: event_count 146 | rules: 147 | - test_rule 148 | group-by: 149 | - fieldB 150 | timespan: 36h 151 | condition: 152 | lte: 5000 153 | """ 154 | ) 155 | loki_backend = LogQLBackend(processing_pipeline=pipeline) 156 | queries = loki_backend.convert(rules) 157 | assert queries == [ 158 | 'sum by (fieldC) (count_over_time({job=~".+"} | logfmt | ' 159 | "fieldA=~`(?i)^valueA$` and fieldC=~`(?i).*valueB.*` [36h])) <= 5000" 160 | ] 161 | 162 | 163 | def test_loki_default_event_count_log_source(loki_backend: LogQLBackend): 164 | rules = SigmaCollection.from_yaml( 165 | """ 166 | title: Test Rule 167 | name: test_rule 168 | status: test 169 | logsource: 170 | category: network_connection 171 | product: windows 172 | detection: 173 | sel: 174 | fieldA: valueA 175 | condition: sel 176 | --- 177 | title: Test Correlation 178 | status: test 179 | correlation: 180 | type: event_count 181 | rules: 182 | - test_rule 183 | timespan: 1d 184 | condition: 185 | gte: 100 186 | """ 187 | ) 188 | queries = loki_backend.convert(rules) 189 | assert queries == [ 190 | 'sum(count_over_time({job=~"eventlog|winlog|windows|fluentbit.*"} | json | ' 191 | "fieldA=~`(?i)^valueA$` [1d])) >= 100" 192 | ] 193 | 194 | 195 | def test_loki_default_event_count_absent_over_time_eq_0(loki_backend: LogQLBackend): 196 | rules = SigmaCollection.from_yaml( 197 | """ 198 | title: Test Rule 199 | name: test_rule 200 | status: test 201 | logsource: 202 | category: test_category 203 | product: test_product 204 | detection: 205 | sel: 206 | fieldA: valueA 207 | condition: sel 208 | --- 209 | title: Test Correlation 210 | status: test 211 | correlation: 212 | type: event_count 213 | rules: 214 | - test_rule 215 | timespan: 30s 216 | condition: 217 | eq: 0 218 | """ 219 | ) 220 | queries = loki_backend.convert(rules) 221 | assert queries == [ 222 | 'sum(absent_over_time({job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` [30s])) == 1' 223 | ] 224 | 225 | 226 | def test_loki_default_event_count_absent_over_time_lt_1(loki_backend: LogQLBackend): 227 | rules = SigmaCollection.from_yaml( 228 | """ 229 | title: Test Rule 230 | name: test_rule 231 | status: test 232 | logsource: 233 | category: test_category 234 | product: test_product 235 | detection: 236 | sel: 237 | fieldA: valueA 238 | condition: sel 239 | --- 240 | title: Test Correlation 241 | status: test 242 | correlation: 243 | type: event_count 244 | rules: 245 | - test_rule 246 | group-by: 247 | - fieldB 248 | timespan: 5m 249 | condition: 250 | lt: 1 251 | """ 252 | ) 253 | queries = loki_backend.convert(rules) 254 | assert queries == [ 255 | 'sum by (fieldB) (absent_over_time({job=~".+"} | logfmt | ' 256 | "fieldA=~`(?i)^valueA$` [5m])) == 1" 257 | ] 258 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![PyPI](https://img.shields.io/pypi/v/pysigma-backend-loki) 2 | ![Tests](https://github.com/grafana/pySigma-backend-loki/actions/workflows/test.yml/badge.svg) 3 | [![Coverage Status](https://coveralls.io/repos/github/grafana/pySigma-backend-loki/badge.svg?branch=main&t=lvM1Ns)](https://coveralls.io/github/grafana/pySigma-backend-loki?branch=main) 4 | 5 | # pySigma Loki Backend 6 | 7 | This is the Loki backend for pySigma. It provides the package `sigma.backends.loki` with the `LogQLBackend` class. 8 | 9 | It supports the following output formats for Sigma rules: 10 | 11 | * `default`: plain Loki LogQL queries 12 | * `ruler`: creates Loki LogQL queries in the ruler (YAML) format for generating alerts 13 | * `grafana_alerting`: creates Grafana Alerting provisioning YAML format for deploying alerts directly to Grafana 14 | 15 | It also supports the following query formats for and categories of [Sigma Correlation rules](https://github.com/SigmaHQ/sigma-specification/blob/version_2/Sigma_meta_rules.md): 16 | * `default` format using [LogQL metric queries](https://grafana.com/docs/loki/latest/query/metric_queries/): 17 | * `event_count` 18 | * `value_count` 19 | 20 | It includes the following pipeline transformations in `sigma.pipelines.loki`: 21 | 22 | * `SetCustomAttributeTransformation`: adds a specified custom attribute to a rule, which can be used to introduce a [stream selector](https://grafana.com/docs/loki/latest/logql/log_queries/#log-stream-selector) or [parser expression](https://grafana.com/docs/loki/latest/logql/log_queries/#parser-expression) into the generated query 23 | * The `LokiCustomAttributes` enum contains the relevant custom attribute names used by the backend 24 | 25 | Further, it contains the processing pipelines in `sigma.pipelines.loki`: 26 | 27 | * `loki_log_parser`: converts field names to logfmt labels used by Grafana 28 | * `loki_promtail_sysmon`: parse and adjust field names for Windows sysmon data produced by promtail 29 | * Note: most rules lack the `sysmon` service tag, and hence this pipeline should be used in combination with the [generic sysmon pipeline](https://github.com/SigmaHQ/pySigma-pipeline-sysmon) 30 | * `loki_okta_system_log`: parse the Okta System Log event json, adjusting field-names appropriately 31 | 32 | When converting rules into queries, the backend has the following optional arguments: 33 | 34 | * `add_line_filters` (boolean, default: `False`): if `True`, attempts to infer and add new line filters to queries without line filters, to [improve Loki query performance](https://grafana.com/docs/loki/latest/logql/log_queries/#line-filter-expression) 35 | * `case_sensitive` (boolean, default: `False`): if `True`, defaults to generating case-sensitive query filters, instead of case-insensitive filters that [the Sigma specification expects](https://github.com/SigmaHQ/sigma-specification/blob/main/Sigma_specification.md#general), trading between Loki query performance and potentially missing data with unexpected casing 36 | * Note: if the generated query will be executed on Loki v2.8.2 or older, this argument **should** be set to `False`, as these versions of Loki may contain issues with case-insensitive filters, which cause such queries to fail to match desired data 37 | 38 | When using the `grafana_alerting` output format, the following additional options are available: 39 | 40 | * `grafana_datasource_uid` (string, default: `loki`): the UID of the Loki datasource in Grafana 41 | * `grafana_folder` (string, default: `sigma`): the folder name where alerts will be provisioned in Grafana 42 | * `grafana_org_id` (integer, default: `1`): the Grafana organization ID 43 | * `grafana_interval` (string, default: `1m`): the evaluation interval for the alert rules 44 | * `grafana_contact_point` (string, default: `default`): the contact point to use for notifications 45 | * `loki_group_by_field` (string, default: empty): optional field name to group alerts by (e.g., `hostname`). If not specified, no grouping is applied. 46 | 47 | Example usage: 48 | 49 | ```bash 50 | sigma convert -t loki -f grafana_alerting \ 51 | -O grafana_datasource_uid=my-loki \ 52 | -O grafana_folder=security-alerts \ 53 | -O grafana_contact_point=slack-security \ 54 | -o alerts.yml \ 55 | ./rules/ 56 | ``` 57 | 58 | This backend is currently maintained by: 59 | 60 | * [Nick Moore](https://github.com/kelnage) 61 | * [Mostafa Moradian](https://github.com/mostafa) 62 | 63 | ## Installation 64 | 65 | To get started developing/testing pySigma-backend-loki, these steps may help you get started: 66 | 67 | 1. [Install poetry](https://python-poetry.org/docs/#installation) 68 | 2. Clone this repository and open a terminal/shell in the top-level directory 69 | 3. Run `poetry install` to install the Python dependencies 70 | 4. Run `poetry shell` to activate the poetry environment 71 | 5. Check it all works by running `poetry run pytest` 72 | 6. (Optional) If you wish to validate the generated rules using sigma\_backend\_tester.py, install 73 | [LogCLI](https://grafana.com/docs/loki/latest/tools/logcli/) 74 | 7. (Optional, but recommended) To enable the Git hooks, run the following command from the root directory of the repository: 75 | ```sh 76 | git config --local core.hooksPath .githooks/ 77 | ``` 78 | 79 | ## Releasing 80 | 81 | To release new versions of pySigma-backend-loki, we use GitHub actions to update PyPI. When the main branch is in state that is ready to release, the process is as follows: 82 | 83 | 1. Determine the correct version number using the [Semantic Versioning](https://semver.org/) methodology. All version numbers should be in the format `\d+\.\d+\.\d+(-[0-9A-Za-z-]+)?` 84 | 2. Update [pyproject.toml](https://github.com/grafana/pySigma-backend-loki/blob/main/pyproject.toml) with the new version number 85 | 3. Commit and push the change to GitHub, validate that the GitHub actions tests pass, and merge the PR into main 86 | 4. Checkout main and create a signed tag for the release, named the version number prefixed with a v, e.g., `git tag --sign --message="Release vX.X.X" vX.X.X` 87 | 5. Push the tag to GitHub, e.g., `git push --tags`, and validate that the release to the test instance of PyPI is successful 88 | 6. Run `poetry build` to produce distributable versions in `dist/` 89 | 7. Create a release in GitHub against the appropriate tag. If the version number starts with `v0`, or ends with `-alpha/beta` etc., mark it as a pre-release, and attach the distributable files to the release 90 | 8. Validate that the release to PyPI GitHub action is successful 91 | 9. If this release supports a new minor or major version of `pySigma`, do a pull request on the [pySigma-plugin-directory](https://github.com/SigmaHQ/pySigma-plugin-directory) to reflect that 92 | 93 | ## Work in progress 94 | 95 | These features are currently either WIP or are planned to be implemented in the near future. 96 | 97 | * Various processing pipelines for other applications and log sources 98 | * Generating more accurate log stream selectors based on logsource 99 | * Translate field names in Sigma signatures into relevant labels for Loki using pipelines 100 | 101 | ## Won't implement (probably) 102 | 103 | These features are not easily supported by the backend, and hence are unlikely to be implemented. 104 | 105 | * More complex keyword/line filter searches than ANDs of ORs 106 | -------------------------------------------------------------------------------- /tests/sigma_backend_tester.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import collections 3 | import operator 4 | import os 5 | import subprocess 6 | from typing import Any, Dict 7 | 8 | from sigma.exceptions import SigmaError 9 | 10 | from sigma.backends.loki import LogQLBackend 11 | from sigma.collection import SigmaCollection 12 | from sigma.pipelines.loki import loki_grafana_logfmt 13 | from sigma.rule import SigmaDetection 14 | 15 | parser = argparse.ArgumentParser( 16 | description="A script to help test pySigma backends using Sigma signature files", 17 | epilog="For any issues or requests for support, see the GitHub page", 18 | ) 19 | parser.add_argument( 20 | "signature_path", 21 | help="A path to either a single Sigma signature YAML file or a directory containing " 22 | "one or more signatures (incl. sub-folders)", 23 | ) 24 | parser.add_argument( 25 | "-a", 26 | "--add-line-filters", 27 | action="store_true", 28 | help="Attempt to add a single line filter to queries that otherwise lack any, to help improve " 29 | "the overall performance of the query when being run on Loki.", 30 | ) 31 | parser.add_argument( 32 | "-c", 33 | "--counts", 34 | action="store_true", 35 | help="Produce counts of the numbers of signatures processed and counts of successes/fails, " 36 | "along with error counts", 37 | ) 38 | parser.add_argument( 39 | "-p", 40 | "--print", 41 | action="store_true", 42 | help="Print the query(s) that were generated for the backend, and validation stdout if " 43 | "applicable", 44 | ) 45 | parser.add_argument( 46 | "-s", 47 | "--summarize", 48 | action="store_true", 49 | help="Produce a summary of the processed signature's log source information", 50 | ) 51 | parser.add_argument( 52 | "-t", 53 | "--tests", 54 | type=str, 55 | help="A path to either a single test log file or a directory containing one or more test log " 56 | "files, used during the validation of the generated rule(s). If a path to a directory is " 57 | "provided, this script will look for .log files with the same directory structure of the " 58 | "signature_path (i.e., if there is a rules/bad.yml file within signature_path, the script " 59 | "will look for a rules/bad.log file within the path specified in tests)", 60 | ) 61 | parser.add_argument( 62 | "-u", 63 | "--unique", 64 | action="store_true", 65 | help="Print the unique types and messages of errors that occur during the process", 66 | ) 67 | parser.add_argument( 68 | "-v", 69 | "--validate", 70 | action="store_true", 71 | help="Validate the generated rule(s) through the backend engine, where one or more lines of " 72 | "stdout denotes a successful validation", 73 | ) 74 | 75 | 76 | args = parser.parse_args() 77 | 78 | rule_path = args.signature_path 79 | 80 | pipeline = loki_grafana_logfmt() 81 | 82 | backend = LogQLBackend( 83 | processing_pipeline=pipeline, 84 | add_line_filters=args.add_line_filters, 85 | ) 86 | 87 | counters: Dict[str, Any] = { 88 | "parse_error": 0, 89 | "convert_error": 0, 90 | "validate_error": 0, 91 | "total_sigs": 0, 92 | "total_queries": 0, 93 | "total_files": 0, 94 | "total_test_logs": 0, 95 | "convert_success": 0, 96 | "validate_success": 0, 97 | "fields": {}, 98 | "error_types": {}, 99 | "error_messages": {}, 100 | "validate_stdout": {}, 101 | "validate_stderr": {}, 102 | "categories": {}, 103 | "products": {}, 104 | "services": {}, 105 | } 106 | 107 | 108 | def validate_with_backend(query, test_file=subprocess.DEVNULL): 109 | if test_file is None: 110 | test_file = subprocess.DEVNULL 111 | result = subprocess.run( 112 | ["logcli", "--stdin", "query", query], stdin=test_file, capture_output=True 113 | ) 114 | stdout = result.stdout.decode() 115 | stderr = result.stderr.decode() 116 | valid = False 117 | # If we have an input file, the query is valid if logcli produces one or more lines 118 | # of output. Otherwise, check logcli's return code to ensure the query is 119 | # syntactically valid 120 | if test_file is not subprocess.DEVNULL: 121 | valid = stdout.count(os.linesep) > 0 122 | else: 123 | valid = result.returncode == 0 124 | return (result.returncode, stdout, stderr, valid) 125 | 126 | 127 | def find_all_detection_items(detection, acc): 128 | for value in detection.detection_items: 129 | if isinstance(value, SigmaDetection): 130 | return find_all_detection_items(value, acc) 131 | else: 132 | return acc + [value] 133 | 134 | 135 | def process_file(file_path, test_file, args, counters): 136 | with open(file_path) as rule_file: 137 | sigma_rules = None 138 | yaml = rule_file.read() 139 | counters["total_files"] += 1 140 | try: 141 | sigma_rules = SigmaCollection.from_yaml(yaml) 142 | counters["total_sigs"] += len(sigma_rules) 143 | if args.summarize: 144 | for rule in sigma_rules: 145 | fields = ( 146 | list( 147 | item.field 148 | for detection in rule.detection.detections.values() 149 | for item in find_all_detection_items(detection, []) 150 | if item.field is not None 151 | ) 152 | + rule.fields 153 | ) 154 | for field in fields: 155 | counters["fields"][field] = counters["fields"].get(field, 0) + 1 156 | cat = rule.logsource.category 157 | prod = rule.logsource.product 158 | serv = rule.logsource.service 159 | if prod: 160 | counters["products"][prod] = counters["products"].get(prod, 0) + 1 161 | if serv: 162 | counters["services"][serv] = counters["services"].get(serv, 0) + 1 163 | if cat: 164 | counters["categories"][cat] = counters["categories"].get(cat, 0) + 1 165 | except SigmaError as err: 166 | counters["parse_error"] += 1 167 | if args.unique: 168 | error_type = type(err).__name__ 169 | counters["error_types"][error_type] = counters["error_types"].get(error_type, 0) + 1 170 | counters["error_messages"][str(err).strip()] = ( 171 | counters["error_messages"].get(str(err).strip(), 0) + 1 172 | ) 173 | return 174 | try: 175 | loki_rules = backend.convert(sigma_rules) 176 | counters["convert_success"] += len(sigma_rules) 177 | counters["total_queries"] += len(loki_rules) 178 | if args.validate or args.print: 179 | for loki_query in loki_rules: 180 | if args.print: 181 | print(loki_query) 182 | if args.validate: 183 | (returncode, stdout, stderr, valid) = validate_with_backend( 184 | loki_query, test_file 185 | ) 186 | if returncode != 0: 187 | counters["validate_error"] += 1 188 | elif valid: 189 | counters["validate_success"] += 1 190 | if args.print and len(stdout) > 0: 191 | print(stdout.strip()) 192 | if args.unique and len(stderr) > 0: 193 | counters["validate_stderr"][stderr.strip()] = ( 194 | counters["validate_stderr"].get(stderr.strip(), 0) + 1 195 | ) 196 | except SigmaError as err: 197 | counters["convert_error"] += 1 198 | if args.unique: 199 | error_type = type(err).__name__ 200 | counters["error_types"][error_type] = counters["error_types"].get(error_type, 0) + 1 201 | counters["error_messages"][str(err)] = ( 202 | counters["error_messages"].get(str(err), 0) + 1 203 | ) 204 | 205 | 206 | def print_counts(dct): 207 | if len(dct) == 0: 208 | print("\tNo results") 209 | return 210 | for k, v in collections.OrderedDict( 211 | sorted(dct.items(), key=operator.itemgetter(1), reverse=True) 212 | ).items(): 213 | print(f"\t{k}: {v}") 214 | 215 | 216 | def get_log_file(rule_file_path): 217 | (root, _) = os.path.splitext(os.path.basename(rule_file_path)) 218 | return root + ".log" 219 | 220 | 221 | test_file = None 222 | test_dir = None 223 | if args.tests: 224 | if os.path.isfile(args.tests): 225 | test_file = open(args.tests) 226 | counters["total_test_logs"] += 1 227 | elif os.path.isdir(args.tests): 228 | test_dir = args.tests 229 | else: 230 | print(f"Could not find test file/directory: {args.tests}") 231 | exit(1) 232 | 233 | if os.path.isfile(rule_path): 234 | if test_dir: 235 | test_file_path = os.path.join(test_dir, get_log_file(rule_path)) 236 | if os.path.isfile(test_file_path): 237 | test_file = open(test_file_path) 238 | counters["total_test_logs"] += 1 239 | process_file(rule_path, test_file, args, counters) 240 | elif os.path.isdir(rule_path): 241 | for dirpath, dirnames, filenames in os.walk(rule_path): 242 | for filename in filenames: 243 | rule_file_path = os.path.join(dirpath, filename) 244 | if test_dir: 245 | if test_file: 246 | test_file.close() 247 | test_file_path = os.path.join( 248 | test_dir, 249 | os.path.relpath(dirpath, start=rule_path), 250 | get_log_file(filename), 251 | ) 252 | if os.path.isfile(test_file_path): 253 | test_file = open(test_file_path) 254 | counters["total_test_logs"] += 1 255 | else: 256 | test_file = None 257 | elif test_file: 258 | test_file.seek(0) # reset the stream position each time 259 | process_file(rule_file_path, test_file, args, counters) 260 | else: 261 | print(f"Could not find rule file/directory: {rule_path}") 262 | exit(1) 263 | 264 | if args.counts: 265 | percent_conv = counters["convert_success"] / counters["total_sigs"] * 100 266 | print( 267 | f"Successfully converted {counters['convert_success']} out of " 268 | f"{counters['total_sigs']} ({percent_conv:.2f}%) signatures" 269 | ) 270 | if args.validate: 271 | percent_valid = counters["validate_success"] / counters["total_queries"] * 100 272 | print( 273 | f"Successfully validated {counters['validate_success']} out of " 274 | f"{counters['total_queries']} ({percent_valid:.2f}%) queries" 275 | ) 276 | print(f"YAML parse errors: {counters['parse_error']}") 277 | print(f"Conversion errors: {counters['convert_error']}") 278 | if args.validate: 279 | print(f"Validation errors: {counters['validate_error']}") 280 | if args.tests: 281 | print(f"Test log files used: {counters['total_test_logs']}") 282 | 283 | if args.unique: 284 | print("Error counts:") 285 | print_counts(counters["error_types"]) 286 | print("Error messages:") 287 | print_counts(counters["error_messages"]) 288 | if args.validate: 289 | print("Validation stderr:") 290 | print_counts(counters["validate_stderr"]) 291 | 292 | if args.summarize: 293 | print("Fields:") 294 | print_counts(counters["fields"]) 295 | print("Products:") 296 | print_counts(counters["products"]) 297 | print("Services:") 298 | print_counts(counters["services"]) 299 | print("Categories:") 300 | print_counts(counters["categories"]) 301 | 302 | if test_file: 303 | test_file.close() 304 | -------------------------------------------------------------------------------- /sigma/pipelines/loki/loki.py: -------------------------------------------------------------------------------- 1 | import string 2 | from dataclasses import dataclass 3 | from enum import Enum 4 | from typing import Any, Dict, List, Union, Type 5 | 6 | from sigma.conditions import ( 7 | ConditionValueExpression, 8 | ConditionNOT, 9 | ConditionFieldEqualsValueExpression, 10 | ConditionOR, 11 | ConditionItem, 12 | ConditionType, 13 | ConditionAND, 14 | ConditionIdentifier, 15 | ) 16 | from sigma.correlations import SigmaCorrelationRule 17 | from sigma.exceptions import ( 18 | SigmaFeatureNotSupportedByBackendError, 19 | ) 20 | from sigma.processing.conditions import LogsourceCondition 21 | from sigma.processing.pipeline import ProcessingItem, ProcessingPipeline 22 | from sigma.processing.transformations import ( 23 | transformations, 24 | AddFieldnamePrefixTransformation, 25 | FieldMappingTransformation, 26 | PreprocessingTransformation, 27 | ) 28 | from sigma.rule import SigmaRule, SigmaDetection 29 | from sigma.types import ( 30 | SigmaString, 31 | SigmaRegularExpression, 32 | SigmaFieldReference, 33 | SigmaType, 34 | ) 35 | 36 | from sigma.shared import ( 37 | sanitize_label_key, 38 | quote_string_value, 39 | join_or_values_re, 40 | escape_and_quote_re, 41 | convert_str_to_re, 42 | ) 43 | 44 | 45 | class LokiCustomAttributes(Enum): 46 | """The different custom attributes used by pipelines to store additional Loki-specific 47 | functionality.""" 48 | 49 | PARSER = "loki_parser" 50 | LOGSOURCE_SELECTION = "logsource_loki_selection" 51 | 52 | 53 | @dataclass 54 | class SetCustomAttributeTransformation(PreprocessingTransformation): 55 | """Sets an arbitrary custom attribute on a rule, that will be used during processing.""" 56 | 57 | attribute: str 58 | value: Any 59 | 60 | def apply(self, rule: Union[SigmaRule, SigmaCorrelationRule]) -> None: 61 | super().apply(rule) 62 | rule.custom_attributes[self.attribute] = self.value 63 | 64 | 65 | def traverse_conditions(item: ConditionType): 66 | queue: List[ 67 | Union[ 68 | ConditionIdentifier, 69 | ConditionItem, 70 | ConditionFieldEqualsValueExpression, 71 | ConditionValueExpression, 72 | None, 73 | ] 74 | ] = [item] 75 | while len(queue) > 0: 76 | cond = queue.pop(0) 77 | if isinstance(cond, ConditionItem): 78 | queue.extend(cond.args) 79 | else: 80 | yield cond 81 | 82 | 83 | def count_negated(classes: List[Type[Any]]) -> int: 84 | return len([neg for neg in classes if neg == ConditionNOT]) 85 | 86 | 87 | @dataclass 88 | class CustomLogSourceTransformation(PreprocessingTransformation): 89 | """Allow the definition of a log source selector using YAML structured data, including 90 | referencing log source and/or detection fields from the rule""" 91 | 92 | selection: Dict[str, Union[str, List[str]]] 93 | case_insensitive: bool = False 94 | template: bool = False 95 | 96 | def apply(self, rule: Union[SigmaRule, SigmaCorrelationRule]): 97 | if isinstance(rule, SigmaRule): 98 | selectors: List[str] = [] 99 | logsource_detections = SigmaDetection.from_definition(self.selection) 100 | conds = logsource_detections.postprocess(rule.detection) 101 | fields_set = set() 102 | args: List[ 103 | Union[ 104 | ConditionIdentifier, 105 | ConditionItem, 106 | ConditionFieldEqualsValueExpression, 107 | ConditionValueExpression, 108 | None, 109 | ] 110 | ] = [] 111 | if isinstance(conds, (ConditionOR, ConditionFieldEqualsValueExpression)): 112 | args.append(conds) 113 | elif isinstance(conds, ConditionAND): 114 | args.extend(conds.args) 115 | else: 116 | raise SigmaFeatureNotSupportedByBackendError( 117 | "the custom log source selector only supports field equals value conditions" 118 | ) 119 | for cond in args: 120 | field: Union[str, None] = None 121 | op: Union[str, None] = None 122 | value: Union[SigmaType, str, None] = None 123 | if isinstance(cond, ConditionValueExpression): 124 | raise SigmaFeatureNotSupportedByBackendError( 125 | "the custom log source selector only supports field equals value conditions" 126 | ) 127 | elif isinstance(cond, ConditionOR): 128 | op = "=~" 129 | values = [] 130 | for arg in cond.args: 131 | if not isinstance(arg, ConditionFieldEqualsValueExpression): 132 | raise SigmaFeatureNotSupportedByBackendError( 133 | "the custom log source selector only supports a single nesting of " 134 | "OR'd values" 135 | ) 136 | if field is None: 137 | field = arg.field 138 | elif field != arg.field: 139 | raise SigmaFeatureNotSupportedByBackendError( 140 | "the custom log source selector only supports ORs on a single field" 141 | ) 142 | if isinstance(arg.value, (SigmaString, SigmaRegularExpression)): 143 | values.append(arg.value) 144 | value = join_or_values_re(values, self.case_insensitive) 145 | elif isinstance(cond, ConditionFieldEqualsValueExpression): 146 | field = cond.field 147 | op = "=" 148 | value = cond.value 149 | if not isinstance( 150 | value, 151 | (SigmaFieldReference, SigmaString, SigmaRegularExpression), 152 | ): 153 | raise SigmaFeatureNotSupportedByBackendError( 154 | "the custom log selector pipeline only supports: string values, field " 155 | "references and regular expressions" 156 | ) 157 | if field in fields_set: 158 | raise SigmaFeatureNotSupportedByBackendError( 159 | "the custom log source selector only allows one required value for a field" 160 | ) 161 | 162 | skip = False 163 | rule_conditions = [] 164 | for parsed_conds in rule.detection.parsed_condition: 165 | rule_conditions.extend(traverse_conditions(parsed_conds.parsed)) # type: ignore 166 | 167 | # Note: the order of these if statements is important and should be preserved 168 | if isinstance(value, SigmaFieldReference): 169 | values = [] 170 | negated = None 171 | for item in rule_conditions: 172 | if ( 173 | isinstance(item, ConditionFieldEqualsValueExpression) 174 | and item.field == value.field 175 | and isinstance(item.value, (SigmaString, SigmaRegularExpression)) 176 | ): 177 | classes = item.parent_chain_condition_classes() 178 | new_negated = count_negated(classes) % 2 == 1 179 | if negated is not None and negated != new_negated: 180 | # Skipping fields refs with both negated and un-negated values 181 | skip = True 182 | else: 183 | negated = new_negated 184 | values.append(item.value) 185 | if len(values) == 0 or skip: 186 | continue 187 | if len(values) == 1 and isinstance(values[0], SigmaString): 188 | if negated: 189 | op = "!=" 190 | value = values[0] 191 | else: 192 | op = "=~" 193 | if negated: 194 | op = "!~" 195 | value = join_or_values_re(values, self.case_insensitive) 196 | if isinstance(value, SigmaString): 197 | if value.contains_special(): 198 | value = convert_str_to_re(value, self.case_insensitive, False) 199 | else: 200 | value = quote_string_value(value) 201 | # Not elif, as if the value was a string containing wildcards, it is now a RegEx 202 | if isinstance(value, SigmaRegularExpression): 203 | op = "=~" 204 | value = escape_and_quote_re(value) 205 | if field and op and value: 206 | fields_set.add(field) 207 | selectors.append(f"{sanitize_label_key(field)}{op}{value}") 208 | formatted_selectors = "{" + ",".join(selectors) + "}" 209 | if self.template: 210 | formatted_selectors = string.Template(formatted_selectors).safe_substitute( 211 | category=rule.logsource.category, 212 | product=rule.logsource.product, 213 | service=rule.logsource.service, 214 | ) 215 | rule.custom_attributes[LokiCustomAttributes.LOGSOURCE_SELECTION.value] = ( 216 | formatted_selectors 217 | ) 218 | super().apply(rule) 219 | else: 220 | for ruleref in rule.rules: 221 | self.apply(ruleref.rule) 222 | 223 | 224 | # Update pySigma transformations to include the above 225 | # mypy type: ignore required due to incorrect type annotation on the transformations dict 226 | transformations["set_custom_attribute"] = SetCustomAttributeTransformation # type: ignore 227 | transformations["set_custom_log_source"] = CustomLogSourceTransformation # type: ignore 228 | 229 | 230 | def loki_grafana_logfmt() -> ProcessingPipeline: 231 | return ProcessingPipeline( 232 | name="Loki Grafana logfmt field names", 233 | priority=20, 234 | allowed_backends=frozenset({"loki"}), 235 | items=[ 236 | ProcessingItem( 237 | identifier="loki_grafana_field_mapping", 238 | transformation=FieldMappingTransformation( 239 | { 240 | "ClientIP": "remote_addr", 241 | "Endpoint": "path", 242 | "User": "uname", 243 | "c-ip": "remote_addr", 244 | "c-uri": "path", 245 | "cs-uri-query": "path", 246 | "client_ip": "remote_addr", 247 | "cs-method": "method", 248 | "sc-status": "status", 249 | } 250 | ), 251 | ) 252 | ], 253 | ) 254 | 255 | 256 | def loki_promtail_sysmon() -> ProcessingPipeline: 257 | return ProcessingPipeline( 258 | name="Loki Promtail Windows Sysmon Message Parser", 259 | priority=20, 260 | allowed_backends=frozenset({"loki"}), 261 | items=[ 262 | ProcessingItem( 263 | identifier="loki_promtail_sysmon_field_mapping", 264 | # Using the fieldnames in loki/clients/pkg/promtail/targets/windows/format.go 265 | transformation=FieldMappingTransformation( 266 | { 267 | "Source": "source", 268 | "Channel": "channel", 269 | "Computer": "computer", 270 | "EventID": "event_id", 271 | "Version": "version", 272 | "Level": "level", 273 | "Task": "task", 274 | "Opcode": "opCode", 275 | "LevelText": "levelText", 276 | "TaskText": "taskText", 277 | "OpcodeText": "opCodeText", 278 | "Keywords": "keywords", 279 | "TimeCreated": "timeCreated", 280 | "EventRecordID": "eventRecordID", 281 | "Correlation": "correlation", 282 | "Execution": "execution", 283 | "Security": "security", 284 | "UserData": "user_data", 285 | "EventData": "event_data", 286 | "Message": "message", 287 | } 288 | ), 289 | ), 290 | ProcessingItem( 291 | identifier="loki_promtail_sysmon_parser", 292 | transformation=SetCustomAttributeTransformation( 293 | attribute=LokiCustomAttributes.PARSER.value, 294 | value='json | label_format Message=`{{ .message | replace "\\\\" "\\\\\\\\" | replace "\\"" "\\\\\\"" }}` ' # noqa: E501 295 | '| line_format `{{ regexReplaceAll "([^:]+): ?((?:[^\\\\r]*|$))(\\r\\n|$)" .Message "${1}=\\"${2}\\" "}}` ' # noqa: E501 296 | "| logfmt", 297 | ), 298 | rule_conditions=[ 299 | LogsourceCondition( 300 | product="windows", 301 | service="sysmon", 302 | ) 303 | ], 304 | ), 305 | ], 306 | ) 307 | 308 | 309 | def loki_okta_system_log() -> ProcessingPipeline: 310 | return ProcessingPipeline( 311 | name="Loki Okta System Log json", 312 | priority=20, 313 | allowed_backends=frozenset({"loki"}), 314 | items=[ 315 | ProcessingItem( 316 | identifier="loki_okta_event_json_formatter", 317 | transformation=SetCustomAttributeTransformation( 318 | attribute=LokiCustomAttributes.PARSER.value, 319 | value="json", 320 | ), 321 | rule_conditions=[ 322 | LogsourceCondition( 323 | product="okta", 324 | service="okta", 325 | ) 326 | ], 327 | ), 328 | ProcessingItem( 329 | identifier="loki_okta_field_name_mapping", 330 | # Transform event fields names that should be camelCase 331 | # See https://developer.okta.com/docs/reference/api/system-log/#logevent-object-annotated-example # noqa: E501 332 | transformation=FieldMappingTransformation( 333 | { 334 | v.lower().replace("_", "."): v 335 | for v in [ 336 | "eventType", 337 | "legacyEventType", 338 | "displayMessage", 339 | "actor_alternateId", 340 | "actor_displayName", 341 | "client_userAgent_rawUserAgent", 342 | "client_userAgent_os", 343 | "client_userAgent_browser", 344 | "client_geographicalContext_geolocation_lat", 345 | "client_geographicalContext_geolocation_lon", 346 | "client_geographicalContext_city", 347 | "client_geographicalContext_state", 348 | "client_geographicalContext_country", 349 | "client_geographicalContext_postalCode", 350 | "client_ipAddress", 351 | "debugContext_debugData_requestUri", 352 | "debugContext_debugData_originalPrincipal_id", 353 | "debugContext_debugData_originalPrincipal_type", 354 | "debugContext_debugData_originalPrincipal_alternateId", 355 | "debugContext_debugData_originalPrincipal_displayName", 356 | "debugContext_debugData_behaviors", 357 | "debugContext_debugData_logOnlySecurityData", 358 | "authenticationContext_authenticationProvider", 359 | "authenticationContext_authenticationStep", 360 | "authenticationContext_credentialProvider", 361 | "authenticationContext_credentialType", 362 | "authenticationContext_issuer_id", 363 | "authenticationContext_issuer_type", 364 | "authenticationContext_externalSessionId", 365 | "authenticationContext_interface", 366 | "securityContext_asNumber", 367 | "securityContext_asOrg", 368 | "securityContext_isp", 369 | "securityContext_domain", 370 | "securityContext_isProxy", 371 | "target_alternateId", 372 | "target_displayName", 373 | "target_detailEntry", 374 | ] 375 | } 376 | ), 377 | rule_conditions=[ 378 | LogsourceCondition( 379 | product="okta", 380 | service="okta", 381 | ) 382 | ], 383 | ), 384 | ProcessingItem( 385 | identifier="loki_okta_field_event_prefix", 386 | transformation=AddFieldnamePrefixTransformation("event_"), 387 | rule_conditions=[ 388 | LogsourceCondition( 389 | product="okta", 390 | service="okta", 391 | ) 392 | ], 393 | ), 394 | ], 395 | ) 396 | -------------------------------------------------------------------------------- /tests/test_backend_negation_loki.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sigma.backends.loki import LogQLBackend 3 | from sigma.collection import SigmaCollection 4 | from sigma.exceptions import SigmaFeatureNotSupportedByBackendError 5 | 6 | 7 | @pytest.fixture 8 | def loki_backend(): 9 | return LogQLBackend() 10 | 11 | 12 | # Simple field equality test 13 | def test_loki_field_not_eq(loki_backend: LogQLBackend): 14 | assert loki_backend.convert( 15 | SigmaCollection.from_yaml( 16 | """ 17 | title: Test 18 | status: test 19 | logsource: 20 | category: test_category 21 | product: test_product 22 | detection: 23 | sel: 24 | fieldA: valueA 25 | condition: not sel 26 | """ 27 | ) 28 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$`'] 29 | 30 | 31 | def test_loki_field_not_eq_num(loki_backend: LogQLBackend): 32 | assert loki_backend.convert( 33 | SigmaCollection.from_yaml( 34 | """ 35 | title: Test 36 | status: test 37 | logsource: 38 | category: test_category 39 | product: test_product 40 | detection: 41 | sel: 42 | fieldA: 100 43 | condition: not sel 44 | """ 45 | ) 46 | ) == ['{job=~".+"} | logfmt | fieldA!=100'] 47 | 48 | 49 | def test_loki_field_not_not_eq(loki_backend: LogQLBackend): 50 | assert loki_backend.convert( 51 | SigmaCollection.from_yaml( 52 | """ 53 | title: Test 54 | status: test 55 | logsource: 56 | category: test_category 57 | product: test_product 58 | detection: 59 | sel: 60 | fieldA: valueA 61 | condition: not (not sel) 62 | """ 63 | ) 64 | ) == ['{job=~".+"} | logfmt | fieldA=~`(?i)^valueA$`'] 65 | 66 | 67 | # Testing boolean logic 68 | def test_loki_not_and_expression(loki_backend: LogQLBackend): 69 | assert loki_backend.convert( 70 | SigmaCollection.from_yaml( 71 | """ 72 | title: Test 73 | status: test 74 | logsource: 75 | category: test_category 76 | product: test_product 77 | detection: 78 | sel: 79 | fieldA: valueA 80 | fieldB: valueB 81 | condition: not sel 82 | """ 83 | ) 84 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$` or fieldB!~`(?i)^valueB$`'] 85 | 86 | 87 | def test_loki_not_or_expression(loki_backend: LogQLBackend): 88 | assert loki_backend.convert( 89 | SigmaCollection.from_yaml( 90 | """ 91 | title: Test 92 | status: test 93 | logsource: 94 | category: test_category 95 | product: test_product 96 | detection: 97 | sel1: 98 | fieldA: valueA 99 | sel2: 100 | fieldB: valueB 101 | condition: not 1 of sel* 102 | """ 103 | ) 104 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$` and fieldB!~`(?i)^valueB$`'] 105 | 106 | 107 | def test_loki_not_and_or_expression(loki_backend: LogQLBackend): 108 | assert loki_backend.convert( 109 | SigmaCollection.from_yaml( 110 | """ 111 | title: Test 112 | status: test 113 | logsource: 114 | category: test_category 115 | product: test_product 116 | detection: 117 | sel: 118 | fieldA: 119 | - valueA1 120 | - valueA2 121 | fieldB: 122 | - valueB1 123 | - valueB2 124 | condition: not sel 125 | """ 126 | ) 127 | ) == [ 128 | '{job=~".+"} | logfmt | fieldA!~`(?i)^valueA1$` and fieldA!~`(?i)^valueA2$` ' 129 | "or fieldB!~`(?i)^valueB1$` and fieldB!~`(?i)^valueB2$`" 130 | ] 131 | 132 | 133 | def test_loki_not_or_and_expression(loki_backend: LogQLBackend): 134 | assert loki_backend.convert( 135 | SigmaCollection.from_yaml( 136 | """ 137 | title: Test 138 | status: test 139 | logsource: 140 | category: test_category 141 | product: test_product 142 | detection: 143 | sel1: 144 | fieldA: valueA1 145 | fieldB: valueB1 146 | sel2: 147 | fieldA: valueA2 148 | fieldB: valueB2 149 | condition: not 1 of sel* 150 | """ 151 | ) 152 | ) == [ 153 | '{job=~".+"} | logfmt | (fieldA!~`(?i)^valueA1$` or fieldB!~`(?i)^valueB1$`) and ' 154 | "(fieldA!~`(?i)^valueA2$` or fieldB!~`(?i)^valueB2$`)" 155 | ] 156 | 157 | 158 | # Loki doesn't support in expressions, so in this case, multiple or conditions should be produced 159 | def test_loki_not_in_expression(loki_backend: LogQLBackend): 160 | assert loki_backend.convert( 161 | SigmaCollection.from_yaml( 162 | """ 163 | title: Test 164 | status: test 165 | logsource: 166 | category: test_category 167 | product: test_product 168 | detection: 169 | sel: 170 | fieldA: 171 | - valueA 172 | - valueB 173 | - valueC 174 | condition: not sel 175 | """ 176 | ) 177 | ) == [ 178 | '{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$` and fieldA!~`(?i)^valueB$` and ' 179 | "fieldA!~`(?i)^valueC$`" 180 | ] 181 | 182 | 183 | def test_loki_not_all_query(loki_backend: LogQLBackend): 184 | assert loki_backend.convert( 185 | SigmaCollection.from_yaml( 186 | """ 187 | title: Test 188 | status: test 189 | logsource: 190 | category: test_category 191 | product: test_product 192 | detection: 193 | sel: 194 | fieldA|all: 195 | - valueA 196 | - valueB 197 | condition: not sel 198 | """ 199 | ) 200 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$` or fieldA!~`(?i)^valueB$`'] 201 | 202 | 203 | def test_loki_not_all_bracket_query(loki_backend: LogQLBackend): 204 | assert loki_backend.convert( 205 | SigmaCollection.from_yaml( 206 | """ 207 | title: Test 208 | status: test 209 | logsource: 210 | category: test_category 211 | product: test_product 212 | detection: 213 | sel: 214 | fieldA: 215 | - valueA 216 | - valueB 217 | filter: 218 | fieldB|all: 219 | - valueC 220 | - valueD 221 | condition: sel and not filter 222 | """ 223 | ) 224 | ) == [ 225 | '{job=~".+"} | logfmt | (fieldA=~`(?i)^valueA$` or fieldA=~`(?i)^valueB$`) and ' 226 | "(fieldB!~`(?i)^valueC$` or fieldB!~`(?i)^valueD$`)" 227 | ] 228 | 229 | 230 | def test_loki_not_base64_query(loki_backend: LogQLBackend): 231 | assert loki_backend.convert( 232 | SigmaCollection.from_yaml( 233 | """ 234 | title: Test 235 | status: test 236 | logsource: 237 | category: test_category 238 | product: test_product 239 | detection: 240 | sel: 241 | fieldA|base64: value 242 | condition: not sel 243 | """ 244 | ) 245 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^dmFsdWU=$`'] 246 | 247 | 248 | def test_loki_not_base64offset_query(loki_backend: LogQLBackend): 249 | assert loki_backend.convert( 250 | SigmaCollection.from_yaml( 251 | """ 252 | title: Test 253 | status: test 254 | logsource: 255 | category: test_category 256 | product: test_product 257 | detection: 258 | sel: 259 | fieldA|base64offset: value 260 | condition: not sel 261 | """ 262 | ) 263 | ) == [ 264 | '{job=~".+"} | logfmt | fieldA!~`(?i)^dmFsdW$` and fieldA!~`(?i)^ZhbHVl$` and ' 265 | "fieldA!~`(?i)^2YWx1Z$`" 266 | ] 267 | 268 | 269 | # Testing different search identifiers 270 | def test_loki_not_null(loki_backend: LogQLBackend): 271 | assert loki_backend.convert( 272 | SigmaCollection.from_yaml( 273 | """ 274 | title: Test 275 | status: test 276 | logsource: 277 | category: test_category 278 | product: test_product 279 | detection: 280 | sel: 281 | fieldA: null 282 | condition: not sel 283 | """ 284 | ) 285 | ) == ['{job=~".+"} | logfmt | fieldA!=``'] 286 | 287 | 288 | # Loki does not support wildcards, so we use case-insensitive regular expressions instead 289 | def test_loki_not_wildcard_single(loki_backend: LogQLBackend): 290 | assert loki_backend.convert( 291 | SigmaCollection.from_yaml( 292 | """ 293 | title: Test 294 | status: test 295 | logsource: 296 | category: test_category 297 | product: test_product 298 | detection: 299 | sel: 300 | fieldA: va?ue 301 | condition: not sel 302 | """ 303 | ) 304 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^va.ue$`'] 305 | 306 | 307 | def test_loki_not_wildcard_multi(loki_backend: LogQLBackend): 308 | assert loki_backend.convert( 309 | SigmaCollection.from_yaml( 310 | """ 311 | title: Test 312 | status: test 313 | logsource: 314 | category: test_category 315 | product: test_product 316 | detection: 317 | sel: 318 | fieldA: value* 319 | condition: not sel 320 | """ 321 | ) 322 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^value.*`'] 323 | 324 | 325 | def test_loki_not_wildcard_unbound(loki_backend: LogQLBackend): 326 | assert loki_backend.convert( 327 | SigmaCollection.from_yaml( 328 | """ 329 | title: test 330 | status: test 331 | logsource: 332 | category: test_category 333 | product: test_product 334 | detection: 335 | keywords: 336 | - va?ue* 337 | condition: not keywords 338 | """ 339 | ) 340 | ) == ['{job=~".+"} !~ `(?i)va.ue`'] 341 | 342 | 343 | def test_loki_not_regex_query(loki_backend: LogQLBackend): 344 | assert loki_backend.convert( 345 | SigmaCollection.from_yaml( 346 | """ 347 | title: Test 348 | status: test 349 | logsource: 350 | category: test_category 351 | product: test_product 352 | detection: 353 | sel: 354 | fieldA|re: foo.*bar 355 | fieldB: foo 356 | condition: not sel 357 | """ 358 | ) 359 | ) == ['{job=~".+"} | logfmt | fieldA!~`foo.*bar` or fieldB!~`(?i)^foo$`'] 360 | 361 | 362 | def test_loki_not_cidr_query(loki_backend: LogQLBackend): 363 | assert loki_backend.convert( 364 | SigmaCollection.from_yaml( 365 | """ 366 | title: Test 367 | status: test 368 | logsource: 369 | category: test_category 370 | product: test_product 371 | detection: 372 | sel: 373 | fieldA|cidr: 192.168.0.0/16 374 | condition: not sel 375 | """ 376 | ) 377 | ) == ['{job=~".+"} | logfmt | fieldA!=ip("192.168.0.0/16")'] 378 | 379 | 380 | def test_loki_not_unbound_case_sensitive(loki_backend: LogQLBackend): 381 | loki_backend.case_sensitive = True 382 | assert loki_backend.convert( 383 | SigmaCollection.from_yaml( 384 | """ 385 | title: test 386 | status: test 387 | logsource: 388 | category: test_category 389 | product: test_product 390 | detection: 391 | keywords: 392 | value 393 | condition: not keywords 394 | """ 395 | ) 396 | ) == ['{job=~".+"} != `value`'] 397 | 398 | 399 | def test_loki_not_unbound(loki_backend: LogQLBackend): 400 | assert loki_backend.convert( 401 | SigmaCollection.from_yaml( 402 | """ 403 | title: test 404 | status: test 405 | logsource: 406 | category: test_category 407 | product: test_product 408 | detection: 409 | keywords: 410 | value 411 | condition: not keywords 412 | """ 413 | ) 414 | ) == ['{job=~".+"} !~ `(?i)value`'] 415 | 416 | 417 | def test_loki_not_unbound_num(loki_backend: LogQLBackend): 418 | assert loki_backend.convert( 419 | SigmaCollection.from_yaml( 420 | """ 421 | title: test 422 | status: test 423 | logsource: 424 | category: test_category 425 | product: test_product 426 | detection: 427 | keywords: 428 | 100 429 | condition: not keywords 430 | """ 431 | ) 432 | ) == ['{job=~".+"} != 100'] 433 | 434 | 435 | def test_loki_not_unbound_re_wildcard(loki_backend: LogQLBackend): 436 | assert loki_backend.convert( 437 | SigmaCollection.from_yaml( 438 | """ 439 | title: test 440 | status: test 441 | logsource: 442 | category: test_category 443 | product: test_product 444 | detection: 445 | keywords: 446 | '|re': 'value.*' 447 | condition: not keywords 448 | """ 449 | ) 450 | ) == ['{job=~".+"} !~ `value`'] 451 | 452 | 453 | def test_loki_not_and_unbound(loki_backend: LogQLBackend): 454 | assert loki_backend.convert( 455 | SigmaCollection.from_yaml( 456 | """ 457 | title: test 458 | status: test 459 | logsource: 460 | category: test_category 461 | product: test_product 462 | detection: 463 | keyword1: 464 | valueA 465 | keyword2: 466 | valueB 467 | condition: not (keyword1 and keyword2) 468 | """ 469 | ) 470 | ) == ['{job=~".+"} !~ `(?i)valueA|valueB`'] 471 | 472 | 473 | def test_loki_not_unbound_or_field(loki_backend: LogQLBackend): 474 | assert loki_backend.convert( 475 | SigmaCollection.from_yaml( 476 | """ 477 | title: test 478 | status: test 479 | logsource: 480 | category: test_category 481 | product: test_product 482 | detection: 483 | keywords: 484 | valueA 485 | sel: 486 | fieldA: valueB 487 | condition: not (keywords or sel) 488 | """ 489 | ) 490 | ) == ['{job=~".+"} !~ `(?i)valueA` | logfmt | fieldA!~`(?i)^valueB$`'] 491 | 492 | 493 | def test_loki_not_unbound_and_field(loki_backend: LogQLBackend): 494 | with pytest.raises(SigmaFeatureNotSupportedByBackendError): 495 | loki_backend.convert( 496 | SigmaCollection.from_yaml( 497 | """ 498 | title: Test 499 | status: test 500 | logsource: 501 | category: test_category 502 | product: test_product 503 | detection: 504 | keywords: 505 | valueA 506 | sel: 507 | fieldA: valueB 508 | condition: not (keywords and sel) 509 | """ 510 | ) 511 | ) 512 | 513 | 514 | def test_loki_not_multi_or_unbound(loki_backend: LogQLBackend): 515 | with pytest.raises(SigmaFeatureNotSupportedByBackendError): 516 | loki_backend.convert( 517 | SigmaCollection.from_yaml( 518 | """ 519 | title: Test 520 | status: test 521 | logsource: 522 | category: test_category 523 | product: test_product 524 | detection: 525 | keywordA: 526 | - valueA 527 | - valueB 528 | keywordB: 529 | - valueC 530 | - valueD 531 | condition: not (keywordA and keywordB) 532 | """ 533 | ) 534 | ) 535 | 536 | 537 | def test_loki_not_or_unbound(loki_backend: LogQLBackend): 538 | assert loki_backend.convert( 539 | SigmaCollection.from_yaml( 540 | """ 541 | title: Test 542 | status: test 543 | logsource: 544 | category: test_category 545 | product: test_product 546 | detection: 547 | keywords: 548 | - valueA 549 | - valueB 550 | condition: not keywords 551 | """ 552 | ) 553 | ) == ['{job=~".+"} !~ `(?i)valueA` !~ `(?i)valueB`'] 554 | 555 | 556 | def test_loki_not_unbound_wildcard(loki_backend: LogQLBackend): 557 | assert loki_backend.convert( 558 | SigmaCollection.from_yaml( 559 | """ 560 | title: Test 561 | status: test 562 | logsource: 563 | category: test_category 564 | product: test_product 565 | detection: 566 | keywords: 567 | value* 568 | condition: not keywords 569 | """ 570 | ) 571 | ) == ['{job=~".+"} !~ `(?i)value`'] 572 | 573 | 574 | def test_loki_field_and_not_multi_unbound_expression(loki_backend: LogQLBackend): 575 | assert loki_backend.convert( 576 | SigmaCollection.from_yaml( 577 | """ 578 | title: Test 579 | status: test 580 | logsource: 581 | category: test_category 582 | product: test_product 583 | detection: 584 | sel: 585 | fieldA: valueA 586 | keywords: 587 | - valueB 588 | - valueC 589 | condition: sel and not keywords 590 | """ 591 | ) 592 | ) == ['{job=~".+"} !~ `(?i)valueB` !~ `(?i)valueC` | logfmt | fieldA=~`(?i)^valueA$`'] 593 | 594 | 595 | def test_loki_field_exists_negated(loki_backend: LogQLBackend): 596 | assert loki_backend.convert( 597 | SigmaCollection.from_yaml( 598 | """ 599 | title: Test 600 | status: test 601 | logsource: 602 | category: test_category 603 | product: test_product 604 | detection: 605 | sel: 606 | fieldA|exists: yes 607 | condition: not sel 608 | """ 609 | ) 610 | ) == ['{job=~".+"} | logfmt | fieldA=""'] 611 | 612 | 613 | def test_loki_field_not_exists_negated(loki_backend: LogQLBackend): 614 | assert loki_backend.convert( 615 | SigmaCollection.from_yaml( 616 | """ 617 | title: Test 618 | status: test 619 | logsource: 620 | category: test_category 621 | product: test_product 622 | detection: 623 | sel: 624 | fieldA|exists: no 625 | condition: not sel 626 | """ 627 | ) 628 | ) == ['{job=~".+"} | logfmt | fieldA!=""'] 629 | -------------------------------------------------------------------------------- /tests/test_backend_loki_add_line_filters_case_insensitive.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import random 3 | import re 4 | import string 5 | from sigma.backends.loki import LogQLBackend 6 | from sigma.collection import SigmaCollection 7 | 8 | # from sigma.exceptions import SigmaFeatureNotSupportedByBackendError 9 | 10 | 11 | @pytest.fixture 12 | def loki_backend(): 13 | return LogQLBackend(add_line_filters=True) 14 | 15 | 16 | # Testing line filters introduction 17 | def test_loki_lf_field_eq(loki_backend: LogQLBackend): 18 | assert loki_backend.convert( 19 | SigmaCollection.from_yaml( 20 | """ 21 | title: Test 22 | status: test 23 | logsource: 24 | category: test_category 25 | product: test_product 26 | detection: 27 | sel: 28 | fieldA: valueA 29 | condition: sel 30 | """ 31 | ) 32 | ) == ['{job=~".+"} |~ `(?i)valueA` | logfmt | fieldA=~`(?i)^valueA$`'] 33 | 34 | 35 | def test_loki_field_not_eq(loki_backend: LogQLBackend): 36 | assert loki_backend.convert( 37 | SigmaCollection.from_yaml( 38 | """ 39 | title: Test 40 | status: test 41 | logsource: 42 | category: test_category 43 | product: test_product 44 | detection: 45 | sel: 46 | fieldA: valueA 47 | condition: not sel 48 | """ 49 | ) 50 | ) == ['{job=~".+"} !~ `(?i)valueA` | logfmt | fieldA!~`(?i)^valueA$`'] 51 | 52 | 53 | def test_loki_lf_field_eq_wildcard(loki_backend: LogQLBackend): 54 | assert loki_backend.convert( 55 | SigmaCollection.from_yaml( 56 | """ 57 | title: Test 58 | status: test 59 | logsource: 60 | category: test_category 61 | product: test_product 62 | detection: 63 | sel: 64 | fieldA: value?A 65 | condition: sel 66 | """ 67 | ) 68 | ) == ['{job=~".+"} |~ `(?i)value.A` | logfmt | fieldA=~`(?i)^value.A$`'] 69 | 70 | 71 | def test_loki_lf_field_not_eq_wildcard(loki_backend: LogQLBackend): 72 | assert loki_backend.convert( 73 | SigmaCollection.from_yaml( 74 | """ 75 | title: Test 76 | status: test 77 | logsource: 78 | category: test_category 79 | product: test_product 80 | detection: 81 | sel: 82 | fieldA: value?A 83 | condition: not sel 84 | """ 85 | ) 86 | ) == ['{job=~".+"} !~ `(?i)value.A` | logfmt | fieldA!~`(?i)^value.A$`'] 87 | 88 | 89 | def test_loki_lf_field_eq_num(loki_backend: LogQLBackend): 90 | assert loki_backend.convert( 91 | SigmaCollection.from_yaml( 92 | """ 93 | title: Test 94 | status: test 95 | logsource: 96 | category: test_category 97 | product: test_product 98 | detection: 99 | sel: 100 | fieldA: 100 101 | condition: sel 102 | """ 103 | ) 104 | ) == ['{job=~".+"} |= `fieldA=100` | logfmt | fieldA=100'] 105 | 106 | 107 | # Testing boolean logic 108 | def test_loki_lf_and_expression(loki_backend: LogQLBackend): 109 | assert loki_backend.convert( 110 | SigmaCollection.from_yaml( 111 | """ 112 | title: Test 113 | status: test 114 | logsource: 115 | category: test_category 116 | product: test_product 117 | detection: 118 | sel: 119 | fieldA: valueA 120 | fieldB: valueB 121 | condition: sel 122 | """ 123 | ) 124 | ) == [ 125 | '{job=~".+"} |~ `(?i)valueA` | logfmt | fieldA=~`(?i)^valueA$` and fieldB=~`(?i)^valueB$`' 126 | ] 127 | 128 | 129 | # Must not introduce partial negations, since it would exclude valid log entries 130 | # such as: fieldA=good fieldB=good fieldC=value 131 | def test_loki_lf_not_and_expression(loki_backend: LogQLBackend): 132 | assert loki_backend.convert( 133 | SigmaCollection.from_yaml( 134 | """ 135 | title: Test 136 | status: test 137 | logsource: 138 | category: test_category 139 | product: test_product 140 | detection: 141 | sel: 142 | fieldA: valueA 143 | fieldB: valueB 144 | condition: not sel 145 | """ 146 | ) 147 | ) == ['{job=~".+"} | logfmt | fieldA!~`(?i)^valueA$` or fieldB!~`(?i)^valueB$`'] 148 | 149 | 150 | def test_loki_lf_or_expression(loki_backend: LogQLBackend): 151 | assert loki_backend.convert( 152 | SigmaCollection.from_yaml( 153 | """ 154 | title: Test 155 | status: test 156 | logsource: 157 | category: test_category 158 | product: test_product 159 | detection: 160 | sel1: 161 | fieldA: valueA 162 | sel2: 163 | fieldB: valueB 164 | condition: 1 of sel* 165 | """ 166 | ) 167 | ) == ['{job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` or fieldB=~`(?i)^valueB$`'] 168 | 169 | 170 | def test_loki_lf_not_or_expression(loki_backend: LogQLBackend): 171 | assert loki_backend.convert( 172 | SigmaCollection.from_yaml( 173 | """ 174 | title: Test 175 | status: test 176 | logsource: 177 | category: test_category 178 | product: test_product 179 | detection: 180 | sel1: 181 | fieldA: valueA 182 | sel2: 183 | fieldB: valueB 184 | condition: not 1 of sel* 185 | """ 186 | ) 187 | ) == [ 188 | '{job=~".+"} !~ `(?i)valueA` | logfmt | fieldA!~`(?i)^valueA$` and fieldB!~`(?i)^valueB$`' 189 | ] 190 | 191 | 192 | def test_loki_lf_or_no_filter_expression(loki_backend: LogQLBackend): 193 | assert loki_backend.convert( 194 | SigmaCollection.from_yaml( 195 | """ 196 | title: Test 197 | status: test 198 | logsource: 199 | category: test_category 200 | product: windows 201 | detection: 202 | sel1: 203 | aaaa: bbbb 204 | sel2: 205 | cccc: dddd 206 | condition: 1 of sel* 207 | """ 208 | ) 209 | ) == [ 210 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json | aaaa=~`(?i)^bbbb$` ' 211 | "or cccc=~`(?i)^dddd$`" 212 | ] 213 | 214 | 215 | def test_loki_lf_and_or_expression(loki_backend: LogQLBackend): 216 | assert loki_backend.convert( 217 | SigmaCollection.from_yaml( 218 | """ 219 | title: Test 220 | status: test 221 | logsource: 222 | category: test_category 223 | product: test_product 224 | detection: 225 | sel: 226 | fieldA: 227 | - valueA1 228 | - valueA2 229 | fieldB: 230 | - valueB1 231 | - valueB2 232 | condition: sel 233 | """ 234 | ) 235 | ) == [ 236 | '{job=~".+"} | logfmt | (fieldA=~`(?i)^valueA1$` or fieldA=~`(?i)^valueA2$`) and ' 237 | "(fieldB=~`(?i)^valueB1$` or fieldB=~`(?i)^valueB2$`)" 238 | ] 239 | 240 | 241 | def test_loki_lf_or_and_expression(loki_backend: LogQLBackend): 242 | assert loki_backend.convert( 243 | SigmaCollection.from_yaml( 244 | """ 245 | title: Test 246 | status: test 247 | logsource: 248 | category: test_category 249 | product: test_product 250 | detection: 251 | sel1: 252 | fieldA: valueA1 253 | fieldB: valueB1 254 | sel2: 255 | fieldA: valueA2 256 | fieldB: valueB2 257 | condition: 1 of sel* 258 | """ 259 | ) 260 | ) == [ 261 | '{job=~".+"} | logfmt | fieldA=~`(?i)^valueA1$` and fieldB=~`(?i)^valueB1$` or ' 262 | "fieldA=~`(?i)^valueA2$` and fieldB=~`(?i)^valueB2$`" 263 | ] 264 | 265 | 266 | # Loki doesn't support in expressions, so in this case, multiple or conditions should be produced 267 | def test_loki_lf_in_expression(loki_backend: LogQLBackend): 268 | assert loki_backend.convert( 269 | SigmaCollection.from_yaml( 270 | """ 271 | title: Test 272 | status: test 273 | logsource: 274 | category: test_category 275 | product: test_product 276 | detection: 277 | sel: 278 | fieldA: 279 | - valueA 280 | - valueB 281 | - valueC 282 | condition: sel 283 | """ 284 | ) 285 | ) == [ 286 | '{job=~".+"} | logfmt | fieldA=~`(?i)^valueA$` or fieldA=~`(?i)^valueB$` ' 287 | "or fieldA=~`(?i)^valueC$`" 288 | ] 289 | 290 | 291 | def test_loki_lf_all_query(loki_backend: LogQLBackend): 292 | assert loki_backend.convert( 293 | SigmaCollection.from_yaml( 294 | """ 295 | title: Test 296 | status: test 297 | logsource: 298 | category: test_category 299 | product: test_product 300 | detection: 301 | sel: 302 | fieldA|all: 303 | - valueA 304 | - valueB 305 | condition: sel 306 | """ 307 | ) 308 | ) == [ 309 | '{job=~".+"} |~ `(?i)valueA` | logfmt | fieldA=~`(?i)^valueA$` and fieldA=~`(?i)^valueB$`' 310 | ] 311 | 312 | 313 | def test_loki_lf_all_contains_query(loki_backend: LogQLBackend): 314 | assert loki_backend.convert( 315 | SigmaCollection.from_yaml( 316 | """ 317 | title: Test 318 | status: test 319 | logsource: 320 | category: test_category 321 | product: test_product 322 | detection: 323 | sel: 324 | fieldA|all|contains: 325 | - valueA 326 | - valueB 327 | condition: sel 328 | """ 329 | ) 330 | ) == [ 331 | '{job=~".+"} |~ `(?i).*valueA.*` | logfmt | fieldA=~`(?i).*valueA.*` ' 332 | "and fieldA=~`(?i).*valueB.*`" 333 | ] 334 | 335 | 336 | # Testing different search identifiers 337 | def test_loki_lf_null(loki_backend: LogQLBackend): 338 | assert loki_backend.convert( 339 | SigmaCollection.from_yaml( 340 | """ 341 | title: Test 342 | status: test 343 | logsource: 344 | category: test_category 345 | product: test_product 346 | detection: 347 | sel: 348 | fieldA: null 349 | condition: sel 350 | """ 351 | ) 352 | ) == ['{job=~".+"} |= `fieldA=` | logfmt | fieldA=``'] 353 | 354 | 355 | # Loki does not support wildcards, so we use case-insensitive regular expressions instead 356 | def test_loki_lf_wildcard_single(loki_backend: LogQLBackend): 357 | assert loki_backend.convert( 358 | SigmaCollection.from_yaml( 359 | """ 360 | title: Test 361 | status: test 362 | logsource: 363 | category: test_category 364 | product: test_product 365 | detection: 366 | sel: 367 | fieldA: va?ue 368 | condition: sel 369 | """ 370 | ) 371 | ) == ['{job=~".+"} |~ `(?i)va.ue` | logfmt | fieldA=~`(?i)^va.ue$`'] 372 | 373 | 374 | def test_loki_lf_wildcard_multi(loki_backend: LogQLBackend): 375 | assert loki_backend.convert( 376 | SigmaCollection.from_yaml( 377 | """ 378 | title: Test 379 | status: test 380 | logsource: 381 | category: test_category 382 | product: test_product 383 | detection: 384 | sel: 385 | fieldA: value* 386 | condition: sel 387 | """ 388 | ) 389 | ) == ['{job=~".+"} |~ `(?i)value.*` | logfmt | fieldA=~`(?i)^value.*`'] 390 | 391 | 392 | # Wildcarded searches may include other regex metacharacters - 393 | # these need to be escaped to prevent them from being used in the transformed query 394 | def test_loki_lf_wildcard_escape(loki_backend: LogQLBackend): 395 | assert loki_backend.convert( 396 | SigmaCollection.from_yaml( 397 | """ 398 | title: Test 399 | status: test 400 | logsource: 401 | category: test_category 402 | product: test_product 403 | detection: 404 | sel: 405 | fieldA: ^v)+[al]u(e*$ 406 | condition: sel 407 | """ 408 | ) 409 | ) == [ 410 | '{job=~".+"} |~ `(?i)\\^v\\)\\+\\[al\\]u\\(e.*\\$` | logfmt | ' 411 | "fieldA=~`(?i)^\\^v\\)\\+\\[al\\]u\\(e.*\\$$`" 412 | ] 413 | 414 | 415 | def test_loki_lf_regex_query(loki_backend: LogQLBackend): 416 | assert loki_backend.convert( 417 | SigmaCollection.from_yaml( 418 | """ 419 | title: Test 420 | status: test 421 | logsource: 422 | category: test_category 423 | product: test_product 424 | detection: 425 | sel: 426 | fieldA|re: foo.*bar 427 | fieldB: foo 428 | condition: sel 429 | """ 430 | ) 431 | ) == ['{job=~".+"} |~ `foo.*bar` | logfmt | fieldA=~`foo.*bar` and fieldB=~`(?i)^foo$`'] 432 | 433 | 434 | def test_loki_lf_field_re_tilde(loki_backend: LogQLBackend): 435 | assert loki_backend.convert( 436 | SigmaCollection.from_yaml( 437 | """ 438 | title: Test 439 | status: test 440 | logsource: 441 | category: test_category 442 | product: test_product 443 | detection: 444 | sel: 445 | fieldA|re: value`A 446 | condition: sel 447 | """ 448 | ) 449 | ) == ['{job=~".+"} |~ "value`A" | logfmt | fieldA=~"value`A"'] 450 | 451 | 452 | def test_loki_lf_field_re_tilde_double_quote(loki_backend: LogQLBackend): 453 | assert loki_backend.convert( 454 | SigmaCollection.from_yaml( 455 | """ 456 | title: Test 457 | status: test 458 | logsource: 459 | category: test_category 460 | product: test_product 461 | detection: 462 | sel: 463 | fieldA|re: v"alue`A 464 | condition: sel 465 | """ 466 | ) 467 | ) == ['{job=~".+"} |~ "v\\"alue`A" | logfmt | fieldA=~"v\\"alue`A"'] 468 | 469 | 470 | def test_loki_lf_field_startswith(loki_backend: LogQLBackend): 471 | assert loki_backend.convert( 472 | SigmaCollection.from_yaml( 473 | """ 474 | title: Test 475 | status: test 476 | logsource: 477 | category: test_category 478 | product: test_product 479 | detection: 480 | sel: 481 | fieldA|startswith: foo 482 | condition: sel 483 | """ 484 | ) 485 | ) == ['{job=~".+"} |~ `(?i)foo.*` | logfmt | fieldA=~`(?i)^foo.*`'] 486 | 487 | 488 | def test_loki_lf_field_endswith(loki_backend: LogQLBackend): 489 | assert loki_backend.convert( 490 | SigmaCollection.from_yaml( 491 | """ 492 | title: Test 493 | status: test 494 | logsource: 495 | category: test_category 496 | product: test_product 497 | detection: 498 | sel: 499 | fieldA|endswith: bar 500 | condition: sel 501 | """ 502 | ) 503 | ) == ['{job=~".+"} |~ `(?i).*bar` | logfmt | fieldA=~`(?i).*bar$`'] 504 | 505 | 506 | def test_loki_lf_field_contains(loki_backend: LogQLBackend): 507 | assert loki_backend.convert( 508 | SigmaCollection.from_yaml( 509 | """ 510 | title: Test 511 | status: test 512 | logsource: 513 | category: test_category 514 | product: test_product 515 | detection: 516 | sel: 517 | fieldA|contains: ooba 518 | condition: sel 519 | """ 520 | ) 521 | ) == ['{job=~".+"} |~ `(?i).*ooba.*` | logfmt | fieldA=~`(?i).*ooba.*`'] 522 | 523 | 524 | def test_loki_lf_cidr_query(loki_backend: LogQLBackend): 525 | assert loki_backend.convert( 526 | SigmaCollection.from_yaml( 527 | """ 528 | title: Test 529 | status: test 530 | logsource: 531 | category: test_category 532 | product: test_product 533 | detection: 534 | sel: 535 | fieldA|cidr: 192.168.0.0/16 536 | condition: sel 537 | """ 538 | ) 539 | ) == ['{job=~".+"} |= ip("192.168.0.0/16") | logfmt | fieldA=ip("192.168.0.0/16")'] 540 | 541 | 542 | def test_loki_lf_not_cidr_query(loki_backend: LogQLBackend): 543 | assert loki_backend.convert( 544 | SigmaCollection.from_yaml( 545 | """ 546 | title: Test 547 | status: test 548 | logsource: 549 | category: test_category 550 | product: test_product 551 | detection: 552 | sel: 553 | fieldA|cidr: 192.168.0.0/16 554 | condition: not sel 555 | """ 556 | ) 557 | ) == ['{job=~".+"} != ip("192.168.0.0/16") | logfmt | fieldA!=ip("192.168.0.0/16")'] 558 | 559 | 560 | def test_loki_lf_base64_query(loki_backend: LogQLBackend): 561 | assert loki_backend.convert( 562 | SigmaCollection.from_yaml( 563 | """ 564 | title: Test 565 | status: test 566 | logsource: 567 | category: test_category 568 | product: test_product 569 | detection: 570 | sel: 571 | fieldA|base64: value 572 | condition: sel 573 | """ 574 | ) 575 | ) == ['{job=~".+"} |~ `(?i)dmFsdWU=` | logfmt | fieldA=~`(?i)^dmFsdWU=$`'] 576 | 577 | 578 | def test_loki_lf_base64offset_query(loki_backend: LogQLBackend): 579 | assert loki_backend.convert( 580 | SigmaCollection.from_yaml( 581 | """ 582 | title: Test 583 | status: test 584 | logsource: 585 | category: test_category 586 | product: test_product 587 | detection: 588 | sel: 589 | fieldA|base64offset: value 590 | condition: sel 591 | """ 592 | ) 593 | ) == [ 594 | '{job=~".+"} | logfmt | fieldA=~`(?i)^dmFsdW$` or fieldA=~`(?i)^ZhbHVl$` or ' 595 | "fieldA=~`(?i)^2YWx1Z$`" 596 | ] 597 | 598 | 599 | def test_loki_lf_field_name_with_whitespace(loki_backend: LogQLBackend): 600 | assert loki_backend.convert( 601 | SigmaCollection.from_yaml( 602 | """ 603 | title: Test 604 | status: test 605 | logsource: 606 | category: test_category 607 | product: test_product 608 | detection: 609 | sel: 610 | field name: value 611 | condition: sel 612 | """ 613 | ) 614 | ) == ['{job=~".+"} |~ `(?i)value` | logfmt | field_name=~`(?i)^value$`'] 615 | 616 | 617 | def test_loki_lf_field_name_leading_num(loki_backend: LogQLBackend): 618 | assert loki_backend.convert( 619 | SigmaCollection.from_yaml( 620 | """ 621 | title: Test 622 | status: test 623 | logsource: 624 | category: test_category 625 | product: test_product 626 | detection: 627 | sel: 628 | 0field: value 629 | condition: sel 630 | """ 631 | ) 632 | ) == ['{job=~".+"} |~ `(?i)value` | logfmt | _0field=~`(?i)^value$`'] 633 | 634 | 635 | def test_loki_lf_field_name_invalid(loki_backend: LogQLBackend): 636 | assert loki_backend.convert( 637 | SigmaCollection.from_yaml( 638 | """ 639 | title: Test 640 | status: test 641 | logsource: 642 | category: test_category 643 | product: test_product 644 | detection: 645 | sel: 646 | field.name@A-Z: value 647 | condition: sel 648 | """ 649 | ) 650 | ) == ['{job=~".+"} |~ `(?i)value` | logfmt | field_name_A_Z=~`(?i)^value$`'] 651 | 652 | 653 | # Ensure existing line filters prevent addition of new ones 654 | def test_loki_lf_unbound(loki_backend: LogQLBackend): 655 | assert loki_backend.convert( 656 | SigmaCollection.from_yaml( 657 | """ 658 | title: test 659 | status: test 660 | logsource: 661 | category: test_category 662 | product: test_product 663 | detection: 664 | keywords: 665 | value 666 | sel: 667 | fieldA: valueA 668 | condition: keywords and sel 669 | """ 670 | ) 671 | ) == ['{job=~".+"} |~ `(?i)value` | logfmt | fieldA=~`(?i)^valueA$`'] 672 | 673 | 674 | def test_loki_lf_and_unbound(loki_backend: LogQLBackend): 675 | assert loki_backend.convert( 676 | SigmaCollection.from_yaml( 677 | """ 678 | title: test 679 | status: test 680 | logsource: 681 | category: test_category 682 | product: test_product 683 | detection: 684 | keyword1: 685 | valueA 686 | keyword2: 687 | valueB 688 | sel: 689 | fieldA: valueA 690 | condition: keyword1 and keyword2 and sel 691 | """ 692 | ) 693 | ) == ['{job=~".+"} |~ `(?i)valueA` |~ `(?i)valueB` | logfmt | fieldA=~`(?i)^valueA$`'] 694 | 695 | 696 | def test_loki_lf_or_unbound(loki_backend: LogQLBackend): 697 | assert loki_backend.convert( 698 | SigmaCollection.from_yaml( 699 | """ 700 | title: Test 701 | status: test 702 | logsource: 703 | category: test_category 704 | product: test_product 705 | detection: 706 | keywords: 707 | - valueA 708 | - valueB 709 | sel: 710 | fieldA: valueA 711 | condition: keywords and sel 712 | """ 713 | ) 714 | ) == ['{job=~".+"} |~ `(?i)valueA|valueB` | logfmt | fieldA=~`(?i)^valueA$`'] 715 | 716 | 717 | # Testing specific logsources and other Sigma features 718 | def test_loki_lf_windows_logsource(loki_backend: LogQLBackend): 719 | assert loki_backend.convert( 720 | SigmaCollection.from_yaml( 721 | """ 722 | title: Test 723 | status: test 724 | logsource: 725 | category: test_category 726 | product: windows 727 | detection: 728 | sel: 729 | key1.key2: value 730 | condition: sel 731 | """ 732 | ) 733 | ) == [ 734 | '{job=~"eventlog|winlog|windows|fluentbit.*"} |~ `(?i)value` | json | ' 735 | "key1_key2=~`(?i)^value$`" 736 | ] 737 | 738 | 739 | def test_loki_lf_azure_logsource(loki_backend: LogQLBackend): 740 | assert loki_backend.convert( 741 | SigmaCollection.from_yaml( 742 | """ 743 | title: Test 744 | status: test 745 | logsource: 746 | category: test_category 747 | product: azure 748 | detection: 749 | sel: 750 | key1.key2: value 751 | condition: sel 752 | """ 753 | ) 754 | ) == ['{job="logstash"} |~ `(?i)value` | json | key1_key2=~`(?i)^value$`'] 755 | 756 | 757 | def test_loki_lf_very_long_query_or(loki_backend: LogQLBackend): 758 | long_field = f"longField{'A' * 50}" 759 | yaml = ( 760 | f""" 761 | title: Test 762 | status: test 763 | logsource: 764 | category: test_category 765 | product: test_product 766 | detection: 767 | sel_each: 768 | {long_field}: valueA 769 | sel_partitioned: 770 | longFieldB|contains: 771 | """ 772 | + "\n".join( 773 | " - " + "".join(random.choices(string.ascii_letters, k=50)) 774 | for _ in range(100) 775 | ) 776 | + """ 777 | condition: all of sel_* 778 | """ 779 | ) 780 | test = loki_backend.convert(SigmaCollection.from_yaml(yaml)) 781 | assert ( 782 | len(test) > 1 783 | and all("|~ `(?i)valueA` |" in q and f"{long_field}=~`(?i)^valueA$`" in q for q in test) 784 | and all(len(q) < 5120 for q in test) 785 | ) 786 | 787 | 788 | def test_loki_lf_very_long_query_or_right_filters(loki_backend: LogQLBackend): 789 | yaml = f""" 790 | title: Test 791 | status: test 792 | logsource: 793 | category: test_category 794 | product: test_product 795 | detection: 796 | sel_A: 797 | fieldA: value{"A" * 4000} 798 | sel_B: 799 | fieldB: value{"B" * 4000} 800 | condition: 1 of sel_* 801 | """ 802 | test = loki_backend.convert(SigmaCollection.from_yaml(yaml)) 803 | assert len(test) > 1 and all( 804 | re.match("|= `field([AB])=value\1{4000}` | .* | field\1=~`(\\?i)value\1{4000}`", q) 805 | for q in test 806 | ) 807 | -------------------------------------------------------------------------------- /tests/test_backend_loki_add_line_filters.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import random 3 | import re 4 | import string 5 | from sigma.backends.loki import LogQLBackend 6 | from sigma.collection import SigmaCollection 7 | 8 | # from sigma.exceptions import SigmaFeatureNotSupportedByBackendError 9 | 10 | 11 | @pytest.fixture 12 | def loki_backend(): 13 | return LogQLBackend(add_line_filters=True, case_sensitive=True) 14 | 15 | 16 | # Testing line filters introduction 17 | def test_loki_lf_field_eq(loki_backend: LogQLBackend): 18 | assert loki_backend.convert( 19 | SigmaCollection.from_yaml( 20 | """ 21 | title: Test 22 | status: test 23 | logsource: 24 | category: test_category 25 | product: test_product 26 | detection: 27 | sel: 28 | fieldA: valueA 29 | condition: sel 30 | """ 31 | ) 32 | ) == ['{job=~".+"} |= `fieldA=valueA` | logfmt | fieldA=`valueA`'] 33 | 34 | 35 | # In this context, a full negation of fieldA=valueA would be acceptable to include 36 | # however the likely performance benefit is not obvious 37 | def test_loki_field_not_eq(loki_backend: LogQLBackend): 38 | assert loki_backend.convert( 39 | SigmaCollection.from_yaml( 40 | """ 41 | title: Test 42 | status: test 43 | logsource: 44 | category: test_category 45 | product: test_product 46 | detection: 47 | sel: 48 | fieldA: valueA 49 | condition: not sel 50 | """ 51 | ) 52 | ) == ['{job=~".+"} != `fieldA=valueA` | logfmt | fieldA!=`valueA`'] 53 | 54 | 55 | def test_loki_lf_field_eq_wildcard(loki_backend: LogQLBackend): 56 | assert loki_backend.convert( 57 | SigmaCollection.from_yaml( 58 | """ 59 | title: Test 60 | status: test 61 | logsource: 62 | category: test_category 63 | product: test_product 64 | detection: 65 | sel: 66 | fieldA: value?A 67 | condition: sel 68 | """ 69 | ) 70 | ) == ['{job=~".+"} |~ `(?i)value.A` | logfmt | fieldA=~`(?i)^value.A$`'] 71 | 72 | 73 | def test_loki_lf_field_not_eq_wildcard(loki_backend: LogQLBackend): 74 | assert loki_backend.convert( 75 | SigmaCollection.from_yaml( 76 | """ 77 | title: Test 78 | status: test 79 | logsource: 80 | category: test_category 81 | product: test_product 82 | detection: 83 | sel: 84 | fieldA: value?A 85 | condition: not sel 86 | """ 87 | ) 88 | ) == ['{job=~".+"} !~ `(?i)value.A` | logfmt | fieldA!~`(?i)^value.A$`'] 89 | 90 | 91 | def test_loki_lf_field_eq_num(loki_backend: LogQLBackend): 92 | assert loki_backend.convert( 93 | SigmaCollection.from_yaml( 94 | """ 95 | title: Test 96 | status: test 97 | logsource: 98 | category: test_category 99 | product: test_product 100 | detection: 101 | sel: 102 | fieldA: 100 103 | condition: sel 104 | """ 105 | ) 106 | ) == ['{job=~".+"} |= `fieldA=100` | logfmt | fieldA=100'] 107 | 108 | 109 | # Testing boolean logic 110 | def test_loki_lf_and_expression(loki_backend: LogQLBackend): 111 | assert loki_backend.convert( 112 | SigmaCollection.from_yaml( 113 | """ 114 | title: Test 115 | status: test 116 | logsource: 117 | category: test_category 118 | product: test_product 119 | detection: 120 | sel: 121 | fieldA: valueA 122 | fieldB: valueB 123 | condition: sel 124 | """ 125 | ) 126 | ) == ['{job=~".+"} |= `fieldA=valueA` | logfmt | fieldA=`valueA` and fieldB=`valueB`'] 127 | 128 | 129 | # Must not introduce partial negations, since it would exclude valid log entries 130 | # such as: fieldA=good fieldB=good fieldC=value 131 | def test_loki_lf_not_and_expression(loki_backend: LogQLBackend): 132 | assert loki_backend.convert( 133 | SigmaCollection.from_yaml( 134 | """ 135 | title: Test 136 | status: test 137 | logsource: 138 | category: test_category 139 | product: test_product 140 | detection: 141 | sel: 142 | fieldA: valueA 143 | fieldB: valueB 144 | condition: not sel 145 | """ 146 | ) 147 | ) == ['{job=~".+"} | logfmt | fieldA!=`valueA` or fieldB!=`valueB`'] 148 | 149 | 150 | def test_loki_lf_or_expression(loki_backend: LogQLBackend): 151 | """Test that, when an OR condition contains a common substring in all arguments, 152 | the substring will be used as a line filter. 153 | 154 | In this case, we know: the parser is logfmt and therefore a log entry of interest 155 | should contain at least fieldA=valueA or fieldB=valueB. It means it will contain at 156 | the very least =value (the longest common substring between those two values) and 157 | hence we can use it as a line filter.""" 158 | assert loki_backend.convert( 159 | SigmaCollection.from_yaml( 160 | """ 161 | title: Test 162 | status: test 163 | logsource: 164 | category: test_category 165 | product: test_product 166 | detection: 167 | sel1: 168 | fieldA: valueA 169 | sel2: 170 | fieldB: valueB 171 | condition: 1 of sel* 172 | """ 173 | ) 174 | ) == ['{job=~".+"} |= `=value` | logfmt | fieldA=`valueA` or fieldB=`valueB`'] 175 | 176 | 177 | def test_loki_lf_not_or_expression(loki_backend: LogQLBackend): 178 | assert loki_backend.convert( 179 | SigmaCollection.from_yaml( 180 | """ 181 | title: Test 182 | status: test 183 | logsource: 184 | category: test_category 185 | product: test_product 186 | detection: 187 | sel1: 188 | fieldA: valueA 189 | sel2: 190 | fieldB: valueB 191 | condition: not 1 of sel* 192 | """ 193 | ) 194 | ) == ['{job=~".+"} != `fieldA=valueA` | logfmt | fieldA!=`valueA` and fieldB!=`valueB`'] 195 | 196 | 197 | def test_loki_lf_or_no_filter_expression(loki_backend: LogQLBackend): 198 | assert loki_backend.convert( 199 | SigmaCollection.from_yaml( 200 | """ 201 | title: Test 202 | status: test 203 | logsource: 204 | category: test_category 205 | product: windows 206 | detection: 207 | sel1: 208 | aaaa: bbbb 209 | sel2: 210 | cccc: dddd 211 | condition: 1 of sel* 212 | """ 213 | ) 214 | ) == ['{job=~"eventlog|winlog|windows|fluentbit.*"} | json | aaaa=`bbbb` or cccc=`dddd`'] 215 | 216 | 217 | def test_loki_lf_and_or_expression(loki_backend: LogQLBackend): 218 | assert loki_backend.convert( 219 | SigmaCollection.from_yaml( 220 | """ 221 | title: Test 222 | status: test 223 | logsource: 224 | category: test_category 225 | product: test_product 226 | detection: 227 | sel: 228 | fieldA: 229 | - valueA1 230 | - valueA2 231 | fieldB: 232 | - valueB1 233 | - valueB2 234 | condition: sel 235 | """ 236 | ) 237 | ) == [ 238 | '{job=~".+"} |= `fieldA=valueA` | logfmt | (fieldA=`valueA1` or fieldA=`valueA2`) and ' 239 | "(fieldB=`valueB1` or fieldB=`valueB2`)" 240 | ] 241 | 242 | 243 | def test_loki_lf_or_and_expression(loki_backend: LogQLBackend): 244 | assert loki_backend.convert( 245 | SigmaCollection.from_yaml( 246 | """ 247 | title: Test 248 | status: test 249 | logsource: 250 | category: test_category 251 | product: test_product 252 | detection: 253 | sel1: 254 | fieldA: valueA1 255 | fieldB: valueB1 256 | sel2: 257 | fieldA: valueA2 258 | fieldB: valueB2 259 | condition: 1 of sel* 260 | """ 261 | ) 262 | ) == [ 263 | '{job=~".+"} |= `fieldA=valueA` | logfmt | fieldA=`valueA1` and fieldB=`valueB1` or ' 264 | "fieldA=`valueA2` and fieldB=`valueB2`" 265 | ] 266 | 267 | 268 | # Loki doesn't support in expressions, so in this case, multiple or conditions should be produced 269 | def test_loki_lf_in_expression(loki_backend: LogQLBackend): 270 | assert loki_backend.convert( 271 | SigmaCollection.from_yaml( 272 | """ 273 | title: Test 274 | status: test 275 | logsource: 276 | category: test_category 277 | product: test_product 278 | detection: 279 | sel: 280 | fieldA: 281 | - valueA 282 | - valueB 283 | - valueC 284 | condition: sel 285 | """ 286 | ) 287 | ) == [ 288 | '{job=~".+"} |= `fieldA=value` | logfmt | fieldA=`valueA` or fieldA=`valueB` ' 289 | "or fieldA=`valueC`" 290 | ] 291 | 292 | 293 | def test_loki_lf_all_query(loki_backend: LogQLBackend): 294 | assert loki_backend.convert( 295 | SigmaCollection.from_yaml( 296 | """ 297 | title: Test 298 | status: test 299 | logsource: 300 | category: test_category 301 | product: test_product 302 | detection: 303 | sel: 304 | fieldA|all: 305 | - valueA 306 | - valueB 307 | condition: sel 308 | """ 309 | ) 310 | ) == ['{job=~".+"} |= `fieldA=valueA` | logfmt | fieldA=`valueA` and fieldA=`valueB`'] 311 | 312 | 313 | def test_loki_lf_all_contains_query(loki_backend: LogQLBackend): 314 | assert loki_backend.convert( 315 | SigmaCollection.from_yaml( 316 | """ 317 | title: Test 318 | status: test 319 | logsource: 320 | category: test_category 321 | product: test_product 322 | detection: 323 | sel: 324 | fieldA|all|contains: 325 | - valueA 326 | - valueB 327 | condition: sel 328 | """ 329 | ) 330 | ) == [ 331 | '{job=~".+"} |~ `(?i).*valueA.*` | logfmt | fieldA=~`(?i).*valueA.*` ' 332 | "and fieldA=~`(?i).*valueB.*`" 333 | ] 334 | 335 | 336 | # Testing different search identifiers 337 | def test_loki_lf_null(loki_backend: LogQLBackend): 338 | assert loki_backend.convert( 339 | SigmaCollection.from_yaml( 340 | """ 341 | title: Test 342 | status: test 343 | logsource: 344 | category: test_category 345 | product: test_product 346 | detection: 347 | sel: 348 | fieldA: null 349 | condition: sel 350 | """ 351 | ) 352 | ) == ['{job=~".+"} |= `fieldA=` | logfmt | fieldA=``'] 353 | 354 | 355 | # Loki does not support wildcards, so we use case-insensitive regular expressions instead 356 | def test_loki_lf_wildcard_single(loki_backend: LogQLBackend): 357 | assert loki_backend.convert( 358 | SigmaCollection.from_yaml( 359 | """ 360 | title: Test 361 | status: test 362 | logsource: 363 | category: test_category 364 | product: test_product 365 | detection: 366 | sel: 367 | fieldA: va?ue 368 | condition: sel 369 | """ 370 | ) 371 | ) == ['{job=~".+"} |~ `(?i)va.ue` | logfmt | fieldA=~`(?i)^va.ue$`'] 372 | 373 | 374 | def test_loki_lf_wildcard_multi(loki_backend: LogQLBackend): 375 | assert loki_backend.convert( 376 | SigmaCollection.from_yaml( 377 | """ 378 | title: Test 379 | status: test 380 | logsource: 381 | category: test_category 382 | product: test_product 383 | detection: 384 | sel: 385 | fieldA: value* 386 | condition: sel 387 | """ 388 | ) 389 | ) == ['{job=~".+"} |~ `(?i)value.*` | logfmt | fieldA=~`(?i)^value.*`'] 390 | 391 | 392 | # Wildcarded searches may include other regex metacharacters - 393 | # these need to be escaped to prevent them from being used in the transformed query 394 | def test_loki_lf_wildcard_escape(loki_backend: LogQLBackend): 395 | assert loki_backend.convert( 396 | SigmaCollection.from_yaml( 397 | """ 398 | title: Test 399 | status: test 400 | logsource: 401 | category: test_category 402 | product: test_product 403 | detection: 404 | sel: 405 | fieldA: ^v)+[al]u(e*$ 406 | condition: sel 407 | """ 408 | ) 409 | ) == [ 410 | '{job=~".+"} |~ `(?i)\\^v\\)\\+\\[al\\]u\\(e.*\\$` | logfmt | ' 411 | "fieldA=~`(?i)^\\^v\\)\\+\\[al\\]u\\(e.*\\$$`" 412 | ] 413 | 414 | 415 | def test_loki_lf_regex_query(loki_backend: LogQLBackend): 416 | assert loki_backend.convert( 417 | SigmaCollection.from_yaml( 418 | """ 419 | title: Test 420 | status: test 421 | logsource: 422 | category: test_category 423 | product: test_product 424 | detection: 425 | sel: 426 | fieldA|re: foo.*bar 427 | fieldB: foo 428 | condition: sel 429 | """ 430 | ) 431 | ) == ['{job=~".+"} |= `fieldB=foo` | logfmt | fieldA=~`foo.*bar` and fieldB=`foo`'] 432 | 433 | 434 | def test_loki_lf_field_re_tilde(loki_backend: LogQLBackend): 435 | assert loki_backend.convert( 436 | SigmaCollection.from_yaml( 437 | """ 438 | title: Test 439 | status: test 440 | logsource: 441 | category: test_category 442 | product: test_product 443 | detection: 444 | sel: 445 | fieldA|re: value`A 446 | condition: sel 447 | """ 448 | ) 449 | ) == ['{job=~".+"} |~ "value`A" | logfmt | fieldA=~"value`A"'] 450 | 451 | 452 | def test_loki_lf_field_re_tilde_double_quote(loki_backend: LogQLBackend): 453 | assert loki_backend.convert( 454 | SigmaCollection.from_yaml( 455 | """ 456 | title: Test 457 | status: test 458 | logsource: 459 | category: test_category 460 | product: test_product 461 | detection: 462 | sel: 463 | fieldA|re: v"alue`A 464 | condition: sel 465 | """ 466 | ) 467 | ) == ['{job=~".+"} |~ "v\\"alue`A" | logfmt | fieldA=~"v\\"alue`A"'] 468 | 469 | 470 | def test_loki_lf_field_startswith(loki_backend: LogQLBackend): 471 | assert loki_backend.convert( 472 | SigmaCollection.from_yaml( 473 | """ 474 | title: Test 475 | status: test 476 | logsource: 477 | category: test_category 478 | product: test_product 479 | detection: 480 | sel: 481 | fieldA|startswith: foo 482 | condition: sel 483 | """ 484 | ) 485 | ) == ['{job=~".+"} |~ `(?i)foo.*` | logfmt | fieldA=~`(?i)^foo.*`'] 486 | 487 | 488 | def test_loki_lf_field_endswith(loki_backend: LogQLBackend): 489 | assert loki_backend.convert( 490 | SigmaCollection.from_yaml( 491 | """ 492 | title: Test 493 | status: test 494 | logsource: 495 | category: test_category 496 | product: test_product 497 | detection: 498 | sel: 499 | fieldA|endswith: bar 500 | condition: sel 501 | """ 502 | ) 503 | ) == ['{job=~".+"} |~ `(?i).*bar` | logfmt | fieldA=~`(?i).*bar$`'] 504 | 505 | 506 | def test_loki_lf_field_contains(loki_backend: LogQLBackend): 507 | assert loki_backend.convert( 508 | SigmaCollection.from_yaml( 509 | """ 510 | title: Test 511 | status: test 512 | logsource: 513 | category: test_category 514 | product: test_product 515 | detection: 516 | sel: 517 | fieldA|contains: ooba 518 | condition: sel 519 | """ 520 | ) 521 | ) == ['{job=~".+"} |~ `(?i).*ooba.*` | logfmt | fieldA=~`(?i).*ooba.*`'] 522 | 523 | 524 | def test_loki_lf_cidr_query(loki_backend: LogQLBackend): 525 | assert loki_backend.convert( 526 | SigmaCollection.from_yaml( 527 | """ 528 | title: Test 529 | status: test 530 | logsource: 531 | category: test_category 532 | product: test_product 533 | detection: 534 | sel: 535 | fieldA|cidr: 192.168.0.0/16 536 | condition: sel 537 | """ 538 | ) 539 | ) == ['{job=~".+"} |= ip("192.168.0.0/16") | logfmt | fieldA=ip("192.168.0.0/16")'] 540 | 541 | 542 | def test_loki_lf_not_cidr_query(loki_backend: LogQLBackend): 543 | assert loki_backend.convert( 544 | SigmaCollection.from_yaml( 545 | """ 546 | title: Test 547 | status: test 548 | logsource: 549 | category: test_category 550 | product: test_product 551 | detection: 552 | sel: 553 | fieldA|cidr: 192.168.0.0/16 554 | condition: not sel 555 | """ 556 | ) 557 | ) == ['{job=~".+"} != ip("192.168.0.0/16") | logfmt | fieldA!=ip("192.168.0.0/16")'] 558 | 559 | 560 | def test_loki_lf_base64_query(loki_backend: LogQLBackend): 561 | assert loki_backend.convert( 562 | SigmaCollection.from_yaml( 563 | """ 564 | title: Test 565 | status: test 566 | logsource: 567 | category: test_category 568 | product: test_product 569 | detection: 570 | sel: 571 | fieldA|base64: value 572 | condition: sel 573 | """ 574 | ) 575 | ) == ['{job=~".+"} |= `fieldA=dmFsdWU=` | logfmt | fieldA=`dmFsdWU=`'] 576 | 577 | 578 | def test_loki_lf_base64offset_query(loki_backend: LogQLBackend): 579 | assert loki_backend.convert( 580 | SigmaCollection.from_yaml( 581 | """ 582 | title: Test 583 | status: test 584 | logsource: 585 | category: test_category 586 | product: test_product 587 | detection: 588 | sel: 589 | fieldA|base64offset: value 590 | condition: sel 591 | """ 592 | ) 593 | ) == ['{job=~".+"} | logfmt | fieldA=`dmFsdW` or fieldA=`ZhbHVl` or fieldA=`2YWx1Z`'] 594 | 595 | 596 | def test_loki_lf_field_name_with_whitespace(loki_backend: LogQLBackend): 597 | assert loki_backend.convert( 598 | SigmaCollection.from_yaml( 599 | """ 600 | title: Test 601 | status: test 602 | logsource: 603 | category: test_category 604 | product: test_product 605 | detection: 606 | sel: 607 | field name: value 608 | condition: sel 609 | """ 610 | ) 611 | ) == ['{job=~".+"} |= `field name=value` | logfmt | field_name=`value`'] 612 | 613 | 614 | def test_loki_lf_field_name_leading_num(loki_backend: LogQLBackend): 615 | assert loki_backend.convert( 616 | SigmaCollection.from_yaml( 617 | """ 618 | title: Test 619 | status: test 620 | logsource: 621 | category: test_category 622 | product: test_product 623 | detection: 624 | sel: 625 | 0field: value 626 | condition: sel 627 | """ 628 | ) 629 | ) == ['{job=~".+"} |= `0field=value` | logfmt | _0field=`value`'] 630 | 631 | 632 | def test_loki_lf_field_name_invalid(loki_backend: LogQLBackend): 633 | assert loki_backend.convert( 634 | SigmaCollection.from_yaml( 635 | """ 636 | title: Test 637 | status: test 638 | logsource: 639 | category: test_category 640 | product: test_product 641 | detection: 642 | sel: 643 | field.name@A-Z: value 644 | condition: sel 645 | """ 646 | ) 647 | ) == ['{job=~".+"} |= `field.name@A-Z=value` | logfmt | field_name_A_Z=`value`'] 648 | 649 | 650 | # Ensure existing line filters prevent addition of new ones 651 | def test_loki_lf_unbound(loki_backend: LogQLBackend): 652 | assert loki_backend.convert( 653 | SigmaCollection.from_yaml( 654 | """ 655 | title: test 656 | status: test 657 | logsource: 658 | category: test_category 659 | product: test_product 660 | detection: 661 | keywords: 662 | value 663 | sel: 664 | fieldA: valueA 665 | condition: keywords and sel 666 | """ 667 | ) 668 | ) == ['{job=~".+"} |= `value` | logfmt | fieldA=`valueA`'] 669 | 670 | 671 | def test_loki_lf_and_unbound(loki_backend: LogQLBackend): 672 | assert loki_backend.convert( 673 | SigmaCollection.from_yaml( 674 | """ 675 | title: test 676 | status: test 677 | logsource: 678 | category: test_category 679 | product: test_product 680 | detection: 681 | keyword1: 682 | valueA 683 | keyword2: 684 | valueB 685 | sel: 686 | fieldA: valueA 687 | condition: keyword1 and keyword2 and sel 688 | """ 689 | ) 690 | ) == ['{job=~".+"} |= `valueA` |= `valueB` | logfmt | fieldA=`valueA`'] 691 | 692 | 693 | def test_loki_lf_or_unbound(loki_backend: LogQLBackend): 694 | assert loki_backend.convert( 695 | SigmaCollection.from_yaml( 696 | """ 697 | title: Test 698 | status: test 699 | logsource: 700 | category: test_category 701 | product: test_product 702 | detection: 703 | keywords: 704 | - valueA 705 | - valueB 706 | sel: 707 | fieldA: valueA 708 | condition: keywords and sel 709 | """ 710 | ) 711 | ) == ['{job=~".+"} |~ `valueA|valueB` | logfmt | fieldA=`valueA`'] 712 | 713 | 714 | # Testing specific logsources and other Sigma features 715 | def test_loki_lf_windows_logsource(loki_backend: LogQLBackend): 716 | assert loki_backend.convert( 717 | SigmaCollection.from_yaml( 718 | """ 719 | title: Test 720 | status: test 721 | logsource: 722 | category: test_category 723 | product: windows 724 | detection: 725 | sel: 726 | key1.key2: value 727 | condition: sel 728 | """ 729 | ) 730 | ) == ['{job=~"eventlog|winlog|windows|fluentbit.*"} |= `value` | json | key1_key2=`value`'] 731 | 732 | 733 | def test_loki_lf_azure_logsource(loki_backend: LogQLBackend): 734 | assert loki_backend.convert( 735 | SigmaCollection.from_yaml( 736 | """ 737 | title: Test 738 | status: test 739 | logsource: 740 | category: test_category 741 | product: azure 742 | detection: 743 | sel: 744 | key1.key2: value 745 | condition: sel 746 | """ 747 | ) 748 | ) == ['{job="logstash"} |= `value` | json | key1_key2=`value`'] 749 | 750 | 751 | def test_loki_lf_very_long_query_or(loki_backend: LogQLBackend): 752 | long_field = f"longField{'A' * 50}" 753 | yaml = ( 754 | f""" 755 | title: Test 756 | status: test 757 | logsource: 758 | category: test_category 759 | product: test_product 760 | detection: 761 | sel_each: 762 | {long_field}: valueA 763 | sel_partitioned: 764 | longFieldB|contains: 765 | """ 766 | + "\n".join( 767 | " - " + "".join(random.choices(string.ascii_letters, k=50)) 768 | for _ in range(100) 769 | ) 770 | + """ 771 | condition: all of sel_* 772 | """ 773 | ) 774 | test = loki_backend.convert(SigmaCollection.from_yaml(yaml)) 775 | assert ( 776 | len(test) > 1 777 | and all(f"{long_field}=`valueA`" in q for q in test) 778 | and all(f"|= `{long_field}=valueA` |" in q for q in test) 779 | and all(len(q) < 5120 for q in test) 780 | ) 781 | 782 | 783 | def test_loki_lf_very_long_query_or_right_filters(loki_backend: LogQLBackend): 784 | yaml = f""" 785 | title: Test 786 | status: test 787 | logsource: 788 | category: test_category 789 | product: test_product 790 | detection: 791 | sel_A: 792 | fieldA: value{"A" * 4000} 793 | sel_B: 794 | fieldB: value{"B" * 4000} 795 | condition: 1 of sel_* 796 | """ 797 | test = loki_backend.convert(SigmaCollection.from_yaml(yaml)) 798 | assert len(test) > 1 and all( 799 | re.match("|= `field([AB])=value\1{4000}` | .* | field\1=`value\1{4000}`", q) for q in test 800 | ) 801 | -------------------------------------------------------------------------------- /tests/test_pipelines_loki.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sigma.exceptions import SigmaFeatureNotSupportedByBackendError 3 | 4 | from sigma.backends.loki import LogQLBackend 5 | from sigma.collection import SigmaCollection 6 | from sigma.processing.transformations import transformations, FieldMappingTransformation 7 | from sigma.processing.pipeline import ProcessingItem, ProcessingPipeline 8 | from sigma.pipelines.loki import ( 9 | LokiCustomAttributes, 10 | SetCustomAttributeTransformation, 11 | CustomLogSourceTransformation, 12 | loki_grafana_logfmt, 13 | loki_promtail_sysmon, 14 | loki_okta_system_log, 15 | ) 16 | 17 | 18 | @pytest.fixture 19 | def sigma_rules() -> SigmaCollection: 20 | return SigmaCollection.from_yaml( 21 | """ 22 | title: Test 23 | status: test 24 | logsource: 25 | product: test 26 | service: test 27 | detection: 28 | sel: 29 | msg: testing 30 | condition: sel 31 | """ 32 | ) 33 | 34 | 35 | def test_transformations_contains_custom_attribute(): 36 | assert "set_custom_attribute" in transformations 37 | assert transformations["set_custom_attribute"] == SetCustomAttributeTransformation 38 | 39 | 40 | def test_loki_grafana_pipeline(): 41 | pipeline = loki_grafana_logfmt() 42 | backend = LogQLBackend(processing_pipeline=pipeline) 43 | sigma_rule = SigmaCollection.from_yaml( 44 | """ 45 | title: Test 46 | status: test 47 | logsource: 48 | product: test 49 | service: test 50 | detection: 51 | sel-path: 52 | - c-uri: /a/path/to/something 53 | - cs-uri-query: /a/different/path 54 | sel: 55 | sc-status: 200 56 | condition: all of sel* 57 | """ 58 | ) 59 | loki_rule = backend.convert(sigma_rule) 60 | assert loki_rule == [ 61 | '{job=~".+"} | logfmt | (path=~`(?i)^/a/path/to/something$`' 62 | " or path=~`(?i)^/a/different/path$`) and status=200" 63 | ] 64 | 65 | 66 | def test_windows_grafana_pipeline(): 67 | pipeline = loki_promtail_sysmon() 68 | backend = LogQLBackend(processing_pipeline=pipeline) 69 | sigma_rule = SigmaCollection.from_yaml( 70 | """ 71 | title: Test 72 | status: test 73 | logsource: 74 | product: windows 75 | service: sysmon 76 | detection: 77 | sel: 78 | Image|endswith: .exe 79 | EventID: 1 80 | condition: sel 81 | """ 82 | ) 83 | loki_rule = backend.convert(sigma_rule) 84 | assert loki_rule == [ 85 | '{job=~"eventlog|winlog|windows|fluentbit.*"} | json ' 86 | '| label_format Message=`{{ .message | replace "\\\\" "\\\\\\\\" | replace "\\"" "\\\\\\"" }}` ' # noqa: E501 87 | '| line_format `{{ regexReplaceAll "([^:]+): ?((?:[^\\\\r]*|$))(\\r\\n|$)" .Message "${1}=\\"${2}\\" "}}` ' # noqa: E501 88 | "| logfmt | Image=~`(?i).*\\.exe$` and event_id=1" 89 | ] 90 | 91 | 92 | def test_okta_json_pipeline(): 93 | pipeline = loki_okta_system_log() 94 | backend = LogQLBackend(processing_pipeline=pipeline) 95 | sigma_rule = SigmaCollection.from_yaml( 96 | """ 97 | title: Test 98 | status: test 99 | logsource: 100 | product: okta 101 | service: okta 102 | detection: 103 | sel: 104 | eventType: 105 | - policy.lifecycle.update 106 | - policy.lifecycle.delete 107 | legacyeventtype: 'core.user_auth.login_failed' 108 | displaymessage: 'Failed login to Okta' 109 | condition: sel 110 | """ 111 | ) 112 | loki_rule = backend.convert(sigma_rule) 113 | assert loki_rule == [ 114 | '{job=~".+"} | json | (event_eventType=~`(?i)^policy\\.lifecycle\\.update$` or ' 115 | "event_eventType=~`(?i)^policy\\.lifecycle\\.delete$`) and " 116 | "event_legacyEventType=~`(?i)^core\\.user_auth\\.login_failed$` and " 117 | "event_displayMessage=~`(?i)^Failed\\ login\\ to\\ Okta$`" 118 | ] 119 | 120 | 121 | def test_okta_json_pipeline_exclusive_exhaustive(): 122 | pipeline = loki_okta_system_log() 123 | backend = LogQLBackend(processing_pipeline=pipeline) 124 | 125 | boilerplate = """ 126 | title: Test 127 | status: test 128 | logsource: 129 | product: okta 130 | service: okta 131 | detection: 132 | sel: 133 | {fieldName}: test_value 134 | condition: sel 135 | """ 136 | 137 | rules = [ 138 | ("eventtype", ['{job=~".+"} | json | event_eventType=~`(?i)^test_value$`']), 139 | ( 140 | "legacyeventtype", 141 | ['{job=~".+"} | json | event_legacyEventType=~`(?i)^test_value$`'], 142 | ), 143 | ( 144 | "actor.alternateid", 145 | ['{job=~".+"} | json | event_actor_alternateId=~`(?i)^test_value$`'], 146 | ), 147 | ( 148 | "actor.displayname", 149 | ['{job=~".+"} | json | event_actor_displayName=~`(?i)^test_value$`'], 150 | ), 151 | ( 152 | "client.useragent.rawuseragent", 153 | ['{job=~".+"} | json | event_client_userAgent_rawUserAgent=~`(?i)^test_value$`'], 154 | ), 155 | ( 156 | "client.useragent.os", 157 | ['{job=~".+"} | json | event_client_userAgent_os=~`(?i)^test_value$`'], 158 | ), 159 | ( 160 | "client.useragent.browser", 161 | ['{job=~".+"} | json | event_client_userAgent_browser=~`(?i)^test_value$`'], 162 | ), 163 | ( 164 | "client.geographicalcontext.geolocation.lat", 165 | [ 166 | '{job=~".+"} | json | event_client_geographicalContext_geolocation_lat=~`(?i)^test_' 167 | "value$`" 168 | ], 169 | ), 170 | ( 171 | "client.geographicalcontext.geolocation.lon", 172 | [ 173 | '{job=~".+"} | json | event_client_geographicalContext_geolocation_lon=~`(?i)^test_' 174 | "value$`" 175 | ], 176 | ), 177 | ( 178 | "client.geographicalcontext.city", 179 | ['{job=~".+"} | json | event_client_geographicalContext_city=~`(?i)^test_value$`'], 180 | ), 181 | ( 182 | "client.geographicalcontext.state", 183 | ['{job=~".+"} | json | event_client_geographicalContext_state=~`(?i)^test_value$`'], 184 | ), 185 | ( 186 | "client.geographicalcontext.country", 187 | ['{job=~".+"} | json | event_client_geographicalContext_country=~`(?i)^test_value$`'], 188 | ), 189 | ( 190 | "client.geographicalcontext.postalcode", 191 | [ 192 | '{job=~".+"} | json | event_client_geographicalContext_postalCode=~`(?i)^test_' 193 | "value$`" 194 | ], 195 | ), 196 | ( 197 | "client.ipaddress", 198 | ['{job=~".+"} | json | event_client_ipAddress=~`(?i)^test_value$`'], 199 | ), 200 | ( 201 | "debugcontext.debugdata.requesturi", 202 | ['{job=~".+"} | json | event_debugContext_debugData_requestUri=~`(?i)^test_value$`'], 203 | ), 204 | ( 205 | "debugcontext.debugdata.originalprincipal.id", 206 | [ 207 | '{job=~".+"} | json | event_debugContext_debugData_originalPrincipal_id=~`(?i)^test' 208 | "_value$`" 209 | ], 210 | ), 211 | ( 212 | "debugcontext.debugdata.originalprincipal.type", 213 | [ 214 | '{job=~".+"} | json | event_debugContext_debugData_originalPrincipal_type=~`(?i)' 215 | "^test_value$`" 216 | ], 217 | ), 218 | ( 219 | "debugcontext.debugdata.originalprincipal.alternateid", 220 | [ 221 | '{job=~".+"} | json | event_debugContext_debugData_originalPrincipal_alternateId=~`' 222 | "(?i)^test_value$`" 223 | ], 224 | ), 225 | ( 226 | "debugcontext.debugdata.originalprincipal.displayname", 227 | [ 228 | '{job=~".+"} | json | event_debugContext_debugData_originalPrincipal_displayName=~`' 229 | "(?i)^test_value$`" 230 | ], 231 | ), 232 | ( 233 | "debugcontext.debugdata.behaviors", 234 | ['{job=~".+"} | json | event_debugContext_debugData_behaviors=~`(?i)^test_value$`'], 235 | ), 236 | ( 237 | "debugcontext.debugdata.logonlysecuritydata", 238 | [ 239 | '{job=~".+"} | json | event_debugContext_debugData_logOnlySecurityData=~`(?i)^test_' 240 | "value$`" 241 | ], 242 | ), 243 | ( 244 | "authenticationcontext.authenticationprovider", 245 | [ 246 | '{job=~".+"} | json | event_authenticationContext_authenticationProvider=~`(?i)' 247 | "^test_value$`" 248 | ], 249 | ), 250 | ( 251 | "authenticationcontext.authenticationstep", 252 | [ 253 | '{job=~".+"} | json | event_authenticationContext_authenticationStep=~`(?i)^test_' 254 | "value$`" 255 | ], 256 | ), 257 | ( 258 | "authenticationcontext.credentialprovider", 259 | [ 260 | '{job=~".+"} | json | event_authenticationContext_credentialProvider=~`(?i)^test_' 261 | "value$`" 262 | ], 263 | ), 264 | ( 265 | "authenticationcontext.credentialtype", 266 | ['{job=~".+"} | json | event_authenticationContext_credentialType=~`(?i)^test_value$`'], 267 | ), 268 | ( 269 | "authenticationcontext.issuer.id", 270 | ['{job=~".+"} | json | event_authenticationContext_issuer_id=~`(?i)^test_value$`'], 271 | ), 272 | ( 273 | "authenticationcontext.issuer.type", 274 | ['{job=~".+"} | json | event_authenticationContext_issuer_type=~`(?i)^test_value$`'], 275 | ), 276 | ( 277 | "authenticationcontext.externalsessionid", 278 | [ 279 | '{job=~".+"} | json | event_authenticationContext_externalSessionId=~`(?i)^test_' 280 | "value$`" 281 | ], 282 | ), 283 | ( 284 | "authenticationcontext.interface", 285 | ['{job=~".+"} | json | event_authenticationContext_interface=~`(?i)^test_value$`'], 286 | ), 287 | ( 288 | "securitycontext.asnumber", 289 | ['{job=~".+"} | json | event_securityContext_asNumber=~`(?i)^test_value$`'], 290 | ), 291 | ( 292 | "securitycontext.asorg", 293 | ['{job=~".+"} | json | event_securityContext_asOrg=~`(?i)^test_value$`'], 294 | ), 295 | ( 296 | "securitycontext.isp", 297 | ['{job=~".+"} | json | event_securityContext_isp=~`(?i)^test_value$`'], 298 | ), 299 | ( 300 | "securitycontext.domain", 301 | ['{job=~".+"} | json | event_securityContext_domain=~`(?i)^test_value$`'], 302 | ), 303 | ( 304 | "securitycontext.isproxy", 305 | ['{job=~".+"} | json | event_securityContext_isProxy=~`(?i)^test_value$`'], 306 | ), 307 | ( 308 | "target.alternateid", 309 | ['{job=~".+"} | json | event_target_alternateId=~`(?i)^test_value$`'], 310 | ), 311 | ( 312 | "target.displayname", 313 | ['{job=~".+"} | json | event_target_displayName=~`(?i)^test_value$`'], 314 | ), 315 | ( 316 | "target.detailentry", 317 | ['{job=~".+"} | json | event_target_detailEntry=~`(?i)^test_value$`'], 318 | ), 319 | ] 320 | 321 | for rule in rules: 322 | sigma_rule = SigmaCollection.from_yaml(boilerplate.format(fieldName=rule[0])) 323 | loki_rule = backend.convert(sigma_rule) 324 | assert loki_rule == rule[1] 325 | 326 | 327 | def test_loki_parser_pipeline(sigma_rules: SigmaCollection): 328 | pipeline = ProcessingPipeline( 329 | name="Test custom Loki parser pipeline", 330 | priority=20, 331 | items=[ 332 | ProcessingItem( 333 | identifier="set_loki_parser_pattern", 334 | transformation=SetCustomAttributeTransformation( 335 | attribute=LokiCustomAttributes.PARSER.value, 336 | value="pattern ` `", 337 | ), 338 | ) 339 | ], 340 | ) 341 | backend = LogQLBackend(processing_pipeline=pipeline) 342 | loki_rule = backend.convert(sigma_rules) 343 | assert loki_rule == ['{job=~".+"} | pattern ` ` | msg=~`(?i)^testing$`'] 344 | 345 | 346 | def test_loki_logsource_selection_pipeline(sigma_rules: SigmaCollection): 347 | pipeline = ProcessingPipeline( 348 | name="Test custom Loki logsource pipeline", 349 | priority=20, 350 | items=[ 351 | ProcessingItem( 352 | identifier="set_loki_logsource_selection", 353 | transformation=SetCustomAttributeTransformation( 354 | attribute=LokiCustomAttributes.LOGSOURCE_SELECTION.value, 355 | value="{job=`mylogs`,filename=~`.*[\\d]+.log$`}", 356 | ), 357 | ) 358 | ], 359 | ) 360 | backend = LogQLBackend(processing_pipeline=pipeline) 361 | loki_rule = backend.convert(sigma_rules) 362 | assert loki_rule == ["{job=`mylogs`,filename=~`.*[\\d]+.log$`} | logfmt | msg=~`(?i)^testing$`"] 363 | 364 | 365 | def test_single_custom_log_source_pipeline(sigma_rules: SigmaCollection): 366 | pipeline = ProcessingPipeline( 367 | name="Test custom Loki logsource pipeline", 368 | priority=20, 369 | items=[ 370 | ProcessingItem( 371 | identifier="complex_custom_log_source", 372 | transformation=CustomLogSourceTransformation( 373 | selection={ 374 | "message|fieldref": "msg", 375 | }, 376 | template=True, 377 | ), 378 | ) 379 | ], 380 | ) 381 | backend = LogQLBackend(processing_pipeline=pipeline) 382 | loki_rule = backend.convert(sigma_rules) 383 | assert loki_rule == ["{message=`testing`} | logfmt | msg=~`(?i)^testing$`"] 384 | 385 | 386 | def test_simple_custom_log_source_pipeline(sigma_rules: SigmaCollection): 387 | pipeline = ProcessingPipeline( 388 | name="Test custom Loki logsource pipeline", 389 | priority=20, 390 | items=[ 391 | ProcessingItem( 392 | identifier="complex_custom_log_source", 393 | transformation=CustomLogSourceTransformation( 394 | selection={ 395 | "job": ["a", "b", "c"], 396 | "message|fieldref": "msg", 397 | "env": "$product", 398 | }, 399 | template=True, 400 | ), 401 | ) 402 | ], 403 | ) 404 | backend = LogQLBackend(processing_pipeline=pipeline) 405 | loki_rule = backend.convert(sigma_rules) 406 | assert loki_rule == [ 407 | "{job=~`a|b|c`,message=`testing`,env=`test`} | logfmt | msg=~`(?i)^testing$`" 408 | ] 409 | 410 | 411 | def test_field_renamed_custom_log_source_pipeline(sigma_rules: SigmaCollection): 412 | pipeline = ProcessingPipeline( 413 | name="Test custom Loki logsource pipeline", 414 | priority=20, 415 | items=[ 416 | ProcessingItem( 417 | identifier="update_msg_field_name", 418 | transformation=FieldMappingTransformation( 419 | mapping={ 420 | "msg": "message", 421 | } 422 | ), 423 | ), 424 | ProcessingItem( 425 | identifier="complex_custom_log_source", 426 | transformation=CustomLogSourceTransformation( 427 | selection={"msg_lbl|fieldref": "message"} 428 | ), 429 | ), 430 | ], 431 | ) 432 | backend = LogQLBackend(processing_pipeline=pipeline) 433 | loki_rule = backend.convert(sigma_rules) 434 | assert loki_rule == ["{msg_lbl=`testing`} | logfmt | message=~`(?i)^testing$`"] 435 | 436 | 437 | def test_multiple_custom_log_source_pipeline(sigma_rules: SigmaCollection): 438 | pipeline = ProcessingPipeline( 439 | name="Test custom Loki logsource pipeline", 440 | priority=20, 441 | items=[ 442 | ProcessingItem( 443 | identifier="complex_custom_log_source", 444 | transformation=CustomLogSourceTransformation( 445 | selection={ 446 | "name|re": "okta.logs", 447 | "job|contains": "secops", 448 | "eventType|fieldref": "eventType", 449 | }, 450 | template=True, 451 | ), 452 | ) 453 | ], 454 | ) 455 | backend = LogQLBackend(processing_pipeline=pipeline) 456 | sigma_rule = SigmaCollection.from_yaml( 457 | """ 458 | title: Test 459 | status: test 460 | logsource: 461 | product: okta 462 | service: okta 463 | detection: 464 | sel1: 465 | eventType: 466 | - policy.lifecycle.update 467 | - policy.lifecycle.del* 468 | legacyeventtype: 'core.user_auth.login_failed' 469 | displaymessage: 'Failed login to Okta' 470 | sel2: 471 | eventType|re: 'policy\\.life.*\\.' 472 | condition: all of sel* 473 | """ 474 | ) 475 | loki_rule = backend.convert(sigma_rule) 476 | assert loki_rule == [ 477 | "{name=~`okta.logs`,job=~`.*secops.*`," 478 | "eventType=~`policy\\.life.*\\.|policy\\.lifecycle\\.update|policy\\.lifecycle\\.del.*`} " 479 | "| logfmt | (eventType=~`(?i)^policy\\.lifecycle\\.update$` " 480 | "or eventType=~`(?i)^policy\\.lifecycle\\.del.*`) " 481 | "and legacyeventtype=~`(?i)^core\\.user_auth\\.login_failed$` " 482 | "and displaymessage=~`(?i)^Failed\\ login\\ to\\ Okta$` " 483 | "and eventType=~`policy\\.life.*\\.`" 484 | ] 485 | 486 | 487 | def test_custom_log_source_rule_with_keywords(): 488 | pipeline = ProcessingPipeline( 489 | name="Test custom Loki logsource pipeline", 490 | priority=20, 491 | items=[ 492 | ProcessingItem( 493 | identifier="complex_custom_log_source", 494 | transformation=CustomLogSourceTransformation( 495 | selection={ 496 | "job": ["a", "b", "c"], 497 | "message|fieldref": "msg", 498 | "env": "$product", 499 | }, 500 | template=True, 501 | ), 502 | ) 503 | ], 504 | ) 505 | backend = LogQLBackend(processing_pipeline=pipeline) 506 | sigma_rule = SigmaCollection.from_yaml( 507 | """ 508 | title: Test 509 | status: test 510 | logsource: 511 | product: okta 512 | service: okta 513 | detection: 514 | sel: 515 | eventType: 516 | - policy.lifecycle.update 517 | - policy.lifecycle.del* 518 | legacyeventtype: 'core.user_auth.login_failed' 519 | msg: 'Failed login to Okta' 520 | keywords: 521 | - 'policy\\.life.*\\.' 522 | condition: sel and keywords 523 | """ 524 | ) 525 | loki_rule = backend.convert(sigma_rule) 526 | assert loki_rule[0].startswith("{job=~`a|b|c`,message=`Failed login to Okta`,env=`okta`}") 527 | 528 | 529 | def test_skip_both_negated_and_positive_custom_log_source_pipeline( 530 | sigma_rules: SigmaCollection, 531 | ): 532 | pipeline = ProcessingPipeline( 533 | name="Test custom Loki logsource pipeline", 534 | priority=20, 535 | items=[ 536 | ProcessingItem( 537 | identifier="complex_custom_log_source", 538 | transformation=CustomLogSourceTransformation( 539 | selection={ 540 | "name": "okta-logs", 541 | "eventType|fieldref": "eventType", 542 | } 543 | ), 544 | ) 545 | ], 546 | ) 547 | backend = LogQLBackend(processing_pipeline=pipeline) 548 | sigma_rule = SigmaCollection.from_yaml( 549 | """ 550 | title: Test 551 | status: test 552 | logsource: 553 | product: okta 554 | service: okta 555 | detection: 556 | sel1: 557 | eventType|startswith: policy.lifecycle. 558 | sel2: 559 | eventType|endswith: create 560 | condition: sel1 and not sel2 561 | """ 562 | ) 563 | loki_rule = backend.convert(sigma_rule) 564 | assert loki_rule == [ 565 | "{name=`okta-logs`} | logfmt | eventType=~`(?i)^policy\\.lifecycle\\..*` " 566 | "and eventType!~`(?i).*create$`" 567 | ] 568 | 569 | 570 | def test_negated_custom_log_source_pipeline(sigma_rules: SigmaCollection): 571 | pipeline = ProcessingPipeline( 572 | name="Test custom Loki logsource pipeline", 573 | priority=20, 574 | items=[ 575 | ProcessingItem( 576 | identifier="complex_custom_log_source", 577 | transformation=CustomLogSourceTransformation( 578 | selection={ 579 | "eventType|fieldref": "eventType", 580 | "stream|fieldref": "ruleField", 581 | }, 582 | template=True, 583 | ), 584 | ) 585 | ], 586 | ) 587 | backend = LogQLBackend(processing_pipeline=pipeline) 588 | sigma_rule = SigmaCollection.from_yaml( 589 | """ 590 | title: Test 591 | status: test 592 | logsource: 593 | product: okta 594 | service: okta 595 | detection: 596 | sel1: 597 | legacyeventtype: 'core.user_auth.login_failed' 598 | displaymessage: 'Failed login to Okta' 599 | sel2: 600 | eventType: 'policy.lifecycle.update' 601 | ruleField|re: '.*out' 602 | condition: sel1 and not sel2 603 | """ 604 | ) 605 | loki_rule = backend.convert(sigma_rule) 606 | assert loki_rule == [ 607 | "{eventType!=`policy.lifecycle.update`,stream!~`.*out`} " 608 | "| logfmt | legacyeventtype=~`(?i)^core\\.user_auth\\.login_failed$` " 609 | "and displaymessage=~`(?i)^Failed\\ login\\ to\\ Okta$` " 610 | "and (eventType!~`(?i)^policy\\.lifecycle\\.update$` " 611 | "or ruleField!~`.*out`)" 612 | ] 613 | 614 | 615 | def test_unsupported_line_filter_custom_log_source_pipeline( 616 | sigma_rules: SigmaCollection, 617 | ): 618 | pipeline = ProcessingPipeline( 619 | name="Test custom Loki logsource pipeline", 620 | priority=20, 621 | items=[ 622 | ProcessingItem( 623 | identifier="invalid_custom_log_source", 624 | transformation=CustomLogSourceTransformation( 625 | selection={ 626 | "": "value", 627 | } 628 | ), 629 | ) 630 | ], 631 | ) 632 | backend = LogQLBackend(processing_pipeline=pipeline) 633 | raised_error = False 634 | try: 635 | backend.convert(sigma_rules) 636 | except Exception as e: 637 | raised_error = True 638 | assert isinstance(e, SigmaFeatureNotSupportedByBackendError) 639 | assert "only supports field equals value conditions" in str(e) 640 | assert raised_error 641 | 642 | 643 | def test_unsupported_nested_or_custom_log_source_pipeline(sigma_rules: SigmaCollection): 644 | pipeline = ProcessingPipeline( 645 | name="Test custom Loki logsource pipeline", 646 | priority=20, 647 | items=[ 648 | ProcessingItem( 649 | identifier="invalid_custom_log_source", 650 | transformation=CustomLogSourceTransformation( 651 | selection={ 652 | "test|all": ["a", "b", "c"], 653 | } 654 | ), 655 | ) 656 | ], 657 | ) 658 | backend = LogQLBackend(processing_pipeline=pipeline) 659 | raised_error = False 660 | try: 661 | backend.convert(sigma_rules) 662 | except Exception as e: 663 | raised_error = True 664 | assert isinstance(e, SigmaFeatureNotSupportedByBackendError) 665 | assert "allows one required value for a field" in str(e) 666 | assert raised_error 667 | 668 | 669 | def test_unsupported_filter_custom_log_source_pipeline(sigma_rules: SigmaCollection): 670 | pipeline = ProcessingPipeline( 671 | name="Test custom Loki logsource pipeline", 672 | priority=20, 673 | items=[ 674 | ProcessingItem( 675 | identifier="invalid_custom_log_source", 676 | transformation=CustomLogSourceTransformation( 677 | selection={ 678 | "test|cidr": "1.2.0.0/16", 679 | } 680 | ), 681 | ) 682 | ], 683 | ) 684 | backend = LogQLBackend(processing_pipeline=pipeline) 685 | raised_error = False 686 | try: 687 | backend.convert(sigma_rules) 688 | except Exception as e: 689 | raised_error = True 690 | assert isinstance(e, SigmaFeatureNotSupportedByBackendError) 691 | assert "only supports: string values, field references and regular expressions" in str(e) 692 | assert raised_error 693 | 694 | 695 | def test_processing_pipeline_custom_attribute_from_dict(): 696 | pipeline_dict = { 697 | "name": "Testing Custom Pipeline", 698 | "vars": {}, 699 | "priority": 10, 700 | "transformations": [ 701 | { 702 | "id": "custom_pipeline_dict", 703 | "type": "set_custom_attribute", 704 | "rule_conditions": [{"type": "logsource", "category": "web"}], 705 | "detection_item_conditions": [ 706 | {"type": "match_string", "cond": "any", "pattern": "test"} 707 | ], 708 | "field_name_conditions": [ 709 | {"type": "include_fields", "fields": ["fieldA", "fieldB"]} 710 | ], 711 | "rule_cond_op": "or", 712 | "detection_item_cond_op": "or", 713 | "field_name_cond_op": "or", 714 | "rule_cond_not": True, 715 | "detection_item_cond_not": True, 716 | "field_name_cond_not": True, 717 | "attribute": "loki_parser", 718 | "value": "json", 719 | } 720 | ], 721 | } 722 | processing_pipeline = ProcessingPipeline.from_dict(pipeline_dict) 723 | assert processing_pipeline is not None 724 | assert processing_pipeline.priority == int(pipeline_dict["priority"]) 725 | assert len(processing_pipeline.items) == len(pipeline_dict["transformations"]) 726 | processing_item = processing_pipeline.items[0] 727 | assert processing_item is not None 728 | assert processing_item.transformation is not None 729 | assert isinstance(processing_item.transformation, SetCustomAttributeTransformation) 730 | assert ( 731 | processing_item.transformation.attribute == pipeline_dict["transformations"][0]["attribute"] 732 | ) 733 | assert processing_item.transformation.value == pipeline_dict["transformations"][0]["value"] 734 | 735 | 736 | def test_set_custom_attribute_correlation_rule(): 737 | """ 738 | Test that the custom attribute transformation can be applied to a correlation rule. 739 | """ 740 | sigma_rules = SigmaCollection.from_yaml( 741 | """ 742 | title: Test 743 | name: failed_login 744 | status: test 745 | logsource: 746 | product: okta 747 | service: okta 748 | detection: 749 | sel: 750 | legacyeventtype: 'core.user_auth.login_failed' 751 | displaymessage: 'Failed login to Okta' 752 | condition: sel 753 | --- 754 | title: Valid correlation 755 | status: test 756 | correlation: 757 | type: event_count 758 | rules: failed_login 759 | group-by: actor.alternateid 760 | timespan: 10m 761 | condition: 762 | gte: 10 763 | """ 764 | ) 765 | pipeline = ProcessingPipeline( 766 | name="Test custom Loki logsource pipeline", 767 | priority=20, 768 | items=[ 769 | ProcessingItem( 770 | identifier="set_loki_logsource_selection", 771 | transformation=SetCustomAttributeTransformation( 772 | attribute=LokiCustomAttributes.LOGSOURCE_SELECTION.value, 773 | value="{job=`mylogs`,filename=~`.*[\\d]+.log$`}", 774 | ), 775 | ) 776 | ], 777 | ) 778 | backend = LogQLBackend(processing_pipeline=pipeline) 779 | loki_rule = backend.convert(sigma_rules) 780 | assert len(loki_rule) == 1 781 | assert "job=`mylogs`" in loki_rule[0] 782 | assert "filename=~`.*[\\d]+.log$`" in loki_rule[0] 783 | 784 | 785 | def test_set_custom_log_source_correlation_rule(): 786 | """ 787 | Test that the custom log source transformation can be applied to a correlation rule. 788 | """ 789 | sigma_rules = SigmaCollection.from_yaml( 790 | """ 791 | title: Test 792 | name: failed_login 793 | status: test 794 | logsource: 795 | product: okta 796 | service: okta 797 | detection: 798 | sel: 799 | legacyeventtype: 'core.user_auth.login_failed' 800 | displaymessage: 'Failed login to Okta' 801 | condition: sel 802 | --- 803 | title: Valid correlation 804 | status: test 805 | correlation: 806 | type: event_count 807 | rules: failed_login 808 | group-by: actor.alternateid 809 | timespan: 10m 810 | condition: 811 | gte: 10 812 | """ 813 | ) 814 | pipeline = ProcessingPipeline( 815 | name="Test custom Loki logsource pipeline", 816 | priority=20, 817 | items=[ 818 | ProcessingItem( 819 | identifier="set_loki_custom_logsource_selection", 820 | transformation=CustomLogSourceTransformation( 821 | selection={ 822 | "name": "okta-logs", 823 | "eventType|fieldref": "legacyeventtype", 824 | } 825 | ), 826 | ) 827 | ], 828 | ) 829 | backend = LogQLBackend(processing_pipeline=pipeline) 830 | loki_rule = backend.convert(sigma_rules) 831 | assert len(loki_rule) == 1 832 | assert loki_rule[0].startswith("sum by (actor_alternateid)") 833 | assert "name=`okta-logs`" in loki_rule[0] 834 | assert "eventType=`core.user_auth.login_failed`" in loki_rule[0] 835 | --------------------------------------------------------------------------------