├── .github ├── scripts │ └── kql_query_executor │ │ ├── requirements-dev.txt │ │ ├── requirements.txt │ │ ├── utils.py │ │ ├── main.py │ │ ├── model.py │ │ ├── execute.py │ │ ├── tests │ │ ├── test_model.py │ │ ├── test_config.py │ │ └── test_execute.py │ │ └── config.py ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── workflows │ └── execute-queries.yaml └── pull_request_template.md ├── library └── device │ ├── network-events.kql │ ├── process-events.kql │ └── .kql-config.yaml ├── .pre-commit-config.yaml ├── LICENSE ├── CODE_OF_CONDUCT.md ├── kql-config-schema.json ├── CONTRIBUTING.md ├── .gitignore └── README.md /.github/scripts/kql_query_executor/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest>=8.0, <9.0 2 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml>=6.0 2 | jsonschema>=4.0.0 3 | -------------------------------------------------------------------------------- /library/device/network-events.kql: -------------------------------------------------------------------------------- 1 | // Get the latest network events 2 | // Timeframe: Last 24 hours 3 | DeviceNetworkEvents 4 | | where TimeGenerated > ago(24h) 5 | | project 6 | TimeGenerated, 7 | DeviceName, 8 | ActionType, 9 | RemoteIP, 10 | RemotePort, 11 | Protocol 12 | | order by TimeGenerated desc 13 | | take 10 14 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def setup_logging(level: int) -> None: 5 | """Configure logging with the specified level.""" 6 | log_format = "%(message)s" 7 | if level == logging.DEBUG: 8 | log_format = "%(levelname)s: %(message)s" 9 | 10 | handlers = [logging.StreamHandler()] 11 | logging.basicConfig(level=level, format=log_format, handlers=handlers) 12 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | 10 | - repo: https://github.com/astral-sh/ruff-pre-commit 11 | rev: v0.2.1 12 | hooks: 13 | - id: ruff 14 | args: [--fix] 15 | - id: ruff-format 16 | -------------------------------------------------------------------------------- /library/device/process-events.kql: -------------------------------------------------------------------------------- 1 | // Process Events Analysis 2 | // Analyze process creation patterns and suspicious activities 3 | // Timeframe: Last 24 hours 4 | DeviceProcessEvents 5 | | where TimeGenerated > ago(24h) 6 | | where InitiatingProcessFileName in~ ("cmd.exe", "powershell.exe", "pwsh.exe", "bash", "zsh") 7 | | project 8 | TimeGenerated, 9 | DeviceName, 10 | AccountName, 11 | InitiatingProcessFileName, // Parent process 12 | FileName, // Created process 13 | ProcessCommandLine, // Command line arguments 14 | FolderPath, // Process location 15 | InitiatingProcessCommandLine // Parent command line 16 | | where ProcessCommandLine !has "Get-" // Filter out common Get- commands 17 | and ProcessCommandLine !has "Set-" // Filter out common Set- commands 18 | | extend ProcessType = case( 19 | InitiatingProcessFileName has "powershell", "PowerShell", 20 | InitiatingProcessFileName has "cmd", "Command Prompt", 21 | InitiatingProcessFileName in~ ("bash", "zsh"), "Shell", 22 | "Other" 23 | ) 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Christos Galanopoulos, Michalis Michalos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEAT]" 5 | labels: enhancement 6 | assignees: christosgalano, cyb3rmik3 7 | --- 8 | 9 | 10 | 11 | ## Detailed Description 12 | 13 | 14 | ## Context 15 | 16 | 17 | 18 | ## Use Cases 19 | 20 | 21 | 1. 22 | 2. 23 | 3. 24 | 25 | ## Possible Implementation 26 | 27 | 28 | ## Example Configuration 29 | 30 | ```yaml 31 | # Example configuration showing the proposed feature 32 | ``` 33 | 34 | ## Alternatives Considered 35 | 36 | 37 | 38 | ## Additional Context 39 | 40 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: christosgalano, cyb3rmik3 7 | --- 8 | 9 | 10 | 11 | ## Description 12 | 13 | 14 | ## Expected Behavior 15 | 16 | 17 | ## Actual Behavior 18 | 19 | 20 | ## Steps to Reproduce 21 | 22 | 1. 23 | 2. 24 | 3. 25 | 4. 26 | 27 | ## Environment Information 28 | 29 | - Repository version/branch: 30 | - Operating System (local or GitHub Actions runner): 31 | - Python version: 32 | - Azure CLI version: 33 | - Log Analytics Workspace details: 34 | 35 | ## Related KQL Query 36 | 37 | ```kql 38 | // Your KQL query here 39 | ``` 40 | 41 | ## Configuration File 42 | 43 | ```yaml 44 | # Your configuration file here 45 | ``` 46 | 47 | ## Error Output 48 | 49 | ```text 50 | # Error output here 51 | ``` 52 | 53 | ## Screenshots 54 | 55 | 56 | ## Possible Fix 57 | 58 | 59 | ## Additional Context 60 | 61 | 62 | -------------------------------------------------------------------------------- /.github/workflows/execute-queries.yaml: -------------------------------------------------------------------------------- 1 | name: execute-queries 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | folder: 7 | description: 'Query folder to execute (relative to library)' 8 | required: true 9 | type: string 10 | 11 | jobs: 12 | execute: 13 | name: Execute queries in ${{ github.event.inputs.folder }} 14 | runs-on: ubuntu-latest 15 | permissions: 16 | id-token: write 17 | contents: read 18 | env: 19 | folder: library/${{ github.event.inputs.folder }} 20 | steps: 21 | - name: Checkout repository 22 | uses: actions/checkout@v4 23 | 24 | - name: Validate folder existence 25 | run: | 26 | if [ ! -d ${{ env.folder }} ]; then 27 | echo "Folder ${{ env.folder }} does not exist" 28 | exit 1 29 | fi 30 | 31 | - name: Azure login 32 | uses: azure/login@v2 33 | with: 34 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 35 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 36 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 37 | 38 | - name: Install log-analytics extension 39 | run: | 40 | az config set extension.dynamic_install_allow_preview=true 41 | az extension add --name log-analytics 42 | az extension update --name log-analytics 43 | 44 | - name: Set up Python 45 | uses: actions/setup-python@v5 46 | with: 47 | python-version: '3.12' 48 | cache: 'pip' 49 | 50 | - name: Install Python dependencies 51 | run: | 52 | python -m pip install --upgrade pip 53 | pip install -r .github/scripts/kql_query_executor/requirements.txt 54 | 55 | - name: Execute queries 56 | run: | 57 | python .github/scripts/kql_query_executor/main.py \ 58 | --folder ${{ env.folder }} \ 59 | --workspace-id ${{ secrets.WORKSPACE_ID }} \ 60 | --schema kql-config-schema.json 61 | 62 | - name: Upload results 63 | uses: actions/upload-artifact@v4 64 | with: 65 | name: Query-Results 66 | path: query-results 67 | -------------------------------------------------------------------------------- /library/device/.kql-config.yaml: -------------------------------------------------------------------------------- 1 | version: '1.0' 2 | 3 | queries: 4 | - file: 'process-events.kql' 5 | output: 6 | # Simplified console output with key process info 7 | - format: table 8 | query: >- 9 | [].{ 10 | Time: TimeGenerated, 11 | Device: DeviceName, 12 | Account: AccountName, 13 | ParentProcess: InitiatingProcessFileName, 14 | CreatedProcess: FileName, 15 | CommandLine: ProcessCommandLine 16 | }[:5] 17 | 18 | # Summarized JSON output by process type 19 | - format: json 20 | file: query-results/process-events/by_type.json 21 | query: >- 22 | sort_by([].{ 23 | processType: ProcessType, 24 | device: DeviceName, 25 | user: AccountName, 26 | parent: InitiatingProcessFileName, 27 | created: FileName, 28 | cmdline: ProcessCommandLine 29 | }, &processType) 30 | 31 | - file: 'network-events.kql' 32 | output: 33 | # Simplified network connections table 34 | - format: table 35 | query: >- 36 | reverse(sort_by([].{ 37 | Time: TimeGenerated, 38 | Action: ActionType, 39 | Endpoint: join(':', [to_string(RemoteIP), to_string(RemotePort)]) 40 | }, &Time))[:5] 41 | 42 | # Basic network events JSON 43 | - format: json 44 | file: query-results/network-events/connections.json 45 | query: >- 46 | reverse(sort_by([].{ 47 | timestamp: TimeGenerated, 48 | device: DeviceName, 49 | action: ActionType, 50 | protocol: Protocol, 51 | remote_ip: to_string(RemoteIP), 52 | remote_port: to_string(RemotePort), 53 | endpoint: join(':', [to_string(RemoteIP), to_string(RemotePort)]), 54 | summary: join(' ', [ 55 | 'Device:', DeviceName, 56 | 'Action:', ActionType, 57 | 'Protocol:', Protocol, 58 | 'Endpoint:', join(':', [to_string(RemoteIP), to_string(RemotePort)]) 59 | ]) 60 | }, ×tamp)) 61 | compression: gzip 62 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 6 | ## Related Issue 7 | 8 | 9 | 10 | 11 | 12 | ## Motivation and Context 13 | 14 | 15 | ## How Has This Been Tested? 16 | 17 | 18 | 19 | 20 | ## Screenshots (if appropriate) 21 | 22 | 23 | ## Types of changes 24 | 25 | - [ ] Bug fix (non-breaking change which fixes an issue) 26 | - [ ] New query (adds a new KQL query file) 27 | - [ ] Query modification (non-breaking changes to existing queries) 28 | - [ ] Configuration update (changes to .kql-config.yaml files) 29 | - [ ] Documentation improvement 30 | - [ ] Infrastructure change (GitHub Actions, script modifications) 31 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 32 | 33 | ## Checklist 34 | 35 | 36 | - [ ] My code follows the code style of this project. 37 | - [ ] My KQL queries are properly documented with comments. 38 | - [ ] My query configurations follow the established patterns. 39 | - [ ] My change requires a change to the documentation. 40 | - [ ] I have updated the documentation accordingly. 41 | - [ ] I have read the [**CONTRIBUTING**](/CONTRIBUTING.md) document. 42 | - [ ] I have tested my queries against a Log Analytics workspace. 43 | - [ ] I have verified my JMESPath transformations work as expected. 44 | - [ ] All new and existing tests passed. 45 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | We are committed to providing a friendly, safe, and welcoming environment for all contributors and participants in the sKaleQL project. By participating in this project, you agree to abide by the following code of conduct: 4 | 5 | ## Expected Behavior 6 | 7 | - **Be respectful:** Treat everyone with respect and kindness. We embrace diversity and value different perspectives. 8 | 9 | - **Be inclusive:** Welcome and support individuals of all backgrounds and identities. Create an environment where everyone feels comfortable and can actively participate. 10 | 11 | - **Be collaborative:** Foster collaboration and cooperation among contributors. Emphasize teamwork and help each other to achieve the best possible outcomes. 12 | 13 | ## Unacceptable Behavior 14 | 15 | The following behaviors are considered unacceptable within the project community: 16 | 17 | - **Harassment:** Any form of harassment, offensive comments, or personal attacks. 18 | 19 | - **Discrimination:** Discrimination based on race, gender, sexual orientation, disability, age, religion, or any other protected characteristic. 20 | 21 | - **Disruptive Behavior:** Intentionally disruptive behavior that interferes with the project's progress or creates a hostile environment. 22 | 23 | - **Violation of Privacy:** Sharing personal information of others without their explicit consent. 24 | 25 | ## Reporting and Enforcement 26 | 27 | If you encounter any violations of the code of conduct or witness any unacceptable behavior, please report it to the project maintainers by contacting [christosgalano](https://github.com/christosgalano) or [cyb3rmik3](https://github.com/cyb3rmik3). All reports will be reviewed and investigated promptly, and appropriate actions will be taken as necessary. 28 | 29 | ## Consequences 30 | 31 | Unacceptable behavior will not be tolerated. Anyone engaged in such behavior may be subject to consequences, including but not limited to warnings, temporary or permanent bans, or removal from the project community. 32 | 33 | ## Scope 34 | 35 | This code of conduct applies to all project-related spaces, including but not limited to GitHub repositories, issue trackers, communication channels, and project events. 36 | 37 | ## Acknowledgment 38 | 39 | We appreciate your commitment to creating a positive and inclusive community. By contributing to the sKaleQL project, you are helping us maintain a welcoming environment for everyone. 40 | 41 | Thank you for your cooperation and understanding. 42 | -------------------------------------------------------------------------------- /kql-config-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "https://raw.githubusercontent.com/christosgalano/kql-template-repo/main/kql-config-schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "type": "object", 5 | "title": "KQL Query Execution Configuration", 6 | "description": "Defines execution rules for KQL queries, including output settings.", 7 | "properties": { 8 | "version": { 9 | "type": "string", 10 | "description": "Configuration schema version." 11 | }, 12 | "queries": { 13 | "type": "array", 14 | "description": "List of query configurations. By default, all KQL files will be executed; specific configurations override defaults.", 15 | "items": { 16 | "$ref": "#/definitions/queryConfig" 17 | }, 18 | "uniqueItems": true 19 | } 20 | }, 21 | "definitions": { 22 | "queryConfig": { 23 | "type": "object", 24 | "required": [ 25 | "file" 26 | ], 27 | "properties": { 28 | "file": { 29 | "type": "string", 30 | "description": "Path to the .kql file containing the query. Relative to the configuration file." 31 | }, 32 | "output": { 33 | "type": "array", 34 | "description": "Output formats for this query.", 35 | "items": { 36 | "$ref": "#/definitions/outputFormat" 37 | }, 38 | "uniqueItems": true 39 | } 40 | }, 41 | "additionalProperties": false 42 | }, 43 | "outputFormat": { 44 | "type": "object", 45 | "properties": { 46 | "format": { 47 | "type": "string", 48 | "enum": [ 49 | "json", 50 | "jsonc", 51 | "none", 52 | "table", 53 | "tsv", 54 | "yaml", 55 | "yamlc" 56 | ], 57 | "description": "Output format." 58 | }, 59 | "query": { 60 | "type": "string", 61 | "description": "JMESPath query string to apply. See http://jmespath.org/ for more information and examples." 62 | }, 63 | "file": { 64 | "type": "string", 65 | "description": "If specified output is written to this file. Directories are created if they do not exist. Relative to the configuration file." 66 | }, 67 | "compression": { 68 | "type": "string", 69 | "enum": [ 70 | "gzip", 71 | "zip" 72 | ], 73 | "description": "Compression format to use. If not specified, no compression is used." 74 | } 75 | }, 76 | "additionalProperties": false, 77 | "required": [ 78 | "format" 79 | ] 80 | } 81 | }, 82 | "additionalProperties": false 83 | } 84 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import logging 5 | import os 6 | import sys 7 | 8 | from config import find_config_file, get_applicable_files, load_config 9 | from execute import execute_query 10 | from model import KQLConfig 11 | from utils import setup_logging 12 | 13 | 14 | def main() -> None: 15 | """ 16 | Execute KQL queries from a folder based on configuration. 17 | 18 | By default, all KQL files in the specified folder are executed with JSON output to stdout. 19 | If a configuration file is found, query-specific settings are applied. 20 | """ 21 | parser = argparse.ArgumentParser(description="Execute KQL queries from a folder") 22 | parser.add_argument( 23 | "-f", "--folder", required=True, help="Path to the folder containing KQL files" 24 | ) 25 | parser.add_argument( 26 | "-w", "--workspace-id", required=True, help="Azure Log Analytics workspace ID" 27 | ) 28 | parser.add_argument( 29 | "-c", 30 | "--config", 31 | help="Path to the config file (default: look in folder then repo root)", 32 | ) 33 | parser.add_argument( 34 | "-s", 35 | "--schema", 36 | help="Path to the schema file (default: kql-config-schema.json in repo root)", 37 | ) 38 | parser.add_argument( 39 | "-l", 40 | "--log-level", 41 | choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], 42 | default="ERROR", 43 | help="Set logging level", 44 | ) 45 | 46 | args = parser.parse_args() 47 | setup_logging(getattr(logging, args.log_level)) 48 | 49 | # Validate folder exists 50 | folder_path = args.folder 51 | if not os.path.isdir(folder_path): 52 | logging.error(f"Folder {folder_path} does not exist") 53 | sys.exit(1) 54 | 55 | # Load configuration 56 | config_path = args.config or find_config_file(folder_path) 57 | if config_path: 58 | logging.info(f"Using config file: {config_path}") 59 | config = load_config(config_path, args.schema) 60 | else: 61 | logging.info("No config file found, using default JSON output for all queries") 62 | config = KQLConfig() # Default config with JSON output to stdout 63 | 64 | # Find all KQL files in the folder 65 | applicable_files = get_applicable_files(folder_path, config) 66 | if not applicable_files: 67 | logging.info("No KQL files found in the specified folder") 68 | sys.exit(0) 69 | 70 | logging.info(f"Found {len(applicable_files)} KQL file(s) to execute") 71 | 72 | # Execute each query 73 | success_count, fail_count = 0, 0 74 | for file_name in applicable_files: 75 | if execute_query(folder_path, file_name, args.workspace_id, config): 76 | success_count += 1 77 | else: 78 | fail_count += 1 79 | 80 | # Report results 81 | logging.info(f"Execution completed: {success_count} succeeded, {fail_count} failed") 82 | if fail_count > 0: 83 | sys.exit(1) 84 | 85 | 86 | if __name__ == "__main__": 87 | main() 88 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to kql-template-repo 2 | 3 | Thank you for considering contributing to kql-template-repo! We welcome your input and appreciate your efforts to improve this resource. To ensure a smooth and collaborative experience, please review the guidelines below before making any contributions. 4 | 5 | ## Ways to Contribute 6 | 7 | There are several ways you can contribute to kql-template-repo: 8 | 9 | - **Feedback:** Share your thoughts, suggestions, or bug reports by opening an issue in the repository. We value your feedback and will carefully consider any ideas that can enhance the template. 10 | 11 | - **Documentation:** Help us improve the existing documentation or add new sections to provide more clarity and guidance to users. Feel free to open a pull request with your proposed changes. 12 | 13 | - **Feature Contributions:** If you have an idea for a new feature or enhancement, please open an issue to discuss it first. This allows us to evaluate the feasibility and alignment with the repository's goals before proceeding with implementation. 14 | 15 | - **Bug Fixes:** If you discover a bug or issue in the template repository, please open an issue describing the problem and, if possible, provide steps to reproduce it. We appreciate any bug fixes in the form of pull requests. 16 | 17 | ## Contribution Guidelines 18 | 19 | To ensure a productive and inclusive environment, please adhere to the following guidelines: 20 | 21 | - **Be respectful:** Treat everyone with respect and kindness. We embrace diversity and value different perspectives. 22 | 23 | - **Follow coding standards:** If you contribute code changes, maintain a consistent coding style and adhere to any existing conventions. 24 | 25 | - **Provide clear and concise explanations:** When opening an issue or submitting a pull request, be explicit about the problem or proposed changes. This helps us understand your intentions better. 26 | 27 | - **Test your changes:** If applicable, include tests with your code changes to ensure they work as expected. 28 | 29 | - **Give credit:** If you include any external resources or references in your contributions, provide appropriate attribution and give credit to the original authors. 30 | 31 | ## Getting Started 32 | 33 | To start contributing, follow these steps: 34 | 35 | 1. Fork the repository to your GitHub account. 36 | 2. Create a new branch for your contribution. 37 | 3. Make your changes or additions. 38 | 4. Test your changes locally to ensure they function correctly. 39 | 5. Commit your changes and push them to your forked repository. 40 | 6. Open a pull request in this repository, explaining your changes and their purpose. 41 | 7. Wait for a review and collaborate with the maintainers to address any feedback. 42 | 8. Once your contribution is accepted, it will be merged into the main repository. Thank you for your contribution! 43 | 9. By participating in this project, you agree to abide by the [Code of Conduct](CODE_OF_CONDUCT.md). 44 | 10. We appreciate your dedication to improving kql-template-repo. 45 | 46 | If you have any questions or need further assistance, feel free to reach out to us. 47 | 48 | ## Reporting Bugs & Requesting Features 49 | 50 | We use GitHub issue templates to standardize reporting and ensure faster response times. 51 | 52 | - For bugs, [follow this guide](https://github.com/christosgalano/kql-template-repo/wiki/Feedback#bug-report-template) 53 | - For new ideas or enhancements, [use the feature request template](https://github.com/christosgalano/kql-template-repo/wiki/Feedback#feature-request-template) 54 | 55 | Before submitting, please check if the issue already exists. 56 | 57 | Happy contributing! 58 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/model.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | 4 | 5 | class OutputFormat(str, Enum): 6 | """ 7 | Enumeration of supported output types for KQL query results. 8 | 9 | Attributes: 10 | NONE: No output 11 | JSON: JSON output 12 | JSONC: JSON output with colorization 13 | TABLE: Tabular output 14 | TSV: Tab-separated values 15 | YAML: YAML output 16 | YAMLC: YAML output with color 17 | """ 18 | 19 | NONE = "none" 20 | JSON = "json" 21 | JSONC = "jsonc" 22 | TABLE = "table" 23 | TSV = "tsv" 24 | YAML = "yaml" 25 | YAMLC = "yamlc" 26 | 27 | def extension(self) -> str: 28 | """Returns the file extension for the output format.""" 29 | return { 30 | OutputFormat.NONE: "", 31 | OutputFormat.JSON: ".json", 32 | OutputFormat.JSONC: ".json", 33 | OutputFormat.TABLE: ".txt", 34 | OutputFormat.TSV: ".tsv", 35 | OutputFormat.YAML: ".yaml", 36 | OutputFormat.YAMLC: ".yaml", 37 | }.get(self, "") 38 | 39 | 40 | class CompressionType(str, Enum): 41 | """ 42 | Enumeration of supported compression formats for file outputs. 43 | 44 | Attributes: 45 | GZIP: GZip compression (.gz) 46 | ZIP: Zip archive (.zip) 47 | """ 48 | 49 | GZIP = "gzip" 50 | ZIP = "zip" 51 | 52 | 53 | @dataclass 54 | class OutputConfig: 55 | """ 56 | Configuration for an individual output format. 57 | 58 | Attributes: 59 | format: Output destination type (console or file) 60 | query: JMESPath query string to apply. See http://jmespath.org/ for more information and examples. 61 | file: Output file. If not specified, print to stdout. 62 | compression: Compression format for file outputs. 63 | """ 64 | 65 | format: OutputFormat 66 | query: str | None = None 67 | file: str | None = None 68 | compression: CompressionType | None = None 69 | 70 | def __post_init__(self) -> None: 71 | # Reject whitespace in filenames 72 | if self.file and any(c.isspace() for c in self.file): 73 | raise ValueError( 74 | f"Filename should not contain whitespace: '{self.file}'. " 75 | f"Use underscores or dashes instead." 76 | ) 77 | 78 | 79 | @dataclass 80 | class QueryConfig: 81 | """ 82 | Configuration for an individual KQL query. 83 | 84 | Attributes: 85 | file: Path to the KQL file containing the query 86 | output: Query-specific output configurations (overrides defaults) 87 | """ 88 | 89 | file: str 90 | output: list[OutputConfig] | None = None 91 | 92 | def __post_init__(self) -> None: 93 | # Verify file ends with ".kql" 94 | if not self.file.endswith(".kql"): 95 | raise ValueError(f"Query file must end with .kql: {self.file}") 96 | 97 | # Check for whitespace in file path 98 | if any(c.isspace() for c in self.file): 99 | raise ValueError( 100 | f"Query file path should not contain whitespace: '{self.file}'. " 101 | f"Use underscores or dashes instead." 102 | ) 103 | 104 | 105 | @dataclass 106 | class KQLConfig: 107 | """ 108 | Root configuration for KQL query execution. 109 | 110 | Attributes: 111 | version: Schema version string 112 | queries: List of specific query configurations 113 | """ 114 | 115 | version: str = "1.0" 116 | queries: list[QueryConfig] = None 117 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # General 2 | .vscode/ 3 | .env 4 | test.sh 5 | prompt.txt 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # UV 104 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | #uv.lock 108 | 109 | # poetry 110 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 111 | # This is especially recommended for binary packages to ensure reproducibility, and is more 112 | # commonly ignored for libraries. 113 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 114 | #poetry.lock 115 | 116 | # pdm 117 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 118 | #pdm.lock 119 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 120 | # in version control. 121 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 122 | .pdm.toml 123 | .pdm-python 124 | .pdm-build/ 125 | 126 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 127 | __pypackages__/ 128 | 129 | # Celery stuff 130 | celerybeat-schedule 131 | celerybeat.pid 132 | 133 | # SageMath parsed files 134 | *.sage.py 135 | 136 | # Environments 137 | .env 138 | .venv 139 | env/ 140 | venv/ 141 | ENV/ 142 | env.bak/ 143 | venv.bak/ 144 | 145 | # Spyder project settings 146 | .spyderproject 147 | .spyproject 148 | 149 | # Rope project settings 150 | .ropeproject 151 | 152 | # mkdocs documentation 153 | /site 154 | 155 | # mypy 156 | .mypy_cache/ 157 | .dmypy.json 158 | dmypy.json 159 | 160 | # Pyre type checker 161 | .pyre/ 162 | 163 | # pytype static type analyzer 164 | .pytype/ 165 | 166 | # Cython debug symbols 167 | cython_debug/ 168 | 169 | # Ruff stuff: 170 | .ruff_cache/ 171 | 172 | # PyPI configuration file 173 | .pypirc 174 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/execute.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import logging 3 | import os 4 | import shutil 5 | import subprocess 6 | from pathlib import Path 7 | 8 | from config import get_output_configs_for_query 9 | from model import CompressionType, KQLConfig, OutputFormat 10 | 11 | 12 | def execute_query( 13 | folder_path: str, query_file: str, workspace_id: str, config: KQLConfig 14 | ) -> bool: # noqa: C901 15 | """Execute a KQL query and process the results.""" 16 | # Convert folder_path to Path object 17 | base_path = Path(folder_path) 18 | query_path = base_path / query_file 19 | 20 | # Verify query file exists 21 | if not query_path.exists(): 22 | logging.error(f"Query file not found: {query_path}") 23 | return False 24 | 25 | logging.info(f"Executing {query_path}...") 26 | 27 | # Get output configurations for this query 28 | output_configs = get_output_configs_for_query(config, query_file) 29 | if not output_configs: 30 | logging.error(f"No output configuration found for query: {query_path}") 31 | return False 32 | 33 | try: 34 | # Process each output format 35 | for fmt in output_configs: 36 | # Skip processing if format is NONE 37 | if fmt.format == OutputFormat.NONE: 38 | logging.info(f"Skipping output for {query_path} (format: none)") 39 | continue 40 | 41 | # Build Azure CLI command with output format 42 | cmd = [ 43 | "az", 44 | "monitor", 45 | "log-analytics", 46 | "query", 47 | "-w", 48 | workspace_id, 49 | "--analytics-query", 50 | f"@{query_path}", 51 | "--output", 52 | fmt.format.value, # Use format enum value directly 53 | ] 54 | 55 | # Add JMESPath query if specified 56 | if fmt.query: 57 | # Clean up the query string: 58 | # 1. Remove newlines and extra spaces 59 | # 2. Escape any existing quotes 60 | # 3. Wrap in quotes if needed 61 | clean_query = fmt.query.strip().replace("\n", " ").replace(" ", " ") 62 | cmd.extend(["--query", clean_query]) 63 | 64 | # Execute the command 65 | result = subprocess.run(cmd, capture_output=True, text=True, check=True) # noqa: S603 66 | filtered_result = result.stdout.strip() 67 | 68 | # Process the results 69 | if not fmt.file: 70 | # Display to console 71 | print(f"Results for {query_file}") 72 | print("-" * 80) 73 | print(filtered_result) 74 | print(end="\n\n") 75 | logging.info(f"Results for {query_file} displayed to console") 76 | continue 77 | 78 | # Process file output 79 | output_file = fmt.file 80 | 81 | # Ensure output directory exists 82 | os.makedirs(os.path.dirname(output_file), exist_ok=True) 83 | 84 | # Write the results to file 85 | with open(output_file, "w") as f: 86 | f.write(filtered_result) 87 | logging.info(f"Results saved to {output_file}") 88 | 89 | # Apply compression if specified 90 | if fmt.compression: 91 | if fmt.compression == CompressionType.GZIP: 92 | with ( 93 | open(output_file, "rb") as f_in, 94 | gzip.open(f"{output_file}.gz", "wb") as f_out, 95 | ): 96 | shutil.copyfileobj(f_in, f_out) 97 | os.remove(output_file) 98 | logging.info(f"Compressed results with gzip: {output_file}.gz") 99 | elif fmt.compression == CompressionType.ZIP: 100 | zip_base = os.path.splitext(output_file)[0] 101 | shutil.make_archive( 102 | zip_base, 103 | "zip", 104 | root_dir=os.path.dirname(output_file), 105 | base_dir=os.path.basename(output_file), 106 | ) 107 | os.remove(output_file) 108 | logging.info(f"Compressed results with zip: {zip_base}.zip") 109 | 110 | return True 111 | 112 | except subprocess.CalledProcessError as e: 113 | logging.error(f"Error executing query {query_path}: {e}") 114 | if hasattr(e, "stderr") and e.stderr: 115 | logging.error(f"Error details: {e.stderr}") 116 | return False 117 | except Exception as e: 118 | logging.error(f"Unexpected error processing {query_path}: {e}") 119 | return False 120 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/tests/test_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | 5 | # Add parent directory to path so we can import modules 6 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 7 | 8 | import pytest 9 | from model import CompressionType, KQLConfig, OutputConfig, OutputFormat, QueryConfig 10 | 11 | 12 | def test_output_format_enum(): 13 | """Test OutputFormat enumeration values.""" 14 | assert OutputFormat.NONE == "none" 15 | assert OutputFormat.JSON == "json" 16 | assert OutputFormat.JSONC == "jsonc" 17 | assert OutputFormat.TABLE == "table" 18 | assert OutputFormat.TSV == "tsv" 19 | assert OutputFormat.YAML == "yaml" 20 | assert OutputFormat.YAMLC == "yamlc" 21 | 22 | 23 | def test_output_format_extensions(): 24 | """Test OutputFormat file extensions.""" 25 | assert OutputFormat.NONE.extension() == "" 26 | assert OutputFormat.JSON.extension() == ".json" 27 | assert OutputFormat.JSONC.extension() == ".json" 28 | assert OutputFormat.TABLE.extension() == ".txt" 29 | assert OutputFormat.TSV.extension() == ".tsv" 30 | assert OutputFormat.YAML.extension() == ".yaml" 31 | assert OutputFormat.YAMLC.extension() == ".yaml" 32 | 33 | 34 | def test_compression_type_enum(): 35 | """Test CompressionType enumeration values.""" 36 | assert CompressionType.GZIP == "gzip" 37 | assert CompressionType.ZIP == "zip" 38 | 39 | 40 | def test_output_config_defaults(): 41 | """Test OutputConfig default values.""" 42 | config = OutputConfig(format=OutputFormat.JSON) 43 | assert config.format == OutputFormat.JSON 44 | assert config.query is None 45 | assert config.file is None 46 | assert config.compression is None 47 | 48 | 49 | def test_output_config_with_query(): 50 | """Test OutputConfig with JMESPath query.""" 51 | config = OutputConfig( 52 | format=OutputFormat.JSON, 53 | query="[].{File: FileName, PID: ProcessId, Command: ProcessCommandLine}", 54 | ) 55 | assert config.format == OutputFormat.JSON 56 | assert ( 57 | config.query 58 | == "[].{File: FileName, PID: ProcessId, Command: ProcessCommandLine}" 59 | ) 60 | 61 | 62 | def test_output_config_with_file(): 63 | """Test OutputConfig with file.""" 64 | config = OutputConfig( 65 | format=OutputFormat.JSON, 66 | file="results/output.json", 67 | ) 68 | assert config.format == OutputFormat.JSON 69 | assert config.file == "results/output.json" 70 | 71 | 72 | def test_output_config_with_compression(): 73 | """Test OutputConfig with compression.""" 74 | config = OutputConfig( 75 | format=OutputFormat.JSON, 76 | file="results/output.json", 77 | compression=CompressionType.GZIP, 78 | ) 79 | assert config.format == OutputFormat.JSON 80 | assert config.compression == CompressionType.GZIP 81 | 82 | 83 | def test_output_config_whitespace_in_file(): 84 | """Test that OutputConfig rejects files with whitespace.""" 85 | expected_message = "Filename should not contain whitespace: 'output file.json'. Use underscores or dashes instead." 86 | with pytest.raises(ValueError, match=re.escape(expected_message)): 87 | OutputConfig(format=OutputFormat.JSON, file="output file.json") 88 | 89 | 90 | def test_query_config_basic(): 91 | """Test QueryConfig with basic properties.""" 92 | config = QueryConfig(file="query.kql") 93 | assert config.file == "query.kql" 94 | assert config.output is None 95 | 96 | 97 | def test_query_config_with_outputs(): 98 | """Test QueryConfig with output configurations.""" 99 | outputs = [ 100 | OutputConfig(format=OutputFormat.JSON), 101 | OutputConfig(format=OutputFormat.YAML, file="results/output.yaml"), 102 | ] 103 | config = QueryConfig(file="query.kql", output=outputs) 104 | assert config.file == "query.kql" 105 | assert len(config.output) == 2 106 | assert config.output[0].format == OutputFormat.JSON 107 | assert config.output[1].format == OutputFormat.YAML 108 | assert config.output[1].file == "results/output.yaml" 109 | 110 | 111 | def test_kql_config_defaults(): 112 | """Test KQLConfig default values.""" 113 | config = KQLConfig() 114 | assert config.version == "1.0" 115 | assert config.queries is None 116 | 117 | 118 | def test_kql_config_with_queries(): 119 | """Test KQLConfig with query configurations.""" 120 | queries = [ 121 | QueryConfig(file="query1.kql"), 122 | QueryConfig(file="query2.kql"), 123 | ] 124 | config = KQLConfig(version="1.0", queries=queries) 125 | assert config.version == "1.0" 126 | assert len(config.queries) == 2 127 | assert config.queries[0].file == "query1.kql" 128 | assert config.queries[1].file == "query2.kql" 129 | 130 | 131 | def test_kql_config_invalid_query_extension(): 132 | """Test KQLConfig validation for query file extension.""" 133 | with pytest.raises(ValueError, match="Query file must end with .kql"): 134 | KQLConfig(version="1.0", queries=[QueryConfig(file="query.txt")]) 135 | 136 | 137 | def test_query_config_whitespace_in_file(): 138 | """Test that QueryConfig rejects file paths with whitespace.""" 139 | with pytest.raises( 140 | ValueError, match="Query file path should not contain whitespace" 141 | ): 142 | QueryConfig(file="query file.kql") 143 | 144 | 145 | def test_query_config_valid_file_formats(): 146 | """Test that QueryConfig accepts valid file paths.""" 147 | # Underscore instead of space 148 | config1 = QueryConfig(file="query_file.kql") 149 | assert config1.file == "query_file.kql" 150 | 151 | # Nested path with underscores 152 | config2 = QueryConfig(file="my_folder/query.kql") 153 | assert config2.file == "my_folder/query.kql" 154 | 155 | # Dash instead of space 156 | config3 = QueryConfig(file="query-file.kql") 157 | assert config3.file == "query-file.kql" 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sKaleQL 2 | 3 | ![Badge](https://img.shields.io/badge/Microsoft%20Azure-4169E1) 4 | ![Badge](https://img.shields.io/badge/Log%20Analytics%20Workspace-87CEEB) 5 | ![Badge](https://img.shields.io/badge/Kusto%20Query%20Language-5C2D91) 6 | ![Badge](https://img.shields.io/badge/GitHub%20Actions-000000) 7 | 8 | ```text 9 | 10 | ,--, 11 | ,--. ,---.'| 12 | ,--/ /| ,--, ,----.. | | : 13 | ,---,': / ' ,--.'| / / \ : : | 14 | : : '/ / | | : / . : | ' : 15 | .--.--. | ' , : : ' . / ;. \ ; ; ' 16 | / / ' ' | / ,--.--. | ' | ,---. . ; / ` ; ' | |__ 17 | | : /`./ | ; ; / \ ' | | / \ ; | ; \ ; | | | :.'| 18 | | : ;_ : ' \ .--. .-. | | | : / / | | : | ; | ' ' : ; 19 | \ \ `. | | ' \__\/: . . ' : |__ . ' / | . | ' ' ' : | | ./ 20 | `----. \ ' : |. \ ," .--.; | | | '.'| ' ; /| ' ; \; / | ; : ; 21 | / /`--' / | | '_\.' / / ,. | ; : ; ' | / | \ \ ', . \ | ,/ 22 | '--'. / ' : | ; : .' \ | , / | : | ; : ; | '---' 23 | `--'---' ; |,' | , .-./ ---`-' \ \ / \ \ .'`--" 24 | '---' `--`---' `----' `---` 25 | ``` 26 | 27 | > 🚀 **Automate and run KQL queries at scale, using GitHub Actions.** 28 | 29 | ## Table of Contents 30 | 31 | - [Introduction](#introduction) 32 | - [What is sKaleQL](#what-is-skaleql) 33 | - [Why should you use sKaleQL](#why-should-you-use-skaleql) 34 | - [Features](#features) 35 | - [Documentation](#documentation) 36 | - [Getting Started](#getting-started) 37 | - [1. Use This Template](#1-use-this-template) 38 | - [2. Azure Setup](#2-azure-setup) 39 | - [3. GitHub Configuration](#3-github-configuration) 40 | - [4. Execute Queries](#4-execute-queries) 41 | - [Contributing](#contributing) 42 | - [License](#license) 43 | 44 | > [!TIP] 45 | > Detailed documentation can be found in the corresponding [wiki page](https://github.com/christosgalano/kql-template-repo/wiki). 46 | 47 | ## Introduction 48 | 49 | ### What is sKaleQL 50 | 51 | **sKaleQL** is a comprehensive template repository for managing, executing, and organizing Kusto Query Language (KQL) queries against Azure Log Analytics Workspaces. It provides a structured approach to query management with flexible output formats, automated execution, and comprehensive documentation. 52 | 53 | ### Why should you use sKaleQL 54 | 55 | There is really no limit to how you can use this tool, it all depends on why you're using Log Analytics Workspaces in the first place. Whether it's for security, monitoring, auditing, or something else entirely, if you have a set of KQL queries you want to automate, **sKaleQL** is here to help. By combining the power of KQL with GitHub Actions, you can bring automation and efficiency into your workflows effortlessly. 56 | 57 | Here are just a few examples of how sKaleQL can make a difference: 58 | 59 | - Automated Health Checks: Regularly query for service errors, performance issues, or failed logins. 60 | - Security and Threat Monitoring: Run scheduled KQL queries to detect anomalies, threats, or suspicious activity. 61 | - Compliance Validation: Ensure logs reflect adherence to security and compliance standards. 62 | - Reporting and Data Export: Generate and store logs or metrics as artifacts for analysis. 63 | - Cost and Usage Monitoring: Track ingestion rates and resource usage for optimization. 64 | - Incident Response Automation: Pre-build incident queries to speed up investigations. 65 | - Incident Retrospectives: Pull relevant logs automatically after an incident for analysis and RCA (Root Cause Analysis). 66 | - Business Metrics Tracking: Monitor and extract business-level KPIs (e.g., signups, payments, errors) if logged to Azure Monitor. 67 | - Resource Inventory Tracking: Automatically query and export lists of resources (VMs, Containers, Storage) for auditing purposes. 68 | 69 | ## Features 70 | 71 | - **Structured Query Management**: Organize queries in logical folders 72 | - **Flexible Output Formats**: JSON, Table, TSV, YAML, and more 73 | - **Multiple Output Destinations**: Console display or file output 74 | - **Advanced Transformations**: Filter results using JMESPath queries 75 | - **Compression Options**: Optimize storage with GZIP or ZIP compression 76 | - **Automation**: GitHub Actions workflow for scheduled query execution 77 | - **Local Execution**: Run queries from your development environment 78 | 79 | ## Documentation 80 | 81 | For detailed information on using this repository, refer to: 82 | 83 | - [Azure Guide](https://github.com/christosgalano/kql-template-repo/wiki/Azure): Azure setup 84 | - [GitHub Guide](https://github.com/christosgalano/kql-template-repo/wiki/GitHub): GitHub Actions setup 85 | - [Configuration Guide](https://github.com/christosgalano/kql-template-repo/wiki/Configuration): KQL config file format and options 86 | - [Usage Guide](https://github.com/christosgalano/kql-template-repo/wiki/Usage): General usage instructions 87 | 88 | > [!NOTE] 89 | > The `device` folder under `library` is a sample folder containing example queries. You can create your own folders and queries as needed. All new folder must be created under the `library` directory. 90 | 91 | ## Getting Started 92 | 93 | ### 1. Use This Template 94 | 95 | Click the **Use this template** button to create your own repository based on this template. 96 | 97 | ### 2. Azure Setup 98 | 99 | 1. Register an app in Azure Active Directory 100 | 2. Assign `Log Analytics Reader` permissions to the app 101 | 3. Configure federated credentials for GitHub Actions 102 | 103 | ### 3. GitHub Configuration 104 | 105 | 1. Set required secrets: `AZURE_CLIENT_ID`, `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID`, `WORKSPACE_ID` 106 | 2. Store your KQL queries in folders under the `library` directory 107 | 3. Configure query outputs using `.kql-config.yaml` files 108 | 109 | ### 4. Execute Queries 110 | 111 | #### Via GitHub Actions 112 | 113 | 1. Go to the **Actions** tab and run the **execute-queries** workflow 114 | 2. Specify the folder containing your queries 115 | 116 | #### Via Python Script 117 | 118 | ```sh 119 | # Set up virtual environment 120 | python3 -m venv venv 121 | source venv/bin/activate 122 | pip install -r .github/scripts/kql_query_executor/requirements.txt 123 | 124 | # Login to Azure 125 | az login 126 | 127 | # Make sure you are in the repository root 128 | python .github/scripts/kql_query_executor/main.py \ 129 | -w \ 130 | -f library/ \ 131 | -s kql-config-schema.json 132 | ``` 133 | 134 | ## Contributing 135 | 136 | Information about contributing to this project can be found [here](CONTRIBUTING.md). 137 | 138 | ## Creation & Curation 139 | 140 | A project created with love in Greece by [christosgalano](https://github.com/christosgalano) & [cyb3rmik3](https://github.com/cyb3rmik3). 141 | 142 | ## License 143 | 144 | This project is licensed under the [MIT License](LICENSE). 145 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import sys 5 | from pathlib import Path 6 | from typing import Any 7 | 8 | import jsonschema 9 | import yaml 10 | from model import ( 11 | CompressionType, 12 | KQLConfig, 13 | OutputConfig, 14 | OutputFormat, 15 | QueryConfig, 16 | ) 17 | 18 | 19 | def find_config_file(folder_path: str) -> str | None: 20 | """Find the config file in the folder or repository root.""" 21 | # First check in the specified folder 22 | for ext in ["yaml", "yml"]: 23 | config_file = os.path.join(folder_path, f".kql-config.{ext}") 24 | if os.path.exists(config_file): 25 | return config_file 26 | 27 | # If not found in folder, check repository root 28 | repo_root = Path(__file__).parent.parent.parent.parent 29 | for ext in ["yaml", "yml"]: 30 | root_config_file = repo_root / f".kql-config.{ext}" 31 | if root_config_file.exists(): 32 | logging.debug(f"Found config file in repository root: {root_config_file}") 33 | return str(root_config_file) 34 | 35 | return None 36 | 37 | 38 | def validate_file_path(file_path: str, config_dir: str) -> str: 39 | """Validate that a query file exists and ends with .kql.""" 40 | if not file_path.endswith(".kql"): 41 | raise ValueError(f"Query file must end with .kql: {file_path}") 42 | 43 | # Handle relative paths from config file location 44 | abs_path = os.path.normpath(os.path.join(config_dir, file_path)) 45 | if not os.path.isfile(abs_path): 46 | raise ValueError( 47 | f"Query file does not exist: {file_path} (resolved to {abs_path})" 48 | ) 49 | 50 | return file_path 51 | 52 | 53 | def convert_dict_to_config(config_dict: dict[str, Any], config_dir: str) -> KQLConfig: 54 | """Convert a dictionary to a KQLConfig object with validation.""" 55 | try: 56 | # Process queries section 57 | queries = [] 58 | if "queries" in config_dict: 59 | for query_dict in config_dict["queries"]: 60 | validate_file_path(query_dict["file"], config_dir) 61 | 62 | query_output = None 63 | if "output" in query_dict: 64 | query_output = [] 65 | for fmt in query_dict["output"]: 66 | # Parse output format 67 | format_str = fmt["format"] 68 | try: 69 | output_format = OutputFormat(format_str) 70 | except ValueError: 71 | raise ValueError(f"Invalid output format: {format_str}") 72 | 73 | # Handle output file based on format 74 | output_file = fmt.get("file") 75 | if output_file and output_format != OutputFormat.NONE: 76 | # Verify file path doesn't contain whitespace 77 | if any(c.isspace() for c in output_file): 78 | raise ValueError( 79 | f"Output file path should not contain whitespace: '{output_file}'. " 80 | f"Use underscores or dashes instead." 81 | ) 82 | 83 | # Check if file exists (will be overwritten) 84 | abs_file_path = os.path.normpath( 85 | os.path.join(config_dir, output_file) 86 | ) 87 | if os.path.exists(abs_file_path): 88 | logging.warning( 89 | f"Output file exists and will be overwritten: {output_file}" 90 | ) 91 | 92 | # Ensure directory exists 93 | dir_path = os.path.dirname(abs_file_path) 94 | if not os.path.exists(dir_path): 95 | logging.info( 96 | f"Directory does not exist and will be created: {dir_path}" 97 | ) 98 | 99 | # Handle compression 100 | compression_str = fmt.get("compression") 101 | compression = None 102 | if compression_str: 103 | try: 104 | compression = CompressionType(compression_str) 105 | except ValueError: 106 | raise ValueError( 107 | f"Invalid compression type: {compression_str}" 108 | ) 109 | 110 | # Create output config 111 | format_config = OutputConfig( 112 | format=output_format, 113 | query=fmt.get("query"), # JMESPath query 114 | file=output_file, # Full file path if specified 115 | compression=compression, 116 | ) 117 | query_output.append(format_config) 118 | 119 | query_config = QueryConfig( 120 | file=query_dict["file"], 121 | output=query_output, 122 | ) 123 | queries.append(query_config) 124 | 125 | return KQLConfig( 126 | version=config_dict.get("version", "1.0"), 127 | queries=queries, 128 | ) 129 | except Exception as e: 130 | logging.error(f"Error converting configuration: {e}") 131 | sys.exit(1) 132 | 133 | 134 | def load_config(config_path: str, schema_path: str | None = None) -> KQLConfig: 135 | """Load and validate the configuration file against schema.""" 136 | try: 137 | # Try to load YAML config 138 | try: 139 | with open(config_path, "r") as file: 140 | config_dict = yaml.safe_load(file) or {} 141 | except yaml.YAMLError as e: 142 | logging.error(f"Invalid YAML format in {config_path}: {str(e)}") 143 | sys.exit(1) 144 | except FileNotFoundError: 145 | logging.error(f"Configuration file not found: {config_path}") 146 | sys.exit(1) 147 | 148 | # Handle schema validation 149 | if not schema_path: 150 | repo_root = Path(__file__).parent.parent.parent.parent 151 | schema_path = str(repo_root / "kql-config-schema.json") 152 | 153 | if not os.path.exists(schema_path): 154 | logging.warning( 155 | f"Schema file not found at {schema_path}, skipping validation" 156 | ) 157 | else: 158 | try: 159 | with open(schema_path, "r") as schema_file: 160 | schema = json.load(schema_file) 161 | jsonschema.validate(instance=config_dict, schema=schema) 162 | except json.JSONDecodeError as e: 163 | logging.error(f"Invalid JSON schema file {schema_path}: {str(e)}") 164 | sys.exit(1) 165 | except jsonschema.exceptions.ValidationError as e: 166 | # Provide more detailed validation error message 167 | logging.error( 168 | f"Config validation failed for {config_path}: {e.message}\n" 169 | ) 170 | sys.exit(1) 171 | 172 | # Convert config dictionary to object 173 | config_dir = os.path.dirname(os.path.abspath(config_path)) 174 | try: 175 | return convert_dict_to_config(config_dict, config_dir) 176 | except ValueError as e: 177 | logging.error(f"Invalid configuration value: {str(e)}") 178 | sys.exit(1) 179 | except KeyError as e: 180 | logging.error(f"Missing required configuration key: {str(e)}") 181 | sys.exit(1) 182 | 183 | except Exception as e: 184 | # Catch-all for unexpected errors with more context 185 | logging.error( 186 | f"Unexpected error loading config {config_path}:\n" 187 | f" Type: {type(e).__name__}\n" 188 | f" Error: {str(e)}" 189 | ) 190 | sys.exit(1) 191 | 192 | 193 | def get_applicable_files(folder_path: str, config: KQLConfig) -> list[str]: 194 | """Get list of applicable KQL files based on config.""" 195 | # If no queries are configured, find all KQL files in the folder and subfolders 196 | if not config.queries: 197 | all_files = [] 198 | for root, _, files in os.walk(folder_path): 199 | for file in files: 200 | if file.endswith(".kql"): 201 | # Get path relative to the specified folder 202 | rel_path = os.path.relpath(os.path.join(root, file), folder_path) 203 | all_files.append(rel_path) 204 | return all_files 205 | 206 | # If queries are configured, use only those files 207 | config_files = [] 208 | 209 | # The folder_path is used as the base for relative paths 210 | # We don't need to get parent directory which causes the "../" prefix issue 211 | 212 | for query in config.queries: 213 | file_path = query.file 214 | try: 215 | # Check if the file exists in the folder or a subdirectory 216 | # First try as relative to folder_path 217 | full_path = os.path.join(folder_path, file_path) 218 | 219 | if os.path.isfile(full_path): 220 | # File exists, use the relative path directly 221 | config_files.append(file_path) 222 | else: 223 | # File doesn't exist, log warning and skip 224 | logging.warning(f"Query file does not exist: {file_path}") 225 | # Don't add to config_files 226 | except Exception as e: 227 | logging.warning(f"Skipping invalid query file {file_path}: {e}") 228 | continue 229 | 230 | if not config_files: 231 | logging.warning("No valid query files found in configuration") 232 | 233 | return config_files 234 | 235 | 236 | def get_output_configs_for_query(config: KQLConfig, file: str) -> list[OutputConfig]: 237 | """Determine which output configs to use for a query.""" 238 | # Get the base name without extension 239 | file_path_variations = [ 240 | file, # Just the file name 241 | os.path.join("*", file), # Any subfolder/file name 242 | f"**/{file}", # Any nested path to file name 243 | ] 244 | 245 | # Check if there's a specific query configuration 246 | if config.queries: 247 | for query in config.queries: 248 | # Check if this config applies to our file 249 | # Match by either exact path or filename 250 | query_file = query.file 251 | file_basename = os.path.basename(query_file) 252 | 253 | if file == file_basename or any( 254 | file_pattern in query_file for file_pattern in file_path_variations 255 | ): 256 | if query.output: 257 | logging.debug(f"Found specific output config for {file}") 258 | return query.output 259 | 260 | # If no specific configuration found, use default JSON output to console 261 | logging.debug(f"Using default JSON output for {file}") 262 | return [OutputConfig(format=OutputFormat.JSON)] 263 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tempfile 4 | from unittest import mock 5 | 6 | import pytest 7 | import yaml 8 | 9 | # Add parent directory to path so we can import modules 10 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 11 | 12 | from config import ( 13 | convert_dict_to_config, 14 | find_config_file, 15 | get_applicable_files, 16 | get_output_configs_for_query, 17 | load_config, 18 | validate_file_path, 19 | ) 20 | from model import CompressionType, KQLConfig, OutputConfig, OutputFormat, QueryConfig 21 | 22 | 23 | @pytest.fixture 24 | def temp_dir_with_files(): 25 | """Create a temporary directory with test files.""" 26 | with tempfile.TemporaryDirectory() as temp_dir: 27 | # Create a KQL file 28 | kql_file = os.path.join(temp_dir, "test_query.kql") 29 | with open(kql_file, "w") as f: 30 | f.write("// Test KQL query\nTable | take 10") 31 | 32 | # Create a non-KQL file 33 | txt_file = os.path.join(temp_dir, "test_file.txt") 34 | with open(txt_file, "w") as f: 35 | f.write("This is not a KQL file") 36 | 37 | # Create a subdirectory with another KQL file 38 | subdir = os.path.join(temp_dir, "subdir") 39 | os.makedirs(subdir) 40 | sub_kql_file = os.path.join(subdir, "subdir_query.kql") 41 | with open(sub_kql_file, "w") as f: 42 | f.write("// Subdirectory KQL query\nTable | where Column == 'value'") 43 | 44 | yield temp_dir 45 | 46 | 47 | @pytest.fixture 48 | def basic_config_dict(): 49 | """Return a basic configuration dictionary.""" 50 | return { 51 | "version": "1.0", 52 | "queries": [ 53 | { 54 | "file": "test_query.kql", 55 | "output": [{"format": "json"}], 56 | } 57 | ], 58 | } 59 | 60 | 61 | @pytest.fixture 62 | def complex_config_dict(): 63 | """Return a complex configuration dictionary with multiple outputs and compression.""" 64 | return { 65 | "version": "1.0", 66 | "queries": [ 67 | { 68 | "file": "test_query.kql", 69 | "output": [ 70 | {"format": "json", "query": "[].{Name: name, Count: count}"}, 71 | {"format": "json", "file": "results/test_output.json"}, 72 | { 73 | "format": "json", 74 | "file": "compressed_results/compressed_output.json", 75 | "compression": "gzip", 76 | }, 77 | ], 78 | }, 79 | { 80 | "file": "subdir/subdir_query.kql", 81 | "output": [{"format": "yaml", "file": "subdir_results/output.yaml"}], 82 | }, 83 | ], 84 | } 85 | 86 | 87 | def test_find_config_file_in_folder(temp_dir_with_files): 88 | """Test finding a config file in the specified folder.""" 89 | # Create config file in the temp directory 90 | config_path = os.path.join(temp_dir_with_files, ".kql-config.yaml") 91 | with open(config_path, "w") as f: 92 | f.write("version: '1.0'") 93 | 94 | # Test finding the config file 95 | found_path = find_config_file(temp_dir_with_files) 96 | assert found_path == config_path 97 | 98 | 99 | def test_find_config_file_yml_extension(temp_dir_with_files): 100 | """Test finding a config file with .yml extension.""" 101 | # Create config file with .yml extension 102 | config_path = os.path.join(temp_dir_with_files, ".kql-config.yml") 103 | with open(config_path, "w") as f: 104 | f.write("version: '1.0'") 105 | 106 | # Test finding the config file 107 | found_path = find_config_file(temp_dir_with_files) 108 | assert found_path == config_path 109 | 110 | 111 | @mock.patch("config.Path") 112 | def test_find_config_file_not_found(mock_path, temp_dir_with_files): 113 | """Test behavior when config file is not found.""" 114 | # Mock repo_root path to ensure it doesn't find a real config file 115 | mock_repo_root = mock.MagicMock() 116 | mock_path.return_value.parent.parent.parent.parent = mock_repo_root 117 | mock_repo_root.__truediv__.return_value.exists.return_value = False 118 | 119 | # No config file created 120 | found_path = find_config_file(temp_dir_with_files) 121 | assert found_path is None 122 | 123 | 124 | def test_validate_file_path_valid(temp_dir_with_files): 125 | """Test validating a valid KQL file path.""" 126 | file_path = "test_query.kql" 127 | result = validate_file_path(file_path, temp_dir_with_files) 128 | assert result == file_path 129 | 130 | 131 | def test_validate_file_path_not_kql(temp_dir_with_files): 132 | """Test validating a file path that doesn't end with .kql.""" 133 | file_path = "test_file.txt" 134 | with pytest.raises(ValueError, match="Query file must end with .kql"): 135 | validate_file_path(file_path, temp_dir_with_files) 136 | 137 | 138 | def test_validate_file_path_not_exist(temp_dir_with_files): 139 | """Test validating a file path that doesn't exist.""" 140 | file_path = "nonexistent.kql" 141 | with pytest.raises(ValueError, match="Query file does not exist"): 142 | validate_file_path(file_path, temp_dir_with_files) 143 | 144 | 145 | def test_validate_file_path_in_subdir(temp_dir_with_files): 146 | """Test validating a file path in a subdirectory.""" 147 | file_path = "subdir/subdir_query.kql" 148 | result = validate_file_path(file_path, temp_dir_with_files) 149 | assert result == file_path 150 | 151 | 152 | def test_convert_dict_to_config_invalid_file(temp_dir_with_files): 153 | """Test that files with invalid characters raise an exception.""" 154 | # Create config with invalid file path 155 | invalid_config = { 156 | "version": "1.0", 157 | "queries": [ 158 | { 159 | "file": "test_query.kql", 160 | "output": [ 161 | { 162 | "format": "json", 163 | "file": "invalid/ path/file.json", # Contains whitespace 164 | } 165 | ], 166 | } 167 | ], 168 | } 169 | 170 | # We expect this to raise a ValueError, but it's caught inside convert_dict_to_config 171 | # and converted to a SystemExit exception. 172 | with pytest.raises(SystemExit): 173 | convert_dict_to_config(invalid_config, temp_dir_with_files) 174 | 175 | 176 | def test_convert_dict_to_config_basic(temp_dir_with_files, basic_config_dict): 177 | """Test converting a basic config dictionary to a KQLConfig object.""" 178 | config = convert_dict_to_config(basic_config_dict, temp_dir_with_files) 179 | 180 | assert config.version == "1.0" 181 | assert len(config.queries) == 1 182 | assert config.queries[0].file == "test_query.kql" 183 | assert config.queries[0].output[0].format == OutputFormat.JSON 184 | 185 | 186 | def test_convert_dict_to_config_complex(temp_dir_with_files, complex_config_dict): 187 | """Test converting a complex config dictionary to a KQLConfig object.""" 188 | config = convert_dict_to_config(complex_config_dict, temp_dir_with_files) 189 | 190 | assert config.version == "1.0" 191 | assert len(config.queries) == 2 192 | 193 | # First query checks 194 | query1 = config.queries[0] 195 | assert query1.file == "test_query.kql" 196 | assert len(query1.output) == 3 197 | 198 | # Check JMESPath query output 199 | assert query1.output[0].format == OutputFormat.JSON 200 | assert query1.output[0].query == "[].{Name: name, Count: count}" 201 | 202 | # Check file output without compression 203 | assert query1.output[1].format == OutputFormat.JSON 204 | assert query1.output[1].file == "results/test_output.json" 205 | 206 | # Check file output with compression 207 | assert query1.output[2].format == OutputFormat.JSON 208 | assert query1.output[2].file == "compressed_results/compressed_output.json" 209 | assert query1.output[2].compression == CompressionType.GZIP 210 | 211 | # Second query checks 212 | query2 = config.queries[1] 213 | assert query2.file == "subdir/subdir_query.kql" 214 | assert len(query2.output) == 1 215 | assert query2.output[0].format == OutputFormat.YAML 216 | assert query2.output[0].file == "subdir_results/output.yaml" 217 | 218 | 219 | def test_convert_dict_to_config_empty(): 220 | """Test converting an empty config dictionary.""" 221 | config = convert_dict_to_config({}, "/tmp") 222 | 223 | assert config.version == "1.0" 224 | assert config.queries == [] 225 | 226 | 227 | @mock.patch("config.jsonschema.validate") 228 | def test_load_config_valid(mock_validate, temp_dir_with_files, basic_config_dict): 229 | """Test loading a valid configuration file.""" 230 | # Create a valid config file 231 | config_path = os.path.join(temp_dir_with_files, ".kql-config.yaml") 232 | with open(config_path, "w") as f: 233 | yaml.dump(basic_config_dict, f) 234 | 235 | # Create a mock schema file 236 | schema_path = os.path.join(temp_dir_with_files, "schema.json") 237 | with open(schema_path, "w") as f: 238 | f.write('{"type":"object"}') 239 | 240 | # Load the config 241 | config = load_config(config_path, schema_path) 242 | 243 | # Verify the config was loaded correctly 244 | assert isinstance(config, KQLConfig) 245 | assert config.version == "1.0" 246 | assert len(config.queries) == 1 247 | assert config.queries[0].file == "test_query.kql" 248 | 249 | 250 | @mock.patch("config.sys.exit") 251 | def test_load_config_validation_error(mock_exit, temp_dir_with_files): 252 | """Test loading a config that fails schema validation.""" 253 | # Create an invalid config file (missing required 'file' field) 254 | config_path = os.path.join(temp_dir_with_files, "invalid_config.yaml") 255 | with open(config_path, "w") as f: 256 | f.write( 257 | """ 258 | version: '1.0' 259 | queries: 260 | - output: 261 | - format: json 262 | """ 263 | ) 264 | 265 | # Define a mock schema that requires 'file' field 266 | schema_path = os.path.join(temp_dir_with_files, "schema.json") 267 | with open(schema_path, "w") as f: 268 | f.write( 269 | """ 270 | { 271 | "type": "object", 272 | "properties": { 273 | "queries": { 274 | "type": "array", 275 | "items": { 276 | "type": "object", 277 | "required": ["file"] 278 | } 279 | } 280 | } 281 | } 282 | """ 283 | ) 284 | 285 | # This should fail validation and exit 286 | load_config(config_path, schema_path) 287 | 288 | # Verify sys.exit was called 289 | mock_exit.assert_called_once() 290 | 291 | 292 | @mock.patch("config.sys.exit") 293 | def test_load_config_file_not_found(mock_exit): 294 | """Test loading a config file that doesn't exist.""" 295 | load_config("nonexistent_file.yaml") 296 | 297 | # Verify sys.exit was called 298 | mock_exit.assert_called_once() 299 | 300 | 301 | def test_get_applicable_files_no_config(temp_dir_with_files): 302 | """Test getting applicable files when no queries are configured.""" 303 | # Create empty config 304 | config = KQLConfig(version="1.0", queries=[]) 305 | 306 | # Get applicable files 307 | files = get_applicable_files(temp_dir_with_files, config) 308 | 309 | # Should return all KQL files in the directory and subdirectories 310 | assert len(files) == 2 311 | assert "test_query.kql" in files 312 | assert "subdir/subdir_query.kql" in files 313 | 314 | 315 | def test_get_applicable_files_with_config(temp_dir_with_files): 316 | """Test getting applicable files from configuration.""" 317 | # Create config with specific queries 318 | config = KQLConfig( 319 | version="1.0", 320 | queries=[ 321 | QueryConfig( 322 | file="test_query.kql", output=[OutputConfig(format=OutputFormat.JSON)] 323 | ) 324 | ], 325 | ) 326 | 327 | # Get applicable files 328 | files = get_applicable_files(temp_dir_with_files, config) 329 | 330 | # Should return only the specified file 331 | assert len(files) == 1 332 | assert "test_query.kql" in files 333 | 334 | 335 | def test_get_applicable_files_with_subdir_config(temp_dir_with_files): 336 | """Test getting applicable files with subdirectory paths.""" 337 | # Create config with file in subdirectory 338 | config = KQLConfig( 339 | version="1.0", 340 | queries=[ 341 | QueryConfig( 342 | file="subdir/subdir_query.kql", 343 | output=[OutputConfig(format=OutputFormat.JSON)], 344 | ) 345 | ], 346 | ) 347 | 348 | # Get applicable files 349 | files = get_applicable_files(temp_dir_with_files, config) 350 | 351 | # Should return only the specified file 352 | assert len(files) == 1 353 | assert "subdir/subdir_query.kql" in files 354 | 355 | 356 | def test_get_applicable_files_invalid_path(temp_dir_with_files): 357 | """Test getting applicable files with an invalid path in config.""" 358 | # Create config with invalid file path 359 | config = KQLConfig( 360 | version="1.0", 361 | queries=[ 362 | QueryConfig( 363 | file="nonexistent.kql", output=[OutputConfig(format=OutputFormat.JSON)] 364 | ) 365 | ], 366 | ) 367 | 368 | # Get applicable files 369 | files = get_applicable_files(temp_dir_with_files, config) 370 | 371 | # Should return empty list since the path is invalid 372 | assert len(files) == 0 373 | 374 | 375 | def test_get_applicable_files_multiple_subdirs(temp_dir_with_files): 376 | """Test getting applicable files from multiple nested subdirectories.""" 377 | # Create multiple nested subdirectories with KQL files 378 | nested_dir1 = os.path.join(temp_dir_with_files, "logs") 379 | os.makedirs(nested_dir1) 380 | nested_kql1 = os.path.join(nested_dir1, "logs_query.kql") 381 | with open(nested_kql1, "w") as f: 382 | f.write("// Logs KQL query\nLogs | where Level == 'Error'") 383 | 384 | nested_dir2 = os.path.join(temp_dir_with_files, "metrics", "system") 385 | os.makedirs(nested_dir2) 386 | nested_kql2 = os.path.join(nested_dir2, "cpu_usage.kql") 387 | with open(nested_kql2, "w") as f: 388 | f.write("// CPU usage query\nPerf | where CounterName == 'CPU'") 389 | 390 | nested_dir3 = os.path.join(temp_dir_with_files, "metrics", "network") 391 | os.makedirs(nested_dir3) 392 | nested_kql3 = os.path.join(nested_dir3, "network_traffic.kql") 393 | with open(nested_kql3, "w") as f: 394 | f.write("// Network traffic query\nPerf | where CounterName == 'Network'") 395 | 396 | # Create empty config (no specific queries) 397 | config = KQLConfig(version="1.0", queries=[]) 398 | 399 | # Get applicable files 400 | files = get_applicable_files(temp_dir_with_files, config) 401 | 402 | # Should find all KQL files in all subdirectories 403 | assert len(files) == 5 # 2 original + 3 new 404 | 405 | # Original files 406 | assert "test_query.kql" in files 407 | assert "subdir/subdir_query.kql" in files 408 | 409 | # Files in new subdirectories 410 | assert "logs/logs_query.kql" in files 411 | assert "metrics/system/cpu_usage.kql" in files 412 | assert "metrics/network/network_traffic.kql" in files 413 | 414 | 415 | def test_get_output_configs_for_query(temp_dir_with_files): 416 | """Test getting output configs for a specific query.""" 417 | config = KQLConfig( 418 | version="1.0", 419 | queries=[ 420 | QueryConfig( 421 | file="test_query.kql", 422 | output=[ 423 | OutputConfig( 424 | format=OutputFormat.JSON, query="[].{Name: name, Count: count}" 425 | ) 426 | ], 427 | ), 428 | QueryConfig( 429 | file="subdir/subdir_query.kql", 430 | output=[ 431 | OutputConfig( 432 | format=OutputFormat.YAML, file="custom_dir/output.yaml" 433 | ) 434 | ], 435 | ), 436 | ], 437 | ) 438 | 439 | # Get output config for the first query 440 | configs1 = get_output_configs_for_query(config, "test_query.kql") 441 | assert len(configs1) == 1 442 | assert configs1[0].format == OutputFormat.JSON 443 | assert configs1[0].query == "[].{Name: name, Count: count}" 444 | 445 | # Get output config for the second query 446 | configs2 = get_output_configs_for_query(config, "subdir_query.kql") 447 | assert len(configs2) == 1 448 | assert configs2[0].format == OutputFormat.YAML 449 | assert configs2[0].file == "custom_dir/output.yaml" 450 | 451 | 452 | def test_get_output_configs_default(temp_dir_with_files): 453 | """Test getting default output config when no matching query found.""" 454 | config = KQLConfig( 455 | version="1.0", 456 | queries=[ 457 | QueryConfig( 458 | file="different.kql", 459 | output=[ 460 | OutputConfig(format=OutputFormat.YAML, file="results/output.yaml") 461 | ], 462 | ) 463 | ], 464 | ) 465 | 466 | # Get output config for a query not in the config 467 | configs = get_output_configs_for_query(config, "test_query.kql") 468 | 469 | # Should get default JSON output 470 | assert len(configs) == 1 471 | assert configs[0].format == OutputFormat.JSON 472 | -------------------------------------------------------------------------------- /.github/scripts/kql_query_executor/tests/test_execute.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import subprocess 4 | import sys 5 | from unittest import mock 6 | 7 | import pytest 8 | 9 | # Add parent directory to path so we can import modules 10 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 11 | 12 | from execute import execute_query 13 | from model import CompressionType, KQLConfig, OutputConfig, OutputFormat, QueryConfig 14 | 15 | 16 | @pytest.fixture 17 | def basic_config(): 18 | """Basic configuration with a single query.""" 19 | return KQLConfig( 20 | version="1.0", 21 | queries=[ 22 | QueryConfig( 23 | file="example.kql", 24 | output=[ 25 | OutputConfig(format=OutputFormat.JSONC), 26 | OutputConfig( 27 | format=OutputFormat.JSON, 28 | file="query-results/output.json", 29 | compression=CompressionType.GZIP, 30 | ), 31 | ], 32 | ) 33 | ], 34 | ) 35 | 36 | 37 | @mock.patch("execute.subprocess.run") 38 | @mock.patch("execute.get_output_configs_for_query") 39 | def test_basic_execution_to_console(mock_get_configs, mock_subprocess_run, caplog): 40 | """Test basic execution with JSON output to console.""" 41 | # Setup mocks 42 | mock_get_configs.return_value = [OutputConfig(format=OutputFormat.JSON)] 43 | mock_subprocess_run.return_value = mock.MagicMock( 44 | stdout="test output", stderr="", returncode=0 45 | ) 46 | 47 | # Execute query 48 | result = execute_query( 49 | folder_path="/test", 50 | file_path="query.kql", 51 | workspace_id="test-workspace", 52 | config=KQLConfig(), 53 | ) 54 | 55 | # Verify 56 | assert result is True 57 | mock_subprocess_run.assert_called_once() 58 | cmd = mock_subprocess_run.call_args[0][0] 59 | assert "--output" in cmd 60 | assert "json" in cmd 61 | assert "--query" not in cmd # No JMESPath query 62 | 63 | 64 | @mock.patch("execute.subprocess.run") 65 | @mock.patch("execute.get_output_configs_for_query") 66 | @mock.patch("os.makedirs") 67 | @mock.patch("builtins.open", new_callable=mock.mock_open) 68 | def test_output_to_file( 69 | mock_open, mock_makedirs, mock_get_configs, mock_subprocess_run 70 | ): 71 | """Test output to file with directory creation.""" 72 | # Setup mocks 73 | mock_get_configs.return_value = [ 74 | OutputConfig(format=OutputFormat.JSON, file="results/output.json") 75 | ] 76 | mock_subprocess_run.return_value = mock.MagicMock( 77 | stdout="test output", stderr="", returncode=0 78 | ) 79 | 80 | # Execute query 81 | result = execute_query( 82 | folder_path="/test", 83 | file_path="query.kql", 84 | workspace_id="test-workspace", 85 | config=KQLConfig(), 86 | ) 87 | 88 | # Verify 89 | assert result is True 90 | mock_makedirs.assert_called_once_with( 91 | os.path.dirname("results/output.json"), exist_ok=True 92 | ) 93 | mock_open.assert_called_once_with("results/output.json", "w") 94 | mock_open().write.assert_called_once_with("test output") 95 | 96 | 97 | @mock.patch("execute.subprocess.run") 98 | @mock.patch("execute.get_output_configs_for_query") 99 | def test_format_none_skips_execution(mock_get_configs, mock_subprocess_run, caplog): 100 | """Test that NONE format skips execution.""" 101 | # Configure caplog to capture the right log level 102 | caplog.set_level(logging.INFO) 103 | 104 | # Setup mocks 105 | mock_get_configs.return_value = [OutputConfig(format=OutputFormat.NONE)] 106 | 107 | # Execute query 108 | result = execute_query( 109 | folder_path="/test", 110 | file_path="query.kql", 111 | workspace_id="test-workspace", 112 | config=KQLConfig(), 113 | ) 114 | 115 | # Verify 116 | assert result is True 117 | assert mock_subprocess_run.call_count == 0 # Should not execute any command 118 | assert "Skipping output for" in caplog.text 119 | 120 | 121 | @mock.patch("execute.subprocess.run") 122 | @mock.patch("execute.get_output_configs_for_query") 123 | def test_all_output_formats(mock_get_configs, mock_subprocess_run): 124 | """Test all available output formats.""" 125 | formats = [ 126 | OutputFormat.JSON, 127 | OutputFormat.JSONC, 128 | OutputFormat.TABLE, 129 | OutputFormat.TSV, 130 | OutputFormat.YAML, 131 | OutputFormat.YAMLC, 132 | ] 133 | 134 | # Execute each format 135 | for fmt in formats: 136 | # Reset mocks 137 | mock_subprocess_run.reset_mock() 138 | mock_get_configs.return_value = [OutputConfig(format=fmt)] 139 | mock_subprocess_run.return_value = mock.MagicMock( 140 | stdout=f"test {fmt.value} output", stderr="", returncode=0 141 | ) 142 | 143 | # Execute query 144 | result = execute_query( 145 | folder_path="/test", 146 | file_path="query.kql", 147 | workspace_id="test-workspace", 148 | config=KQLConfig(), 149 | ) 150 | 151 | # Verify 152 | assert result is True 153 | mock_subprocess_run.assert_called_once() 154 | cmd = mock_subprocess_run.call_args[0][0] 155 | assert "--output" in cmd 156 | assert fmt.value in cmd 157 | 158 | 159 | @mock.patch("execute.subprocess.run") 160 | @mock.patch("execute.get_output_configs_for_query") 161 | def test_jmespath_query(mock_get_configs, mock_subprocess_run): 162 | """Test JMESPath query parameters are correctly passed.""" 163 | # Setup mocks 164 | mock_get_configs.return_value = [ 165 | OutputConfig(format=OutputFormat.JSON, query="length(@)") 166 | ] 167 | mock_subprocess_run.return_value = mock.MagicMock( 168 | stdout="5", stderr="", returncode=0 169 | ) 170 | 171 | # Execute query 172 | result = execute_query( 173 | folder_path="/test", 174 | file_path="query.kql", 175 | workspace_id="test-workspace", 176 | config=KQLConfig(), 177 | ) 178 | 179 | # Verify 180 | assert result is True 181 | cmd = mock_subprocess_run.call_args[0][0] 182 | assert "--query" in cmd 183 | assert "length(@)" in cmd 184 | 185 | 186 | @mock.patch("execute.subprocess.run") 187 | @mock.patch("execute.get_output_configs_for_query") 188 | def test_jmespath_error_handling(mock_get_configs, mock_subprocess_run, caplog): 189 | """Test handling of invalid JMESPath query.""" 190 | # Setup mocks 191 | mock_get_configs.return_value = [ 192 | OutputConfig(format=OutputFormat.JSON, query="invalid[query") 193 | ] 194 | mock_subprocess_run.side_effect = subprocess.CalledProcessError( 195 | 1, "az monitor", stderr="JMESPath query failed: Invalid syntax at column 7" 196 | ) 197 | 198 | # Execute query 199 | result = execute_query( 200 | folder_path="/test", 201 | file_path="query.kql", 202 | workspace_id="test-workspace", 203 | config=KQLConfig(), 204 | ) 205 | 206 | # Verify failure 207 | assert result is False 208 | assert "Error executing query" in caplog.text 209 | assert "JMESPath query failed" in caplog.text 210 | 211 | 212 | @mock.patch("execute.subprocess.run") 213 | @mock.patch("execute.get_output_configs_for_query") 214 | @mock.patch("os.makedirs") 215 | @mock.patch("builtins.open", new_callable=mock.mock_open) 216 | @mock.patch("gzip.open", new_callable=mock.mock_open) 217 | @mock.patch("shutil.copyfileobj") 218 | @mock.patch("os.remove") 219 | def test_gzip_compression( 220 | mock_remove, 221 | mock_copyfileobj, 222 | mock_gzip_open, 223 | mock_open, 224 | mock_makedirs, 225 | mock_get_configs, 226 | mock_subprocess_run, 227 | ): 228 | """Test GZIP compression of output.""" 229 | # Setup mocks 230 | mock_get_configs.return_value = [ 231 | OutputConfig( 232 | format=OutputFormat.JSON, 233 | file="results/output.json", 234 | compression=CompressionType.GZIP, 235 | ) 236 | ] 237 | mock_subprocess_run.return_value = mock.MagicMock( 238 | stdout="test output", stderr="", returncode=0 239 | ) 240 | 241 | # Execute query 242 | result = execute_query( 243 | folder_path="/test", 244 | file_path="query.kql", 245 | workspace_id="test-workspace", 246 | config=KQLConfig(), 247 | ) 248 | 249 | # Verify 250 | assert result is True 251 | 252 | # Check the sequence of file operations including context manager calls 253 | expected_calls = [ 254 | # Write the initial results 255 | mock.call("results/output.json", "w"), 256 | mock.call().__enter__(), 257 | mock.call().write("test output"), 258 | mock.call().__exit__(None, None, None), 259 | # Read for compression 260 | mock.call("results/output.json", "rb"), 261 | mock.call().__enter__(), 262 | mock.call().__exit__(None, None, None), 263 | ] 264 | mock_open.assert_has_calls(expected_calls, any_order=False) 265 | 266 | # Verify compression operations 267 | mock_gzip_open.assert_called_once_with("results/output.json.gz", "wb") 268 | mock_copyfileobj.assert_called_once() 269 | mock_remove.assert_called_once_with("results/output.json") 270 | 271 | 272 | @mock.patch("execute.subprocess.run") 273 | @mock.patch("execute.get_output_configs_for_query") 274 | @mock.patch("os.makedirs") 275 | @mock.patch("builtins.open", new_callable=mock.mock_open) 276 | @mock.patch("shutil.make_archive") 277 | @mock.patch("os.remove") 278 | def test_zip_compression( 279 | mock_remove, 280 | mock_make_archive, 281 | mock_open, 282 | mock_makedirs, 283 | mock_get_configs, 284 | mock_subprocess_run, 285 | ): 286 | """Test ZIP compression of output.""" 287 | # Setup mocks 288 | mock_get_configs.return_value = [ 289 | OutputConfig( 290 | format=OutputFormat.JSON, 291 | file="results/output.json", 292 | compression=CompressionType.ZIP, 293 | ) 294 | ] 295 | mock_subprocess_run.return_value = mock.MagicMock( 296 | stdout="test output", stderr="", returncode=0 297 | ) 298 | 299 | # Execute query 300 | result = execute_query( 301 | folder_path="/test", 302 | file_path="query.kql", 303 | workspace_id="test-workspace", 304 | config=KQLConfig(), 305 | ) 306 | 307 | # Verify 308 | assert result is True 309 | mock_open.assert_called_once_with("results/output.json", "w") 310 | mock_make_archive.assert_called_once() 311 | assert "results/output" in mock_make_archive.call_args[0][0] # base name 312 | assert "zip" in mock_make_archive.call_args[0][1] # format 313 | mock_remove.assert_called_once_with("results/output.json") 314 | 315 | 316 | @mock.patch("execute.subprocess.run") 317 | @mock.patch("execute.get_output_configs_for_query") 318 | def test_multiple_formats(mock_get_configs, mock_subprocess_run): 319 | """Test execution with multiple output formats.""" 320 | # Setup mocks 321 | mock_get_configs.return_value = [ 322 | OutputConfig(format=OutputFormat.JSONC), 323 | OutputConfig( 324 | format=OutputFormat.JSON, 325 | file="query-results/output.json", 326 | compression=CompressionType.GZIP, 327 | ), 328 | ] 329 | mock_subprocess_run.return_value = mock.MagicMock( 330 | stdout="test output", stderr="", returncode=0 331 | ) 332 | 333 | # Execute query 334 | result = execute_query( 335 | folder_path="/test", 336 | file_path="example.kql", 337 | workspace_id="test-workspace", 338 | config=KQLConfig(), 339 | ) 340 | 341 | # Verify 342 | assert result is True 343 | assert mock_subprocess_run.call_count == 2 # Should call for each format 344 | 345 | # First call (JSONC to console) 346 | first_call = mock_subprocess_run.call_args_list[0] 347 | assert "--output" in first_call[0][0] 348 | assert "jsonc" in first_call[0][0] 349 | 350 | # Second call (JSON to file with compression) 351 | second_call = mock_subprocess_run.call_args_list[1] 352 | assert "--output" in second_call[0][0] 353 | assert "json" in second_call[0][0] 354 | 355 | 356 | @mock.patch("execute.subprocess.run") 357 | @mock.patch("execute.get_output_configs_for_query") 358 | def test_complex_filtering(mock_get_configs, mock_subprocess_run): 359 | """Test execution with multiple filtered outputs.""" 360 | # Setup mocks 361 | mock_get_configs.return_value = [ 362 | OutputConfig(format=OutputFormat.JSONC), # Console output in JSONC 363 | OutputConfig( 364 | format=OutputFormat.JSON, 365 | file="logs/security-events.json", 366 | compression=CompressionType.GZIP, 367 | ), # Compressed file output 368 | OutputConfig( 369 | format=OutputFormat.JSON, 370 | query="events[?severity=='critical']", 371 | file="alerts/critical.json", 372 | ), # Filtered by severity=critical 373 | OutputConfig( 374 | format=OutputFormat.JSON, 375 | query="events[?severity=='high']", 376 | file="alerts/high.json", 377 | ), # Filtered by severity=high 378 | ] 379 | mock_subprocess_run.return_value = mock.MagicMock( 380 | stdout="test output", stderr="", returncode=0 381 | ) 382 | 383 | # Execute query 384 | result = execute_query( 385 | folder_path="/test", 386 | file_path="security-events.kql", 387 | workspace_id="test-workspace", 388 | config=KQLConfig(), 389 | ) 390 | 391 | # Verify 392 | assert result is True 393 | assert mock_subprocess_run.call_count == 4 # Should call for each format 394 | 395 | # Check JMESPath queries were included correctly 396 | calls = mock_subprocess_run.call_args_list 397 | 398 | # No query for first two calls 399 | assert "--query" not in calls[0][0][0] 400 | assert "--query" not in calls[1][0][0] 401 | 402 | # Check filtered queries 403 | assert "--query" in calls[2][0][0] 404 | assert "events[?severity=='critical']" in calls[2][0][0] 405 | 406 | assert "--query" in calls[3][0][0] 407 | assert "events[?severity=='high']" in calls[3][0][0] 408 | 409 | 410 | @mock.patch("execute.subprocess.run") 411 | @mock.patch("execute.get_output_configs_for_query") 412 | def test_error_handling_query_execution(mock_get_configs, mock_subprocess_run, caplog): 413 | """Test handling of query execution errors.""" 414 | # Setup mocks 415 | mock_get_configs.return_value = [OutputConfig(format=OutputFormat.JSON)] 416 | mock_subprocess_run.side_effect = subprocess.CalledProcessError( 417 | 1, "az monitor", stderr="Error: Invalid KQL query syntax at line 5" 418 | ) 419 | 420 | # Execute query 421 | result = execute_query( 422 | folder_path="/test", 423 | file_path="query.kql", 424 | workspace_id="test-workspace", 425 | config=KQLConfig(), 426 | ) 427 | 428 | # Verify 429 | assert result is False 430 | assert "Error executing query" in caplog.text 431 | assert "Error details: Error: Invalid KQL query syntax at line 5" in caplog.text 432 | 433 | 434 | @mock.patch("execute.subprocess.run") 435 | @mock.patch("execute.get_output_configs_for_query") 436 | @mock.patch("os.makedirs") 437 | def test_error_handling_file_operations( 438 | mock_makedirs, mock_get_configs, mock_subprocess_run, caplog 439 | ): 440 | """Test handling of file operation errors.""" 441 | # Setup mocks 442 | mock_get_configs.return_value = [ 443 | OutputConfig(format=OutputFormat.JSON, file="results/output.json") 444 | ] 445 | mock_subprocess_run.return_value = mock.MagicMock( 446 | stdout="test output", stderr="", returncode=0 447 | ) 448 | mock_makedirs.side_effect = PermissionError("Permission denied") 449 | 450 | # Execute query 451 | result = execute_query( 452 | folder_path="/test", 453 | file_path="query.kql", 454 | workspace_id="test-workspace", 455 | config=KQLConfig(), 456 | ) 457 | 458 | # Verify 459 | assert result is False 460 | assert "Unexpected error processing" in caplog.text 461 | assert "Permission denied" in caplog.text 462 | 463 | 464 | @mock.patch("execute.get_output_configs_for_query") 465 | def test_empty_configurations(mock_get_configs, caplog): 466 | """Test behavior with empty configurations.""" 467 | # Configure caplog to capture the right log level 468 | caplog.set_level(logging.INFO) 469 | 470 | # Setup mocks to return empty list 471 | mock_get_configs.return_value = [] 472 | 473 | # Execute query 474 | result = execute_query( 475 | folder_path="/test", 476 | file_path="query.kql", 477 | workspace_id="test-workspace", 478 | config=KQLConfig(), 479 | ) 480 | 481 | # Verify 482 | assert result is True # Should succeed even with no outputs 483 | assert "Executing /test/query.kql" in caplog.text 484 | 485 | 486 | @mock.patch("execute.subprocess.run") 487 | @mock.patch("execute.get_output_configs_for_query") 488 | def test_network_events_count(mock_get_configs, mock_subprocess_run): 489 | """Test example from docs: network events with count query.""" 490 | # Setup mocks 491 | mock_get_configs.return_value = [ 492 | OutputConfig(format=OutputFormat.JSON, query="length(@)") 493 | ] 494 | mock_subprocess_run.return_value = mock.MagicMock( 495 | stdout="42", stderr="", returncode=0 496 | ) 497 | 498 | # Execute query 499 | result = execute_query( 500 | folder_path="/test", 501 | file_path="network-events.kql", 502 | workspace_id="test-workspace", 503 | config=KQLConfig(), 504 | ) 505 | 506 | # Verify 507 | assert result is True 508 | cmd = mock_subprocess_run.call_args[0][0] 509 | assert "--query" in cmd 510 | assert "length(@)" in cmd 511 | 512 | 513 | ### Examples from Configuration Guide ### 514 | @mock.patch("execute.subprocess.run") 515 | @mock.patch("execute.get_output_configs_for_query") 516 | def test_example_basic_configuration(mock_get_configs, mock_subprocess_run): 517 | """Test the basic configuration example from documentation.""" 518 | # Setup mocks 519 | mock_get_configs.return_value = [ 520 | OutputConfig(format=OutputFormat.JSONC), # Console output 521 | OutputConfig( 522 | format=OutputFormat.JSON, 523 | file="query-results/output.json", 524 | compression=CompressionType.GZIP, 525 | ), # File output with compression 526 | ] 527 | mock_subprocess_run.return_value = mock.MagicMock( 528 | stdout="test output", stderr="", returncode=0 529 | ) 530 | 531 | # Execute query 532 | result = execute_query( 533 | folder_path="/test", 534 | file_path="example.kql", 535 | workspace_id="test-workspace", 536 | config=KQLConfig(), 537 | ) 538 | 539 | # Verify 540 | assert result is True 541 | assert mock_subprocess_run.call_count == 2 542 | 543 | # Check first call (JSONC to console) 544 | first_call = mock_subprocess_run.call_args_list[0] 545 | assert "--output" in first_call[0][0] 546 | assert "jsonc" in first_call[0][0] 547 | 548 | # Check second call (JSON to file with compression) 549 | second_call = mock_subprocess_run.call_args_list[1] 550 | assert "--output" in second_call[0][0] 551 | assert "json" in second_call[0][0] 552 | 553 | 554 | @mock.patch("execute.subprocess.run") 555 | @mock.patch("execute.get_output_configs_for_query") 556 | def test_example_multiple_queries(mock_get_configs, mock_subprocess_run): 557 | """Test the multiple queries example from documentation.""" 558 | test_cases = [ 559 | # device.kql with no output 560 | ( 561 | "device.kql", 562 | [OutputConfig(format=OutputFormat.NONE)], 563 | 0, # Expected call count 564 | ), 565 | # user.kql with YAML output 566 | ( 567 | "user.kql", 568 | [OutputConfig(format=OutputFormat.YAML)], 569 | 1, # Expected call count 570 | ), 571 | # network/nsg.kql with TSV output 572 | ( 573 | "network/nsg.kql", 574 | [OutputConfig(format=OutputFormat.TSV)], 575 | 1, # Expected call count 576 | ), 577 | # network/vm.kql with table output 578 | ( 579 | "network/vm.kql", 580 | [OutputConfig(format=OutputFormat.TABLE, file="subdir/vm.txt")], 581 | 1, # Expected call count 582 | ), 583 | ] 584 | 585 | for file_path, configs, expected_calls in test_cases: 586 | # Reset mocks 587 | mock_subprocess_run.reset_mock() 588 | mock_get_configs.return_value = configs 589 | mock_subprocess_run.return_value = mock.MagicMock( 590 | stdout="test output", stderr="", returncode=0 591 | ) 592 | 593 | # Execute query 594 | result = execute_query( 595 | folder_path="/test", 596 | file_path=file_path, 597 | workspace_id="test-workspace", 598 | config=KQLConfig(), 599 | ) 600 | 601 | # Verify 602 | assert result is True 603 | assert mock_subprocess_run.call_count == expected_calls 604 | 605 | # Check format-specific details 606 | if configs[0].format != OutputFormat.NONE: 607 | cmd = mock_subprocess_run.call_args[0][0] 608 | assert "--output" in cmd 609 | assert configs[0].format.value in cmd 610 | 611 | 612 | @mock.patch("execute.subprocess.run") 613 | @mock.patch("execute.get_output_configs_for_query") 614 | def test_example_security_events_filtering(mock_get_configs, mock_subprocess_run): 615 | """Test the security events filtering example from documentation.""" 616 | # Setup mocks 617 | mock_get_configs.return_value = [ 618 | OutputConfig(format=OutputFormat.JSONC), 619 | OutputConfig( 620 | format=OutputFormat.JSON, 621 | file="logs/security-events.json", 622 | compression=CompressionType.GZIP, 623 | ), 624 | *[ 625 | OutputConfig( 626 | format=OutputFormat.JSON, 627 | query=f"events[?severity=='{severity}']", 628 | file=f"alerts/{severity}.json", 629 | ) 630 | for severity in ["critical", "high", "medium", "low"] 631 | ], 632 | ] 633 | mock_subprocess_run.return_value = mock.MagicMock( 634 | stdout="test output", stderr="", returncode=0 635 | ) 636 | 637 | # Execute query 638 | result = execute_query( 639 | folder_path="/test", 640 | file_path="security-events.kql", 641 | workspace_id="test-workspace", 642 | config=KQLConfig(), 643 | ) 644 | 645 | # Verify 646 | assert result is True 647 | assert ( 648 | mock_subprocess_run.call_count == 6 649 | ) # 1 console + 1 compressed + 4 severity filters 650 | 651 | calls = mock_subprocess_run.call_args_list 652 | 653 | # Check format and queries for each call 654 | # First call: JSONC to console 655 | assert "jsonc" in calls[0][0][0] 656 | assert "--query" not in calls[0][0][0] 657 | 658 | # Second call: JSON with compression 659 | assert "json" in calls[1][0][0] 660 | assert "--query" not in calls[1][0][0] 661 | 662 | # Remaining calls: Severity-filtered outputs 663 | severities = ["critical", "high", "medium", "low"] 664 | for idx, severity in enumerate(severities, start=2): 665 | cmd = calls[idx][0][0] 666 | assert "json" in cmd 667 | assert "--query" in cmd 668 | assert f"events[?severity=='{severity}']" in cmd 669 | --------------------------------------------------------------------------------