├── .github ├── dependabot.yml ├── grimoire.png └── workflows │ └── release.yml ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── README.md ├── cmd └── grimoire │ └── main.go ├── go.mod ├── go.sum ├── install.sh └── internal ├── config ├── config.go └── defaults.go ├── core ├── git.go ├── git_test.go ├── runner.go └── walker.go ├── secrets ├── gitleaks.go └── gitleaks.toml ├── serializer ├── detector.go ├── markdown.go ├── serializer.go ├── text.go ├── tree_generator.go └── xml.go └── tokens ├── counter.go └── writer.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | day: thursday 7 | interval: weekly 8 | - package-ecosystem: gomod 9 | directory: / 10 | schedule: 11 | day: thursday 12 | interval: weekly -------------------------------------------------------------------------------- /.github/grimoire.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/foresturquhart/grimoire/6e2bce94bb3f7dfc0f7416792b14261ddee27c7d/.github/grimoire.png -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release Grimoire 2 | permissions: 3 | contents: write 4 | on: 5 | push: 6 | tags: 7 | - 'v*' 8 | 9 | env: 10 | GO_VERSION: 1.24.5 11 | 12 | jobs: 13 | release: 14 | name: Release 15 | strategy: 16 | matrix: 17 | platform: [ubuntu-latest] 18 | runs-on: ${{ matrix.platform }} 19 | steps: 20 | 21 | - name: Set up Go 22 | uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 23 | with: 24 | go-version: ${{ env.GO_VERSION }} 25 | 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 28 | with: 29 | fetch-depth: 0 30 | 31 | - name: Run GoReleaser 32 | uses: goreleaser/goreleaser-action@9c156ee8a17a598857849441385a2041ef570552 33 | with: 34 | version: latest 35 | args: release -f .goreleaser.yml --clean 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.out 2 | coverage.txt 3 | coverage.html 4 | dist/ 5 | .idea 6 | .vscode -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | release: 4 | github: 5 | owner: foresturquhart 6 | name: grimoire 7 | 8 | before: 9 | hooks: 10 | - go mod tidy 11 | 12 | builds: 13 | - goos: 14 | - linux 15 | - windows 16 | - darwin 17 | goarch: 18 | - amd64 19 | - arm64 20 | main: ./cmd/grimoire 21 | binary: grimoire 22 | 23 | archives: 24 | - formats: [ 'tar.gz' ] 25 | wrap_in_directory: true 26 | format_overrides: 27 | - goos: windows 28 | formats: [ 'zip' ] 29 | name_template: '{{ .ProjectName }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 30 | files: 31 | - LICENSE 32 | - README.md 33 | 34 | checksum: 35 | algorithm: sha256 36 | name_template: '{{ .ProjectName }}-{{ .Version }}-checksums.txt' -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Forest Urquhart 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | Grimoire 4 | 5 |
6 | 7 |
8 | 9 | [![go](https://badgen.net/static/go/1.24.5)](https://go.dev/) 10 | [![license](https://badgen.net/github/license/foresturquhart/grimoire)](https://github.com/foresturquhart/grimoire/blob/main/LICENSE) 11 | [![release](https://badgen.net/github/release/foresturquhart/grimoire/stable)](https://github.com/foresturquhart/grimoire/releases) 12 | 13 | Grimoire is a command-line tool that converts the contents of a directory into structured output formats optimized for interpretation by Large Language Models (LLMs) like Claude, ChatGPT, Gemini, DeepSeek, Llama, Grok, and more. It is lightweight, highly configurable, and user-friendly. 14 | 15 | ## Quick Start 16 | 17 | ### Install Grimoire 18 | 19 | The fastest way to get started is with our one-line installation script: 20 | 21 | ```bash 22 | curl -sSL https://raw.githubusercontent.com/foresturquhart/grimoire/main/install.sh | bash 23 | ``` 24 | 25 | This script automatically detects your OS and architecture, downloads the appropriate binary, and installs it to your PATH. 26 | 27 | ### Basic Usage 28 | 29 | ```bash 30 | # Convert current directory to Markdown and copy to clipboard (macOS) 31 | grimoire . | pbcopy 32 | 33 | # Convert current directory to Markdown and copy to clipboard (Linux with xclip) 34 | grimoire . | xclip -selection clipboard 35 | 36 | # Convert current directory and save to file 37 | grimoire -o output.md . 38 | 39 | # Convert current directory to XML format 40 | grimoire --format xml -o output.xml . 41 | 42 | # Convert current directory to plain text format 43 | grimoire --format txt -o output.txt . 44 | 45 | # Convert directory with secret detection and redaction 46 | grimoire --redact-secrets -o output.md . 47 | ``` 48 | 49 | ## Features 50 | 51 | * **Multiple Output Formats:** Generate output in Markdown, XML, or plain text formats to suit your needs. 52 | * **Recursive File Scanning:** Automatically traverses directories and subdirectories to identify eligible files based on customizable extensions. 53 | * **Content Filtering:** Skips ignored directories, temporary files, and patterns defined in the configuration. 54 | * **Directory Tree Visualization:** Includes an optional directory structure representation at the beginning of the output. 55 | * **Git Integration:** Prioritizes files by commit frequency when working within a Git repository. 56 | * **Secret Detection:** Scans files for potential secrets or sensitive information to prevent accidental exposure. 57 | * **Secret Redaction:** Optionally redacts detected secrets in the output while preserving the overall code structure. 58 | * **Token Counting:** Calculates the token count of generated output to help manage LLM context limits. 59 | * **Minified File Detection:** Automatically identifies minified JavaScript and CSS files to warn about high token usage. 60 | * **Flexible Output:** Supports output to stdout or a specified file. 61 | 62 | ## Installation 63 | 64 | ### Prerequisites 65 | 66 | * Git (required for repositories using Git-based sorting). 67 | 68 | ### Quickest: One-line Installation Script 69 | 70 | The easiest way to install Grimoire is with our automatic installation script: 71 | 72 | ```bash 73 | curl -sSL https://raw.githubusercontent.com/foresturquhart/grimoire/main/install.sh | bash 74 | ``` 75 | 76 | This script automatically: 77 | - Detects your operating system and architecture 78 | - Downloads the appropriate binary for your system 79 | - Installs it to `/usr/local/bin` (or `~/.local/bin` if you don't have sudo access) 80 | - Makes the binary executable 81 | 82 | ### Alternative: Download Pre-compiled Binary 83 | 84 | You can also manually download a pre-compiled binary from the [releases page](https://github.com/foresturquhart/grimoire/releases). 85 | 86 | 1. Visit the [releases page](https://github.com/foresturquhart/grimoire/releases). 87 | 2. Download the appropriate archive for your system (e.g., `grimoire-1.2.2-linux-amd64.tar.gz` or `grimoire-1.2.2-darwin-arm64.tar.gz`). 88 | 3. Extract the archive to retrieve the `grimoire` executable. 89 | 4. Move the `grimoire` executable to a directory in your system's `PATH` (e.g., `/usr/local/bin` or `~/.local/bin`). You may need to use `sudo` for system-wide locations: 90 | ```bash 91 | tar -xzf grimoire-1.2.2-linux-amd64.tar.gz 92 | cd grimoire-1.2.2-linux-amd64 93 | sudo mv grimoire /usr/local/bin/ 94 | ``` 95 | 5. Verify the installation: 96 | ```bash 97 | grimoire --version 98 | ``` 99 | 100 | ### Install using `go install` 101 | 102 | For users with Go installed, `go install` offers a straightforward installation method: 103 | 104 | ```bash 105 | go install github.com/foresturquhart/grimoire/cmd/grimoire@latest 106 | ``` 107 | 108 | ### Build from Source 109 | 110 | To build Grimoire from source (useful for development or customization): 111 | 112 | 1. Clone the repository: 113 | ```bash 114 | git clone https://github.com/foresturquhart/grimoire.git 115 | cd grimoire 116 | ``` 117 | 2. Build the binary: 118 | ```bash 119 | go build -o grimoire ./cmd/grimoire 120 | ``` 121 | 3. Move the binary to your `PATH`: 122 | ```bash 123 | mv grimoire /usr/local/bin/ 124 | ``` 125 | 4. Verify the installation: 126 | ```bash 127 | grimoire --version 128 | ``` 129 | 130 | ## Usage 131 | 132 | ### Basic Command 133 | 134 | ```bash 135 | grimoire [options] 136 | ``` 137 | 138 | ### Options 139 | 140 | - `-o, --output `: Specify an output file. Defaults to stdout if omitted. 141 | - `-f, --force`: Overwrite the output file if it already exists. 142 | - `--format `: Specify the output format. Options are `md` (or `markdown`), `xml`, and `txt` (or `text`, `plain`, `plaintext`). Defaults to `md`. 143 | - `--no-tree`: Disable the directory tree visualization at the beginning of the output. 144 | - `--no-sort`: Disable sorting files by Git commit frequency. 145 | - `--ignore-secrets`: Proceed with output generation even if secrets are detected. 146 | - `--redact-secrets`: Redact detected secrets in output rather than failing. 147 | - `--skip-token-count`: Skip counting output tokens. 148 | - `--version`: Display the current version. 149 | 150 | ### Examples 151 | 152 | 1. Convert a directory into Markdown and print the output to stdout: 153 | ```bash 154 | grimoire ./myproject 155 | ``` 156 | 2. Save the output to a file: 157 | ```bash 158 | grimoire -o output.md ./myproject 159 | ``` 160 | 3. Overwrite an existing file: 161 | ```bash 162 | grimoire -o output.md -f ./myproject 163 | ``` 164 | 4. Generate XML output without a directory tree: 165 | ```bash 166 | grimoire --format xml --no-tree -o output.xml ./myproject 167 | ``` 168 | 5. Generate plain text output without Git-based sorting: 169 | ```bash 170 | grimoire --format txt --no-sort -o output.txt ./myproject 171 | ``` 172 | 6. Scan for secrets and redact them in the output: 173 | ```bash 174 | grimoire --redact-secrets -o output.md ./myproject 175 | ``` 176 | 177 | ## Configuration 178 | 179 | ### Allowed File Extensions 180 | 181 | Grimoire processes files with specific extensions. You can customize these by modifying the `DefaultAllowedFileExtensions` constant in the codebase. 182 | 183 | ### Ignored Path Patterns 184 | 185 | Files and directories matching patterns in the `DefaultIgnoredPathPatterns` constant are excluded from processing. This includes temporary files, build artifacts, and version control directories. 186 | 187 | ### Custom Ignore Files 188 | 189 | Grimoire supports two types of ignore files to specify additional exclusion patterns: 190 | 191 | 1. **`.gitignore`**: Standard Git ignore files are honored if present in the target directory. 192 | 2. **`.grimoireignore`**: Grimoire-specific ignore files that follow the same syntax as Git ignore files. 193 | 194 | These files allow you to specify additional ignore rules on a per-directory basis, giving you fine-grained control over which files and directories should be omitted during the conversion process. 195 | 196 | ### Large File Handling 197 | 198 | By default, Grimoire warns when processing files larger than 1MB. These files are still included in the output, but a warning is logged to alert you about potential performance impacts when feeding the output to an LLM. 199 | 200 | ### Minified File Detection 201 | 202 | Grimoire automatically detects minified JavaScript and CSS files using several heuristics: 203 | - Excessive line length 204 | - Low ratio of lines to characters 205 | - Presence of coding patterns typical in minified files 206 | 207 | When a minified file is detected, Grimoire logs a warning, as these files can consume a large number of tokens while providing limited value to the LLM. 208 | 209 | ## Output Formats 210 | 211 | Grimoire supports three output formats: 212 | 213 | 1. **Markdown (md)** - Default format that wraps file contents in code blocks with file paths as headings. 214 | 2. **XML (xml)** - Structures the content in an XML format with file paths as attributes. 215 | 3. **Plain Text (txt)** - Uses separator lines to distinguish between files. 216 | 217 | Each format includes metadata, a summary section, an optional directory tree, and the content of all files. 218 | 219 | ## Token Counting 220 | 221 | Grimoire includes built-in token counting to help you manage LLM context limits. The token count is estimated using the same tokenizer used by many LLMs. You can disable token counting entirely using the `--skip-token-count` flag. 222 | 223 | ## Secret Detection 224 | 225 | Grimoire includes built-in secret detection powered by [gitleaks](https://github.com/gitleaks/gitleaks) to help prevent accidentally sharing sensitive information when using the generated output with LLMs or other tools. 226 | 227 | By default, Grimoire scans for a wide variety of potential secrets including: 228 | 229 | - API keys and tokens (AWS, GitHub, GitLab, etc.) 230 | - Private keys (RSA, SSH, PGP, etc.) 231 | - Authentication credentials 232 | - Service-specific tokens (Stripe, Slack, Twilio, etc.) 233 | 234 | The secret detection behavior can be controlled with the following flags: 235 | 236 | - `--ignore-secrets`: Continues with output generation even if secrets are detected (logs warnings) 237 | - `--redact-secrets`: Automatically redacts any detected secrets with the format `[REDACTED SECRET: description]` 238 | 239 | If a secret is detected and neither of the above flags are specified, Grimoire will abort the operation and display a warning message, helping prevent accidental exposure of sensitive information. 240 | 241 | ## Contributing 242 | 243 | Contributions are welcome! To get started: 244 | 245 | 1. Fork the repository. 246 | 2. Create a new branch for your feature or fix: 247 | ```bash 248 | git checkout -b feature/my-new-feature 249 | ``` 250 | 3. Commit your changes: 251 | ```bash 252 | git commit -m "Add my new feature" 253 | ``` 254 | 4. Push the branch to your fork: 255 | ```bash 256 | git push origin feature/my-new-feature 257 | ``` 258 | 5. Open a pull request. 259 | 260 | ## License 261 | 262 | Grimoire is licensed under the [MIT License](LICENSE). 263 | 264 | ## Acknowledgements 265 | 266 | Grimoire uses the following libraries: 267 | 268 | - [zerolog](https://github.com/rs/zerolog) for structured logging. 269 | - [gitleaks](https://github.com/zricethezav/gitleaks) for secret detection and scanning. 270 | - [go-gitignore](https://github.com/sabhiram/go-gitignore) for handling ignore patterns. 271 | - [urfave/cli](https://github.com/urfave/cli) for command-line interface. 272 | - [tiktoken-go](https://github.com/tiktoken-go/tokenizer) for token counting. 273 | 274 | ## Feedback and Support 275 | 276 | For issues, suggestions, or feedback, please open an issue on the [GitHub repository](https://github.com/foresturquhart/grimoire/issues). -------------------------------------------------------------------------------- /cmd/grimoire/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/foresturquhart/grimoire/internal/config" 8 | "github.com/foresturquhart/grimoire/internal/core" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | "github.com/urfave/cli/v3" 12 | ) 13 | 14 | // Version is injected at build time 15 | var version = "dev" 16 | 17 | func main() { 18 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 19 | 20 | cmd := &cli.Command{ 21 | Name: "grimoire", 22 | Usage: "convert a directory to content suitable for LLM interpretation.", 23 | Version: version, 24 | ArgsUsage: "[target directory]", 25 | Flags: []cli.Flag{ 26 | &cli.StringFlag{ 27 | Name: "output", 28 | Aliases: []string{"o"}, 29 | Usage: "Output file path.", 30 | }, 31 | &cli.BoolFlag{ 32 | Name: "force", 33 | Aliases: []string{"f"}, 34 | Usage: "Overwrite existing file without prompt.", 35 | }, 36 | &cli.BoolFlag{ 37 | Name: "no-tree", 38 | Usage: "Disable directory tree display at the beginning of output.", 39 | }, 40 | &cli.BoolFlag{ 41 | Name: "no-sort", 42 | Usage: "Disable sorting files by Git commit frequency.", 43 | }, 44 | &cli.BoolFlag{ 45 | Name: "ignore-secrets", 46 | Usage: "Proceed with output generation even if secrets are detected.", 47 | }, 48 | &cli.BoolFlag{ 49 | Name: "redact-secrets", 50 | Usage: "Redact detected secrets in output rather than failing.", 51 | }, 52 | &cli.BoolFlag{ 53 | Name: "skip-token-count", 54 | Usage: "Skip counting output tokens.", 55 | }, 56 | &cli.StringFlag{ 57 | Name: "token-count-mode", 58 | Usage: "Token counting mode: fast (default) or exact (slower).", 59 | Value: "fast", 60 | }, 61 | &cli.StringFlag{ 62 | Name: "format", 63 | Usage: "Output format (md, xml, or txt). Defaults to md.", 64 | Value: "md", 65 | }, 66 | &cli.IntFlag{ 67 | Name: "high-token-threshold", 68 | Usage: "Threshold for warning about files with high token counts. Defaults to 5000.", 69 | Value: 5000, 70 | }, 71 | }, 72 | Action: func(ctx context.Context, cmd *cli.Command) error { 73 | return core.Run( 74 | config.NewConfigFromCommand(cmd), 75 | ) 76 | }, 77 | } 78 | 79 | if err := cmd.Run(context.Background(), os.Args); err != nil { 80 | log.Fatal().Msg(err.Error()) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/foresturquhart/grimoire 2 | 3 | go 1.24.5 4 | 5 | require ( 6 | github.com/BurntSushi/toml v1.5.0 7 | github.com/rs/zerolog v1.34.0 8 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 9 | github.com/tiktoken-go/tokenizer v0.6.2 10 | github.com/urfave/cli/v3 v3.3.8 11 | github.com/zricethezav/gitleaks/v8 v8.28.0 12 | golang.org/x/text v0.27.0 13 | ) 14 | 15 | require ( 16 | dario.cat/mergo v1.0.1 // indirect 17 | github.com/BobuSumisu/aho-corasick v1.0.3 // indirect 18 | github.com/Masterminds/goutils v1.1.1 // indirect 19 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 20 | github.com/Masterminds/sprig/v3 v3.3.0 // indirect 21 | github.com/STARRY-S/zip v0.2.1 // indirect 22 | github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect 23 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 24 | github.com/bodgit/plumbing v1.3.0 // indirect 25 | github.com/bodgit/sevenzip v1.6.0 // indirect 26 | github.com/bodgit/windows v1.0.1 // indirect 27 | github.com/charmbracelet/lipgloss v0.5.0 // indirect 28 | github.com/dlclark/regexp2 v1.11.5 // indirect 29 | github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect 30 | github.com/fatih/semgroup v1.2.0 // indirect 31 | github.com/fsnotify/fsnotify v1.8.0 // indirect 32 | github.com/gitleaks/go-gitdiff v0.9.1 // indirect 33 | github.com/google/uuid v1.6.0 // indirect 34 | github.com/h2non/filetype v1.1.3 // indirect 35 | github.com/hashicorp/errwrap v1.1.0 // indirect 36 | github.com/hashicorp/go-multierror v1.1.1 // indirect 37 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 38 | github.com/hashicorp/hcl v1.0.0 // indirect 39 | github.com/huandu/xstrings v1.5.0 // indirect 40 | github.com/klauspost/compress v1.17.11 // indirect 41 | github.com/klauspost/pgzip v1.2.6 // indirect 42 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 43 | github.com/magiconair/properties v1.8.9 // indirect 44 | github.com/mattn/go-colorable v0.1.14 // indirect 45 | github.com/mattn/go-isatty v0.0.20 // indirect 46 | github.com/mattn/go-runewidth v0.0.14 // indirect 47 | github.com/mholt/archives v0.1.2 // indirect 48 | github.com/minio/minlz v1.0.0 // indirect 49 | github.com/mitchellh/copystructure v1.2.0 // indirect 50 | github.com/mitchellh/mapstructure v1.5.0 // indirect 51 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 52 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect 53 | github.com/muesli/termenv v0.15.1 // indirect 54 | github.com/nwaples/rardecode/v2 v2.1.0 // indirect 55 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 56 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 57 | github.com/rivo/uniseg v0.2.0 // indirect 58 | github.com/sagikazarmark/locafero v0.7.0 // indirect 59 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 60 | github.com/shopspring/decimal v1.4.0 // indirect 61 | github.com/sorairolake/lzip-go v0.3.5 // indirect 62 | github.com/sourcegraph/conc v0.3.0 // indirect 63 | github.com/spf13/afero v1.12.0 // indirect 64 | github.com/spf13/cast v1.7.1 // indirect 65 | github.com/spf13/pflag v1.0.6 // indirect 66 | github.com/spf13/viper v1.19.0 // indirect 67 | github.com/subosito/gotenv v1.6.0 // indirect 68 | github.com/tetratelabs/wazero v1.9.0 // indirect 69 | github.com/therootcompany/xz v1.0.1 // indirect 70 | github.com/ulikunitz/xz v0.5.12 // indirect 71 | github.com/wasilibs/go-re2 v1.9.0 // indirect 72 | github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect 73 | go.uber.org/multierr v1.11.0 // indirect 74 | go4.org v0.0.0-20230225012048-214862532bf5 // indirect 75 | golang.org/x/crypto v0.35.0 // indirect 76 | golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect 77 | golang.org/x/sync v0.16.0 // indirect 78 | golang.org/x/sys v0.30.0 // indirect 79 | gopkg.in/ini.v1 v1.67.0 // indirect 80 | gopkg.in/yaml.v3 v3.0.1 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 10 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 11 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 12 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 13 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 14 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 15 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 16 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 17 | dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 18 | dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 19 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 20 | github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= 21 | github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= 22 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 23 | github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= 24 | github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 25 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 26 | github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= 27 | github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 28 | github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= 29 | github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= 30 | github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= 31 | github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= 32 | github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= 33 | github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= 34 | github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= 35 | github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 36 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 37 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 38 | github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= 39 | github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= 40 | github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= 41 | github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= 42 | github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= 43 | github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= 44 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 45 | github.com/charmbracelet/lipgloss v0.5.0 h1:lulQHuVeodSgDez+3rGiuxlPVXSnhth442DATR2/8t8= 46 | github.com/charmbracelet/lipgloss v0.5.0/go.mod h1:EZLha/HbzEt7cYqdFPovlqy5FZPj0xFhg5SaqxScmgs= 47 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 48 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 49 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 50 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 51 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 52 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 53 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 54 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 55 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= 57 | github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= 58 | github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= 59 | github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= 60 | github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= 61 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 62 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 63 | github.com/fatih/semgroup v1.2.0 h1:h/OLXwEM+3NNyAdZEpMiH1OzfplU09i2qXPVThGZvyg= 64 | github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS879kgm8= 65 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 66 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 67 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 68 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 69 | github.com/gitleaks/go-gitdiff v0.9.1 h1:ni6z6/3i9ODT685OLCTf+s/ERlWUNWQF4x1pvoNICw0= 70 | github.com/gitleaks/go-gitdiff v0.9.1/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= 71 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 72 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 73 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 74 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 75 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 76 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 77 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 78 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 79 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 80 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 81 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 82 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 83 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 84 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 85 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 86 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 87 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 88 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 89 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 90 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 91 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 92 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 93 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 94 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 95 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 96 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 97 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 98 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 99 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 100 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 101 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 102 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 103 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 104 | github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= 105 | github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= 106 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 107 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 108 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 109 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 110 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 111 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 112 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 113 | github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= 114 | github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 115 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 116 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 117 | github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= 118 | github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 119 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 120 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 121 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 122 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 123 | github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 124 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 125 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 126 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 127 | github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= 128 | github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= 129 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 130 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 131 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 132 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 133 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 134 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 135 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 136 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 137 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 138 | github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= 139 | github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 140 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 141 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 142 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 143 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 144 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 145 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 146 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 147 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 148 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 149 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 150 | github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= 151 | github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 152 | github.com/mholt/archives v0.1.2 h1:UBSe5NfYKHI1sy+S5dJsEsG9jsKKk8NJA4HCC+xTI4A= 153 | github.com/mholt/archives v0.1.2/go.mod h1:D7QzTHgw3ctfS6wgOO9dN+MFgdZpbksGCxprUOwZWDs= 154 | github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ= 155 | github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= 156 | github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= 157 | github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= 158 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 159 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 160 | github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= 161 | github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 162 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 h1:y1p/ycavWjGT9FnmSjdbWUlLGvcxrY0Rw3ATltrxOhk= 163 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68/go.mod h1:Xk+z4oIWdQqJzsxyjgl3P22oYZnHdZ8FFTHAQQt5BMQ= 164 | github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= 165 | github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= 166 | github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= 167 | github.com/nwaples/rardecode/v2 v2.1.0 h1:JQl9ZoBPDy+nIZGb1mx8+anfHp/LV3NE2MjMiv0ct/U= 168 | github.com/nwaples/rardecode/v2 v2.1.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= 169 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 170 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 171 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 172 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 173 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 174 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 175 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 176 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 177 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 178 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 179 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 180 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 181 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 182 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 183 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 184 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 185 | github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= 186 | github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= 187 | github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= 188 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= 189 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= 190 | github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= 191 | github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= 192 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 193 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 194 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= 195 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= 196 | github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= 197 | github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= 198 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 199 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 200 | github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= 201 | github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= 202 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 203 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 204 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 205 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 206 | github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= 207 | github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= 208 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 209 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 210 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 211 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 212 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 213 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 214 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 215 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 216 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 217 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 218 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 219 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 220 | github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= 221 | github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= 222 | github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= 223 | github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= 224 | github.com/tiktoken-go/tokenizer v0.6.2 h1:t0GN2DvcUZSFWT/62YOgoqb10y7gSXBGs0A+4VCQK+g= 225 | github.com/tiktoken-go/tokenizer v0.6.2/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w= 226 | github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= 227 | github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= 228 | github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= 229 | github.com/urfave/cli/v3 v3.3.8 h1:BzolUExliMdet9NlJ/u4m5vHSotJ3PzEqSAZ1oPMa/E= 230 | github.com/urfave/cli/v3 v3.3.8/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZtpYpo= 231 | github.com/wasilibs/go-re2 v1.9.0 h1:kjAd8qbNvV4Ve2Uf+zrpTCrDHtqH4dlsRXktywo73JQ= 232 | github.com/wasilibs/go-re2 v1.9.0/go.mod h1:0sRtscWgpUdNA137bmr1IUgrRX0Su4dcn9AEe61y+yI= 233 | github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= 234 | github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= 235 | github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= 236 | github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= 237 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 238 | github.com/zricethezav/gitleaks/v8 v8.28.0 h1:XXeibrt4XbdrYm3FnzXR3uUPs9HbgGduroICjBl6PMw= 239 | github.com/zricethezav/gitleaks/v8 v8.28.0/go.mod h1:hcFf0KivlxYt85WxJ0wtUB75OR9qVneuD1OwauHOHx0= 240 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 241 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 242 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 243 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 244 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 245 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 246 | go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= 247 | go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= 248 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 249 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 250 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 251 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 252 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 253 | golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= 254 | golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= 255 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 256 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 257 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 258 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 259 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 260 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 261 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 262 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 263 | golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= 264 | golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= 265 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 266 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 267 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 268 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 269 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 270 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 271 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 272 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 273 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 274 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 275 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 276 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 277 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 278 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 279 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 280 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 281 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 282 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 283 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 284 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 285 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 286 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 287 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 288 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 289 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 290 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 291 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 292 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 293 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 294 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 295 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 296 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 297 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 298 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 299 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 300 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 301 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 302 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 303 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 304 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 305 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 306 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 307 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 308 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 309 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 310 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 311 | golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= 312 | golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 313 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 314 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 315 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 316 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 317 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 318 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 319 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 320 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 321 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 322 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 323 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 324 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 325 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 326 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 327 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 328 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 329 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 330 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 331 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 332 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 333 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 334 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 335 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 336 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 337 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 338 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 339 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 340 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 341 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 342 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 343 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 344 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 345 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 346 | golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= 347 | golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= 348 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 349 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 350 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 351 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 352 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 353 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 354 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 355 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 356 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 357 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 358 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 359 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 360 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 361 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 362 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 363 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 364 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 365 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 366 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 367 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 368 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 369 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 370 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 371 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 372 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 373 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 374 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 375 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 376 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 377 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 378 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 379 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 380 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 381 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 382 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 383 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 384 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 385 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 386 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 387 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 388 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 389 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 390 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 391 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 392 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 393 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 394 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 395 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 396 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 397 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 398 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 399 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 400 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 401 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 402 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 403 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 404 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 405 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 406 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 407 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 408 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 409 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 410 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 411 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 412 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 413 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 414 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 415 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 416 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 417 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 418 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 419 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 420 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 421 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 422 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 423 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 424 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 425 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 426 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 427 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 428 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 429 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 430 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Define variables 5 | GITHUB_REPO="foresturquhart/grimoire" 6 | INSTALL_DIR="/usr/local/bin" 7 | TMP_DIR="$(mktemp -d)" 8 | LATEST_RELEASE_URL="https://api.github.com/repos/${GITHUB_REPO}/releases/latest" 9 | USER_AGENT="Grimoire-Installer-Script" 10 | 11 | cleanup() { 12 | echo "Cleaning up temporary files..." 13 | rm -rf "${TMP_DIR}" 14 | } 15 | 16 | # Set up trap to clean up on exit 17 | trap cleanup EXIT 18 | 19 | # Detect OS and architecture 20 | OS="$(uname -s | tr '[:upper:]' '[:lower:]')" 21 | ARCH="$(uname -m)" 22 | 23 | # Map architecture names 24 | case "${ARCH}" in 25 | x86_64) 26 | ARCH="amd64" 27 | ;; 28 | aarch64|arm64) 29 | ARCH="arm64" 30 | ;; 31 | *) 32 | echo "Unsupported architecture: ${ARCH}" 33 | exit 1 34 | ;; 35 | esac 36 | 37 | # Handle OS-specific variations 38 | case "${OS}" in 39 | darwin) 40 | OS="darwin" 41 | EXT="tar.gz" 42 | ;; 43 | linux) 44 | OS="linux" 45 | EXT="tar.gz" 46 | ;; 47 | windows*) 48 | OS="windows" 49 | EXT="zip" 50 | ;; 51 | *) 52 | echo "Unsupported operating system: ${OS}" 53 | exit 1 54 | ;; 55 | esac 56 | 57 | echo "Detected OS: ${OS}, Architecture: ${ARCH}" 58 | 59 | # Check if curl is available 60 | if ! command -v curl &> /dev/null; then 61 | echo "Error: curl is required but not installed. Please install curl and try again." 62 | exit 1 63 | fi 64 | 65 | # Fetch the latest release information 66 | echo "Fetching latest release information..." 67 | RELEASE_INFO=$(curl -s -H "User-Agent: ${USER_AGENT}" "${LATEST_RELEASE_URL}") 68 | VERSION=$(echo "${RELEASE_INFO}" | grep -o '"tag_name":"[^"]*"' | sed 's/"tag_name":"//;s/"//g') 69 | 70 | if [ -z "${VERSION}" ]; then 71 | echo "Error: Could not extract version from release info." 72 | exit 1 73 | fi 74 | 75 | echo "Latest version: ${VERSION}" 76 | 77 | # Create asset name pattern based on the naming convention in .goreleaser.yml 78 | ASSET_NAME="grimoire-${VERSION#v}-${OS}-${ARCH}.${EXT}" 79 | 80 | # Extract the download URL for the specific asset - using exact matching 81 | DOWNLOAD_URL="" 82 | while read -r url; do 83 | # Extract just the filename from the URL 84 | filename=$(basename "$url") 85 | if [ "$filename" = "$ASSET_NAME" ]; then 86 | DOWNLOAD_URL="$url" 87 | break 88 | fi 89 | done < <(echo "$RELEASE_INFO" | grep -o '"browser_download_url":"[^"]*"' | sed 's/"browser_download_url":"//;s/"//g') 90 | 91 | if [ -z "${DOWNLOAD_URL}" ]; then 92 | echo "Error: Could not find download URL for Grimoire ${VERSION} on ${OS} ${ARCH}." 93 | exit 1 94 | fi 95 | 96 | # Create a temporary directory for downloading and extracting 97 | echo "Downloading Grimoire ${VERSION} for ${OS} ${ARCH}..." 98 | curl -L -o "${TMP_DIR}/${ASSET_NAME}" "${DOWNLOAD_URL}" 99 | 100 | # Extract the archive 101 | echo "Extracting archive..." 102 | cd "${TMP_DIR}" 103 | if [ "${EXT}" = "zip" ]; then 104 | # Check if unzip is available 105 | if ! command -v unzip &> /dev/null; then 106 | echo "Error: unzip is required but not installed. Please install unzip and try again." 107 | exit 1 108 | fi 109 | unzip -q "${ASSET_NAME}" 110 | else 111 | # tar should be available on all POSIX systems 112 | tar -xzf "${ASSET_NAME}" 113 | fi 114 | 115 | # Find the grimoire binary after extraction 116 | echo "Locating binary..." 117 | BINARY_PATH=$(find "${TMP_DIR}" -type f -name "grimoire" -o -name "grimoire.exe" | head -n 1) 118 | 119 | if [ -z "${BINARY_PATH}" ]; then 120 | echo "Error: Could not find grimoire binary in the extracted archive." 121 | exit 1 122 | fi 123 | 124 | # Installation logic based on permissions 125 | if [ "$(id -u)" -eq 0 ]; then 126 | # Running as root 127 | echo "Installing Grimoire to ${INSTALL_DIR}..." 128 | cp "${BINARY_PATH}" "${INSTALL_DIR}/grimoire" 129 | chmod +x "${INSTALL_DIR}/grimoire" 130 | else 131 | # Not running as root, try to install or guide the user 132 | if [ -w "${INSTALL_DIR}" ]; then 133 | echo "Installing Grimoire to ${INSTALL_DIR}..." 134 | cp "${BINARY_PATH}" "${INSTALL_DIR}/grimoire" 135 | chmod +x "${INSTALL_DIR}/grimoire" 136 | else 137 | if command -v sudo >/dev/null 2>&1; then 138 | echo "Installing Grimoire to ${INSTALL_DIR} (requires sudo)..." 139 | sudo cp "${BINARY_PATH}" "${INSTALL_DIR}/grimoire" 140 | sudo chmod +x "${INSTALL_DIR}/grimoire" 141 | else 142 | # No sudo, offer to install to user's bin directory 143 | USER_BIN="${HOME}/.local/bin" 144 | echo "Cannot install to ${INSTALL_DIR} (permission denied and sudo not available)" 145 | 146 | if [ ! -d "${USER_BIN}" ]; then 147 | echo "Creating directory ${USER_BIN}..." 148 | mkdir -p "${USER_BIN}" 149 | fi 150 | 151 | echo "Installing to ${USER_BIN} instead..." 152 | cp "${BINARY_PATH}" "${USER_BIN}/grimoire" 153 | chmod +x "${USER_BIN}/grimoire" 154 | 155 | if [[ ":${PATH}:" != *":${USER_BIN}:"* ]]; then 156 | echo "Note: ${USER_BIN} is not in your PATH. You may need to add it:" 157 | echo " export PATH=\"\$PATH:${USER_BIN}\"" 158 | echo "Add this line to your shell's profile file (~/.bashrc, ~/.zshrc, etc.) to make it permanent." 159 | fi 160 | fi 161 | fi 162 | fi 163 | 164 | # Verify installation 165 | echo "Verifying installation..." 166 | GRIMOIRE_PATH=$(command -v grimoire 2>/dev/null || echo "") 167 | 168 | if [ -n "${GRIMOIRE_PATH}" ]; then 169 | echo "Grimoire ${VERSION} has been successfully installed to ${GRIMOIRE_PATH}!" 170 | echo "Run 'grimoire --help' to get started" 171 | else 172 | echo "Installation completed, but grimoire command is not in your PATH yet." 173 | 174 | # Check if we installed to a non-PATH location 175 | if [ -x "${HOME}/.local/bin/grimoire" ]; then 176 | echo "You can run it with: ${HOME}/.local/bin/grimoire" 177 | echo "Or add ${HOME}/.local/bin to your PATH to use 'grimoire' directly." 178 | elif [ -x "${INSTALL_DIR}/grimoire" ]; then 179 | echo "You can run it with: ${INSTALL_DIR}/grimoire" 180 | echo "Make sure ${INSTALL_DIR} is in your PATH to use 'grimoire' directly." 181 | fi 182 | fi 183 | 184 | echo "Installation complete!" -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/urfave/cli/v3" 12 | ) 13 | 14 | // Config holds all configuration data needed to run the application. 15 | // This includes the target directory to walk, the path (if any) to write output, 16 | // and various options for filtering or overriding existing files. 17 | type Config struct { 18 | // TargetDir is the directory from which files will be walked. 19 | TargetDir string 20 | 21 | // OutputFile is the file where results may be written. If empty, 22 | // output is directed to stdout. 23 | OutputFile string 24 | 25 | // Force indicates whether existing output files should be overwritten. 26 | Force bool 27 | 28 | // ShowTree indicates whether to display a directory tree at the beginning of output. 29 | ShowTree bool 30 | 31 | // DisableSort indicates whether to skip sorting files by Git commit frequency. 32 | DisableSort bool 33 | 34 | // Format specifies the output format (e.g., "md" or "xml") 35 | Format string 36 | 37 | // AllowedFileExtensions is the list of file extensions that the walker should consider. 38 | AllowedFileExtensions map[string]bool 39 | 40 | // IgnoredPathRegexes is a set of compiled regex patterns for ignoring certain paths. 41 | IgnoredPathRegexes []*regexp.Regexp 42 | 43 | // IgnoreSecrets indicates whether to proceed with output generation even if secrets are detected. 44 | IgnoreSecrets bool 45 | 46 | // RedactSecrets indicates whether to redact detected secrets in the output. 47 | RedactSecrets bool 48 | 49 | // LargeFileSizeThreshold defines the size in bytes above which a file is considered "large" 50 | // and a warning will be logged. Default is 1MB. 51 | LargeFileSizeThreshold int64 52 | 53 | // HighTokenThreshold defines the token count above which a file is considered 54 | // to have a high token count and a warning will be logged. Default is 5000. 55 | HighTokenThreshold int 56 | 57 | // SkipTokenCount indicates whether to skip counting output tokens. 58 | SkipTokenCount bool 59 | } 60 | 61 | // NewConfigFromCommand constructs a Config by extracting relevant values from 62 | // the provided cli.Command. 63 | func NewConfigFromCommand(cmd *cli.Command) *Config { 64 | var err error 65 | 66 | // Extract the target directory from the command arguments. 67 | targetDir := cmd.Args().First() 68 | if targetDir == "" { 69 | log.Fatal().Msg("You must specify a target directory") 70 | } 71 | 72 | // Convert target directory to an absolute path. 73 | targetDir, err = filepath.Abs(targetDir) 74 | if err != nil { 75 | log.Fatal().Err(err).Msgf("Failed to resolve target directory %s", targetDir) 76 | } 77 | 78 | // Convert output file to an absolute path. 79 | outputFile := cmd.String("output") 80 | if outputFile != "" { 81 | outputFile, err = filepath.Abs(outputFile) 82 | if err != nil { 83 | log.Fatal().Err(err).Msgf("Failed to resolve output file %s", outputFile) 84 | } 85 | } 86 | 87 | // Fetch value of force command line flag. 88 | force := cmd.Bool("force") 89 | 90 | // Check if tree display is disabled (default is to show tree) 91 | showTree := !cmd.Bool("no-tree") 92 | 93 | // Check if sorting is disabled 94 | disableSort := cmd.Bool("no-sort") 95 | 96 | // Check if we should ignore detected secrets 97 | ignoreSecrets := cmd.Bool("ignore-secrets") 98 | 99 | // Check if we should redact detected secrets 100 | redactSecrets := cmd.Bool("redact-secrets") 101 | 102 | // Check if we should skip counting tokens 103 | skipTokenCount := cmd.Bool("skip-token-count") 104 | 105 | // Get output format 106 | format := cmd.String("format") 107 | // Validate and normalize format 108 | format = strings.ToLower(format) 109 | switch format { 110 | case "md", "markdown": 111 | format = "md" 112 | case "xml": 113 | format = "xml" 114 | case "txt", "text", "plain", "plaintext": 115 | format = "txt" 116 | default: 117 | if format != "" { 118 | log.Fatal().Msgf("Unsupported format: %s", format) 119 | } 120 | // Default to markdown if no format specified 121 | format = "md" 122 | } 123 | 124 | // If an output file is specified, and we are not forcing an overwrite, 125 | // check if the file already exists. 126 | if outputFile != "" && !force { 127 | _, err := os.Stat(outputFile) 128 | if err == nil { 129 | log.Fatal().Msgf("Output file %s already exists, use --force to overwrite", outputFile) 130 | } else if !os.IsNotExist(err) { 131 | log.Fatal().Err(err).Msgf("Error checking output file %s", outputFile) 132 | } 133 | } 134 | 135 | // Set allowed file extensions and ignored path patterns. 136 | allowedFileExtensions := DefaultAllowedFileExtensions 137 | ignoredPathPatterns := DefaultIgnoredPathPatterns 138 | 139 | allowedFileExtensionsMap := make(map[string]bool) 140 | for _, ext := range allowedFileExtensions { 141 | if !strings.HasPrefix(ext, ".") { 142 | ext = "." + ext 143 | } 144 | allowedFileExtensionsMap[ext] = true 145 | } 146 | 147 | // Compile the ignored path regexes. 148 | ignoredPathRegexes, err := compileRegexes(ignoredPathPatterns) 149 | if err != nil { 150 | log.Fatal().Err(err).Msgf("Failed to compile ignored path pattern regexes") 151 | } 152 | 153 | // Use the default large file size threshold 154 | largeFileSizeThreshold := DefaultLargeFileSizeThreshold 155 | 156 | // Get high token count threshold from CLI or use default 157 | highTokenThreshold := cmd.Int("high-token-threshold") 158 | if highTokenThreshold <= 0 { 159 | highTokenThreshold = DefaultHighTokenThreshold 160 | } 161 | 162 | cfg := &Config{ 163 | TargetDir: targetDir, 164 | OutputFile: outputFile, 165 | Force: force, 166 | ShowTree: showTree, 167 | DisableSort: disableSort, 168 | Format: format, 169 | AllowedFileExtensions: allowedFileExtensionsMap, 170 | IgnoredPathRegexes: ignoredPathRegexes, 171 | IgnoreSecrets: ignoreSecrets, 172 | RedactSecrets: redactSecrets, 173 | LargeFileSizeThreshold: largeFileSizeThreshold, 174 | HighTokenThreshold: highTokenThreshold, 175 | SkipTokenCount: skipTokenCount, 176 | } 177 | 178 | return cfg 179 | } 180 | 181 | // ShouldWriteFile returns true if the configuration is set to write output 182 | // to a file (i.e., if OutputFile is non-empty). 183 | func (cfg *Config) ShouldWriteFile() bool { 184 | return cfg.OutputFile != "" 185 | } 186 | 187 | // compileRegexes takes a slice of regex pattern strings and compiles them into 188 | // a slice of *regexp.Regexp. If any pattern is invalid, an error is returned. 189 | func compileRegexes(regexes []string) ([]*regexp.Regexp, error) { 190 | var compiled []*regexp.Regexp 191 | for _, pattern := range regexes { 192 | re, err := regexp.Compile(pattern) 193 | if err != nil { 194 | return nil, fmt.Errorf("invalid pattern %q: %w", pattern, err) 195 | } 196 | compiled = append(compiled, re) 197 | } 198 | return compiled, nil 199 | } 200 | -------------------------------------------------------------------------------- /internal/config/defaults.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // DefaultAllowedFileExtensions defines the default file extensions that are eligible for processing. 4 | // These extensions represent common programming, configuration, and documentation file types. 5 | var DefaultAllowedFileExtensions = []string{ 6 | // Programming languages 7 | "rs", "c", "h", "cpp", "hpp", "py", "java", "go", "rb", "php", "cs", 8 | "fs", "fsx", "fsi", "fsscript", "scala", "kt", "kts", "dart", "swift", 9 | "m", "mm", "r", "pl", "pm", "t", "lua", "elm", "erl", "ex", "exs", "zig", 10 | "psgi", "cgi", "groovy", 11 | 12 | // Web and frontend files 13 | "html", "css", "sass", "scss", "js", "ts", "jsx", "tsx", "vue", "svelte", 14 | "haml", "hbs", "jade", "less", "coffee", "astro", 15 | 16 | // Configuration and data files 17 | "toml", "json", "yaml", "yml", "ini", "conf", "cfg", "properties", "env", 18 | "xml", "sql", "htaccess", 19 | 20 | // Documentation and markup 21 | "md", "mdx", "markdown", "txt", "graphql", "proto", "prisma", "dhall", 22 | 23 | // Build and project files 24 | "gitignore", "lock", "gradle", "pom", "sbt", "gemspec", "podspec", "rake", 25 | 26 | // Infrastructure files 27 | "sh", "fish", "tf", "tfvars", 28 | } 29 | 30 | // DefaultLargeFileSizeThreshold defines the default size in bytes (1MB) above which 31 | // a file is considered "large" and a warning will be logged. 32 | var DefaultLargeFileSizeThreshold int64 = 1024 * 1024 33 | 34 | // DefaultHighTokenThreshold defines the default token count (5000) above which 35 | // a file is considered to have a high token count and a warning will be logged. 36 | var DefaultHighTokenThreshold = 5000 37 | 38 | // DefaultIgnoredPathPatterns defines the default path patterns that are excluded from processing. 39 | // These include directories, build artifacts, caches, and temporary files. 40 | var DefaultIgnoredPathPatterns = []string{ 41 | // Common directories to ignore anywhere in the path 42 | // Using (^|/) to match either the start of a string or after a slash 43 | `(^|/)\.git/`, `(^|/)\.next/`, `(^|/)node_modules/`, 44 | `(^|/)dist/`, `(^|/)build/`, `(^|/)out/`, `(^|/)target/`, 45 | `(^|/)\.cache/`, `(^|/)coverage/`, `(^|/)test-results/`, 46 | `(^|/)\.idea/`, `(^|/)\.vscode/`, `(^|/)\.vs/`, 47 | `(^|/)\.gradle/`, `(^|/)\.mvn/`, `(^|/)\.pytest_cache/`, 48 | `(^|/)__pycache__/`, `(^|/)\.sass-cache/`, `(^|/)\.vercel/`, 49 | `(^|/)\.turbo/`, 50 | 51 | // Directories that should be more specifically matched to avoid false positives 52 | `(^|/)vendor/`, `(^|/)bin/`, `(^|/)obj/`, `(^|/)\.settings/`, 53 | 54 | // Lock files and dependency metadata (full path matches to avoid false positives) 55 | `(^|/)pnpm-lock\.yaml$`, `(^|/)package-lock\.json$`, `(^|/)yarn\.lock$`, 56 | `(^|/)Cargo\.lock$`, `(^|/)Gemfile\.lock$`, `(^|/)composer\.lock$`, 57 | `(^|/)mix\.lock$`, `(^|/)poetry\.lock$`, `(^|/)Pipfile\.lock$`, 58 | `(^|/)packages\.lock\.json$`, `(^|/)paket\.lock$`, 59 | 60 | // Temporary and binary files (match full extensions) 61 | `\.pyc$`, `\.pyo$`, `\.pyd$`, `\.class$`, `\.o$`, `\.obj$`, 62 | `\.dll$`, `\.exe$`, `\.so$`, `\.dylib$`, `\.log$`, `\.tmp$`, 63 | `\.temp$`, `\.swp$`, `\.swo$`, `\.bak$`, `~$`, 64 | 65 | // System files 66 | `\.DS_Store$`, `Thumbs\.db$`, `\.env(\..+)?$`, 67 | 68 | // Specific files 69 | `(^|/)LICENSE$`, `(^|/)\.gitignore$`, 70 | } 71 | -------------------------------------------------------------------------------- /internal/core/git.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "sort" 11 | "strings" 12 | ) 13 | 14 | // GitExecutor defines the interface for running git-related commands. 15 | type GitExecutor interface { 16 | // ListFileChanges returns a ReadCloser that streams file paths that were changed in commits. 17 | // The caller is responsible for closing the returned stream. 18 | ListFileChanges(repoDir string) (io.ReadCloser, error) 19 | 20 | // IsAvailable indicates whether git is installed and can be found in PATH. 21 | IsAvailable() bool 22 | } 23 | 24 | // DefaultGitExecutor is the default, concrete implementation of GitExecutor. 25 | type DefaultGitExecutor struct{} 26 | 27 | // NewDefaultGitExecutor returns a new DefaultGitExecutor instance. 28 | func NewDefaultGitExecutor() *DefaultGitExecutor { 29 | return &DefaultGitExecutor{} 30 | } 31 | 32 | // cmdReadCloser wraps the command’s stdout so that closing it will also Wait() on the command. 33 | // This avoids leaving any zombie processes. 34 | type cmdReadCloser struct { 35 | io.ReadCloser 36 | cmd *exec.Cmd 37 | } 38 | 39 | // Close closes the underlying pipe and then waits for the command to finish. 40 | func (crc *cmdReadCloser) Close() error { 41 | closeErr := crc.ReadCloser.Close() 42 | waitErr := crc.cmd.Wait() 43 | 44 | // If both errors are nil, return nil 45 | if closeErr == nil && waitErr == nil { 46 | return nil 47 | } 48 | 49 | // If only one error is non-nil, return that error 50 | if closeErr == nil { 51 | return waitErr 52 | } 53 | if waitErr == nil { 54 | return closeErr 55 | } 56 | 57 | // If both errors are non-nil, combine them 58 | return fmt.Errorf("multiple errors on close: %w (close); %v (wait)", closeErr, waitErr) 59 | } 60 | 61 | // ListFileChanges runs the `git log --name-only ...` command and returns a stream of file paths. 62 | // Callers must close the returned ReadCloser to free resources and reap the spawned process. 63 | func (e *DefaultGitExecutor) ListFileChanges(repoDir string) (io.ReadCloser, error) { 64 | cmd := exec.Command( 65 | "git", 66 | "-C", repoDir, 67 | "log", 68 | "--name-only", 69 | "-n", "99999", 70 | "--pretty=format:", 71 | "--no-merges", 72 | "--relative", 73 | ) 74 | return e.executeWithReader(cmd, os.Stderr) 75 | } 76 | 77 | // IsAvailable returns true if the `git` executable is found in the system's PATH. 78 | func (e *DefaultGitExecutor) IsAvailable() bool { 79 | _, err := exec.LookPath("git") 80 | return err == nil 81 | } 82 | 83 | // executeWithReader starts the given command and returns a ReadCloser for stdout. 84 | // Once the caller closes it, the child process is reaped via cmd.Wait(). 85 | func (e *DefaultGitExecutor) executeWithReader(cmd *exec.Cmd, stderr io.Writer) (io.ReadCloser, error) { 86 | stdout, err := cmd.StdoutPipe() 87 | if err != nil { 88 | return nil, fmt.Errorf("failed to create stdout pipe: %w", err) 89 | } 90 | 91 | cmd.Stderr = stderr 92 | 93 | if err := cmd.Start(); err != nil { 94 | return nil, fmt.Errorf("failed to start command: %w", err) 95 | } 96 | 97 | // Wrap stdout in cmdReadCloser so that calling Close() will also wait on the command. 98 | return &cmdReadCloser{ 99 | ReadCloser: stdout, 100 | cmd: cmd, 101 | }, nil 102 | } 103 | 104 | // Git is a higher-level interface to a GitExecutor, providing convenience methods 105 | // for repository discovery and commit analysis. 106 | type Git struct { 107 | executor GitExecutor 108 | } 109 | 110 | // NewGit returns a new Git instance that delegates to the provided GitExecutor. 111 | func NewGit(executor GitExecutor) *Git { 112 | return &Git{executor: executor} 113 | } 114 | 115 | // IsAvailable indicates whether the underlying GitExecutor is capable of running git. 116 | func (g *Git) IsAvailable() bool { 117 | return g.executor.IsAvailable() 118 | } 119 | 120 | // FindRepositoryRoot walks up the directory tree from startDir until it finds 121 | // a `.git` directory. It returns the path to that directory, or an error if none is found. 122 | func (g *Git) FindRepositoryRoot(startDir string) (string, error) { 123 | current := startDir 124 | for { 125 | gitPath := filepath.Join(current, ".git") 126 | if info, err := os.Stat(gitPath); err == nil && info.IsDir() { 127 | // Found the Git root 128 | return current, nil 129 | } 130 | 131 | parent := filepath.Dir(current) 132 | if parent == current { 133 | // Reached root of filesystem without finding .git 134 | return "", fmt.Errorf("no repository found starting from %s", startDir) 135 | } 136 | current = parent 137 | } 138 | } 139 | 140 | // GetCommitCounts returns a map of file paths to the number of commits in which each file appears. 141 | func (g *Git) GetCommitCounts(repoDir string) (map[string]int, error) { 142 | output, err := g.executor.ListFileChanges(repoDir) 143 | if err != nil { 144 | return nil, fmt.Errorf("failed to list file changes: %w", err) 145 | } 146 | defer output.Close() 147 | 148 | commitCounts := make(map[string]int) 149 | 150 | scanner := bufio.NewScanner(output) 151 | for scanner.Scan() { 152 | line := strings.TrimSpace(scanner.Text()) 153 | if line != "" { 154 | commitCounts[line]++ 155 | } 156 | } 157 | 158 | if scanErr := scanner.Err(); scanErr != nil { 159 | return nil, fmt.Errorf("error reading git log output: %w", scanErr) 160 | } 161 | 162 | return commitCounts, nil 163 | } 164 | 165 | // CommitCounter defines a function type for counting the number of commits per file in a repository. 166 | type CommitCounter func(repoDir string) (map[string]int, error) 167 | 168 | // SortFilesByCommitCounts sorts the provided files based on their commit counts in ascending order. 169 | // It uses the provided commitCounter function to retrieve commit counts for each file. 170 | func (g *Git) SortFilesByCommitCounts(repoDir string, filePaths []string, commitCounter CommitCounter) ([]string, error) { 171 | commitCounts, err := commitCounter(repoDir) 172 | if err != nil { 173 | return nil, fmt.Errorf("failed to get commit counts: %w", err) 174 | } 175 | 176 | sort.Slice(filePaths, func(i, j int) bool { 177 | countI := 0 178 | if c, ok := commitCounts[filePaths[i]]; ok { 179 | countI = c 180 | } 181 | 182 | countJ := 0 183 | if c, ok := commitCounts[filePaths[j]]; ok { 184 | countJ = c 185 | } 186 | 187 | if countI != countJ { 188 | // Sort by ascending commit count 189 | return countI < countJ 190 | } 191 | 192 | // If counts are equal, sort alphabetically 193 | return filePaths[i] < filePaths[j] 194 | }) 195 | 196 | return filePaths, nil 197 | } 198 | -------------------------------------------------------------------------------- /internal/core/git_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestFindRepositoryRoot(t *testing.T) { 12 | git := NewGit(NewDefaultGitExecutor()) 13 | 14 | // Test case 1: .git directory in the startDir 15 | tempDir1, err := os.MkdirTemp("", "git-test-") 16 | if err != nil { 17 | t.Fatalf("Failed to create temp dir: %v", err) 18 | } 19 | defer os.RemoveAll(tempDir1) 20 | 21 | gitDir1 := filepath.Join(tempDir1, ".git") 22 | if err := os.Mkdir(gitDir1, 0755); err != nil { 23 | t.Fatalf("Failed to create .git dir: %v", err) 24 | } 25 | 26 | root1, err1 := git.FindRepositoryRoot(tempDir1) 27 | if err1 != nil { 28 | t.Fatalf("Test Case 1 Failed: Expected no error, got: %v", err1) 29 | } 30 | if root1 != tempDir1 { 31 | t.Fatalf("Test Case 1 Failed: Expected root to be %s, got: %s", tempDir1, root1) 32 | } 33 | 34 | // Test case 2: .git directory in a parent directory 35 | tempDir2, err := os.MkdirTemp("", "git-test-") 36 | if err != nil { 37 | t.Fatalf("Failed to create temp dir: %v", err) 38 | } 39 | defer os.RemoveAll(tempDir2) 40 | 41 | gitDir2 := filepath.Join(tempDir2, ".git") 42 | if err := os.Mkdir(gitDir2, 0755); err != nil { 43 | t.Fatalf("Failed to create .git dir: %v", err) 44 | } 45 | subdir2 := filepath.Join(tempDir2, "subdir") 46 | if err := os.Mkdir(subdir2, 0755); err != nil { 47 | t.Fatalf("Failed to create subdir: %v", err) 48 | } 49 | 50 | root2, err2 := git.FindRepositoryRoot(subdir2) 51 | if err2 != nil { 52 | t.Errorf("Test Case 2 Failed: Expected no error, got: %v", err2) 53 | } 54 | if root2 != tempDir2 { 55 | t.Errorf("Test Case 2 Failed: Expected root to be %s, got: %s", tempDir2, root2) 56 | } 57 | 58 | // Test case 3: No .git directory found 59 | tempDir3, err := os.MkdirTemp("", "git-test-") 60 | if err != nil { 61 | t.Fatalf("Failed to create temp dir: %v", err) 62 | } 63 | defer os.RemoveAll(tempDir3) 64 | 65 | _, err3 := git.FindRepositoryRoot(tempDir3) 66 | if err3 == nil { 67 | t.Errorf("Test Case 3 Failed: Expected error, got nil") 68 | } 69 | if err3 != nil && !strings.Contains(err3.Error(), "no repository found") { 70 | t.Errorf("Test Case 3 Failed: Expected 'no repository found' error, got: %v", err3) 71 | } 72 | 73 | // Test case 4: Start from root-like directory, no .git (more robust no .git test) 74 | // Create a deeper temp dir structure to simulate starting further from root 75 | tempDir4Base, err := os.MkdirTemp("", "git-test-base-") 76 | if err != nil { 77 | t.Fatalf("Failed to create base temp dir: %v", err) 78 | } 79 | defer os.RemoveAll(tempDir4Base) 80 | tempDir4 := filepath.Join(tempDir4Base, "level1", "level2") 81 | if err := os.MkdirAll(tempDir4, 0755); err != nil { 82 | t.Fatalf("Failed to create deep temp dir: %v", err) 83 | } 84 | 85 | _, err4 := git.FindRepositoryRoot(tempDir4) 86 | if err4 == nil { 87 | t.Errorf("Test Case 4 Failed: Expected error, got nil") 88 | } 89 | if err4 != nil && !strings.Contains(err4.Error(), "no repository found") { 90 | t.Errorf("Test Case 4 Failed: Expected 'no repository found' error, got: %v", err4) 91 | } 92 | } 93 | 94 | // MockGitExecutor is a mock implementation of GitExecutor for testing purposes. 95 | type MockGitExecutor struct { 96 | MockListFileChanges func(repoDir string) (io.ReadCloser, error) 97 | MockIsAvailable func() bool 98 | } 99 | 100 | func (m *MockGitExecutor) ListFileChanges(repoDir string) (io.ReadCloser, error) { 101 | if m.MockListFileChanges != nil { 102 | return m.MockListFileChanges(repoDir) 103 | } 104 | return io.NopCloser(strings.NewReader("")), nil // Default to no changes 105 | } 106 | 107 | func (m *MockGitExecutor) IsAvailable() bool { 108 | if m.MockIsAvailable != nil { 109 | return m.MockIsAvailable() 110 | } 111 | return true // Default to git available 112 | } 113 | 114 | func TestGetCommitCounts(t *testing.T) { 115 | tests := []struct { 116 | name string 117 | listChangesOutput string 118 | listChangesError error 119 | expectedCounts map[string]int 120 | expectError bool 121 | }{ 122 | { 123 | name: "No changes", 124 | listChangesOutput: "", 125 | expectedCounts: map[string]int{}, 126 | }, 127 | { 128 | name: "Single file, single commit", 129 | listChangesOutput: "file1.go\n", 130 | expectedCounts: map[string]int{"file1.go": 1}, 131 | }, 132 | { 133 | name: "Single file, multiple commits", 134 | listChangesOutput: "file1.go\nfile1.go\nfile1.go\n", 135 | expectedCounts: map[string]int{"file1.go": 3}, 136 | }, 137 | { 138 | name: "Multiple files, single commit each", 139 | listChangesOutput: "file1.go\nfile2.go\nfile3.go\n", 140 | expectedCounts: map[string]int{"file1.go": 1, "file2.go": 1, "file3.go": 1}, 141 | }, 142 | { 143 | name: "Multiple files, multiple commits", 144 | listChangesOutput: `file1.go 145 | file2.go 146 | file1.go 147 | file3.go 148 | file2.go 149 | `, 150 | expectedCounts: map[string]int{"file1.go": 2, "file2.go": 2, "file3.go": 1}, 151 | }, 152 | { 153 | name: "Empty lines in output", 154 | listChangesOutput: "\nfile1.go\n\nfile2.go\n\n", 155 | expectedCounts: map[string]int{"file1.go": 1, "file2.go": 1}, 156 | }, 157 | { 158 | name: "Error from ListFileChanges", 159 | listChangesError: ErrTest, // Define a test error 160 | expectError: true, 161 | }, 162 | } 163 | 164 | for _, tt := range tests { 165 | t.Run(tt.name, func(t *testing.T) { 166 | mockExecutor := &MockGitExecutor{ 167 | MockListFileChanges: func(repoDir string) (io.ReadCloser, error) { 168 | if tt.listChangesError != nil { 169 | return nil, tt.listChangesError 170 | } 171 | return io.NopCloser(strings.NewReader(tt.listChangesOutput)), nil 172 | }, 173 | } 174 | git := NewGit(mockExecutor) 175 | 176 | counts, err := git.GetCommitCounts("dummyRepoDir") 177 | 178 | if tt.expectError { 179 | if err == nil { 180 | t.Errorf("Expected error, but got nil") 181 | } 182 | return // Stop here for error cases 183 | } 184 | 185 | if err != nil { 186 | t.Fatalf("Unexpected error: %v", err) 187 | } 188 | 189 | if len(counts) != len(tt.expectedCounts) { 190 | t.Errorf("Counts map length mismatch: got %v, want %v", len(counts), len(tt.expectedCounts)) 191 | } 192 | 193 | for file, expectedCount := range tt.expectedCounts { 194 | actualCount, ok := counts[file] 195 | if !ok { 196 | t.Errorf("Missing file in counts: %s", file) 197 | continue 198 | } 199 | if actualCount != expectedCount { 200 | t.Errorf("Count mismatch for file %s: got %d, want %d", file, actualCount, expectedCount) 201 | } 202 | } 203 | }) 204 | } 205 | } 206 | 207 | func TestSortFilesByCommitCounts(t *testing.T) { 208 | tests := []struct { 209 | name string 210 | filePaths []string 211 | commitCounts map[string]int 212 | expectedOrder []string 213 | expectError bool // While sorting itself unlikely to error, including for completeness 214 | }{ 215 | { 216 | name: "Empty file list", 217 | filePaths: []string{}, 218 | commitCounts: map[string]int{}, 219 | expectedOrder: []string{}, 220 | }, 221 | { 222 | name: "Single file", 223 | filePaths: []string{"file1.go"}, 224 | commitCounts: map[string]int{"file1.go": 1}, 225 | expectedOrder: []string{"file1.go"}, 226 | }, 227 | { 228 | name: "Already sorted by commits", 229 | filePaths: []string{"file1.go", "file2.go", "file3.go"}, 230 | commitCounts: map[string]int{"file1.go": 1, "file2.go": 2, "file3.go": 3}, 231 | expectedOrder: []string{"file1.go", "file2.go", "file3.go"}, 232 | }, 233 | { 234 | name: "Reverse sorted by commits", 235 | filePaths: []string{"file3.go", "file2.go", "file1.go"}, 236 | commitCounts: map[string]int{"file1.go": 1, "file2.go": 2, "file3.go": 3}, 237 | expectedOrder: []string{"file1.go", "file2.go", "file3.go"}, 238 | }, 239 | { 240 | name: "Mixed commit counts", 241 | filePaths: []string{"file3.go", "file1.go", "file2.go"}, 242 | commitCounts: map[string]int{"file1.go": 2, "file2.go": 1, "file3.go": 3}, 243 | expectedOrder: []string{"file2.go", "file1.go", "file3.go"}, 244 | }, 245 | { 246 | name: "Same commit counts, sorted alphabetically", 247 | filePaths: []string{"file3.go", "file1.go", "file2.go"}, 248 | commitCounts: map[string]int{"file1.go": 1, "file2.go": 1, "file3.go": 1}, 249 | expectedOrder: []string{"file1.go", "file2.go", "file3.go"}, 250 | }, 251 | { 252 | name: "Mixed commit counts and same counts, alphabetical tie-breaker", 253 | filePaths: []string{"fileC.go", "fileA.go", "fileB.go", "fileD.go"}, 254 | commitCounts: map[string]int{"fileA.go": 2, "fileB.go": 1, "fileC.go": 2, "fileD.go": 1}, 255 | expectedOrder: []string{"fileB.go", "fileD.go", "fileA.go", "fileC.go"}, // B and D (count 1, then alpha), A and C (count 2, then alpha) 256 | }, 257 | { 258 | name: "File with no commit count (treated as 0 commits)", 259 | filePaths: []string{"file2.go", "file1.go", "file3.go"}, 260 | commitCounts: map[string]int{"file1.go": 1, "file2.go": 2}, // file3.go has no count 261 | expectedOrder: []string{"file3.go", "file1.go", "file2.go"}, // file3.go (0), file1.go (1), file2.go (2) 262 | }, 263 | } 264 | 265 | for _, tt := range tests { 266 | t.Run(tt.name, func(t *testing.T) { 267 | mockCommitCounter := func(repoDir string) (map[string]int, error) { 268 | return tt.commitCounts, nil 269 | } 270 | 271 | git := NewGit(&MockGitExecutor{}) // Executor not used for this test 272 | sortedFiles, err := git.SortFilesByCommitCounts("dummyRepoDir", tt.filePaths, mockCommitCounter) 273 | 274 | if tt.expectError && err == nil { 275 | t.Fatalf("Expected error but got nil") 276 | } else if !tt.expectError && err != nil { 277 | t.Fatalf("Unexpected error: %v", err) 278 | } 279 | 280 | if !tt.expectError { 281 | if !slicesAreEqual(sortedFiles, tt.expectedOrder) { 282 | t.Errorf("Sorted file order mismatch: got %v, want %v", sortedFiles, tt.expectedOrder) 283 | } 284 | } 285 | }) 286 | } 287 | } 288 | 289 | // Define a test error for error case testing 290 | var ErrTest = TestError("test error") 291 | 292 | type TestError string 293 | 294 | func (e TestError) Error() string { return string(e) } 295 | 296 | func slicesAreEqual(s1, s2 []string) bool { 297 | if len(s1) != len(s2) { 298 | return false 299 | } 300 | for i, v := range s1 { 301 | if v != s2[i] { 302 | return false 303 | } 304 | } 305 | return true 306 | } 307 | -------------------------------------------------------------------------------- /internal/core/runner.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/foresturquhart/grimoire/internal/secrets" 9 | "github.com/foresturquhart/grimoire/internal/tokens" 10 | "golang.org/x/text/language" 11 | "golang.org/x/text/message" 12 | 13 | "github.com/foresturquhart/grimoire/internal/config" 14 | "github.com/foresturquhart/grimoire/internal/serializer" 15 | "github.com/rs/zerolog/log" 16 | ) 17 | 18 | // Run is the main entry point for processing files. It uses a Walker to retrieve 19 | // files from cfg.TargetDir, optionally sorts them by Git commit frequency, 20 | // and serializes them (e.g., to Markdown) via the specified Serializer. 21 | // 22 | // The function returns an error if any critical step (such as starting the walker 23 | // or creating the output file) fails. 24 | func Run(cfg *config.Config) error { 25 | // Create a new walker to recursively find and filter files in TargetDir. 26 | walker := NewDefaultWalker(cfg.TargetDir, cfg.AllowedFileExtensions, cfg.IgnoredPathRegexes, cfg.OutputFile) 27 | 28 | // Recursively find and filter files in TargetDir, returning a slice of string paths. 29 | files, err := walker.Walk() 30 | if err != nil { 31 | return fmt.Errorf("error walking target directory: %w", err) 32 | } 33 | 34 | log.Info().Msgf("Found %d files in %s", len(files), cfg.TargetDir) 35 | 36 | // Initialize variables for secret findings 37 | var findings []secrets.Finding 38 | secretsFound := false 39 | 40 | // Get absolute file paths for secret detection 41 | var absoluteFilePaths []string 42 | for _, file := range files { 43 | absPath := filepath.Join(cfg.TargetDir, file) 44 | absoluteFilePaths = append(absoluteFilePaths, absPath) 45 | } 46 | 47 | log.Info().Msg("Checking for secrets in files...") 48 | 49 | // Create a secrets detector 50 | detector, err := secrets.NewDetector() 51 | if err != nil { 52 | return fmt.Errorf("failed to create secrets detector: %w", err) 53 | } 54 | 55 | // Detect secrets in the files 56 | findings, secretsFound, err = detector.DetectSecretsInFiles(absoluteFilePaths) 57 | if err != nil { 58 | return fmt.Errorf("failed to check for secrets: %w", err) 59 | } 60 | 61 | if secretsFound { 62 | // Choose logging level based on how we're handling the secrets 63 | logFn := log.Error 64 | if cfg.IgnoreSecrets || cfg.RedactSecrets { 65 | logFn = log.Warn 66 | } 67 | 68 | for _, finding := range findings { 69 | logFn(). 70 | Str("type", finding.Description). 71 | Str("secret", finding.Secret). 72 | Str("file", finding.File). 73 | Int("line", finding.Line). 74 | Msg("Detected possible secret") 75 | } 76 | 77 | if !cfg.IgnoreSecrets && !cfg.RedactSecrets { 78 | log.Fatal().Msg("Potential secrets detected in codebase. please review findings and remove sensitive data. use --ignore-secrets to bypass or --redact-secrets to redact (recommended)") 79 | } 80 | 81 | if cfg.IgnoreSecrets { 82 | log.Warn().Msg("Continuing despite detected secrets due to --ignore-secrets flag") 83 | } 84 | 85 | if cfg.RedactSecrets { 86 | log.Warn().Msg("Secrets will be redacted in the output due to --redact-secrets flag") 87 | } 88 | } else { 89 | log.Info().Msg("No secrets detected in files") 90 | } 91 | 92 | if !cfg.DisableSort { 93 | // If Git is available, attempt to sort files by commit frequency. 94 | gitExecutor := NewDefaultGitExecutor() 95 | git := NewGit(gitExecutor) 96 | 97 | if git.IsAvailable() { 98 | // If directory is within a Git repository, find the repository root 99 | repoDir, err := git.FindRepositoryRoot(cfg.TargetDir) 100 | if err != nil { 101 | log.Warn().Err(err).Msg("Git repository not found, skipping commit frequency file sorting") 102 | } else { 103 | log.Info().Msgf("Found Git repository at %s, sorting files by commit frequency", repoDir) 104 | 105 | files, err = git.SortFilesByCommitCounts(repoDir, files, git.GetCommitCounts) 106 | if err != nil { 107 | return fmt.Errorf("failed to sort files by commit frequency: %w", err) 108 | } 109 | } 110 | } else { 111 | log.Warn().Msg("Skipped sorting files by commit frequency: git executable not found") 112 | } 113 | } else { 114 | log.Info().Msg("Skipped sorting files by commit frequency: sorting disabled by flag") 115 | } 116 | 117 | // Determine where to write output. If cfg.ShouldWriteFile(), create the file, otherwise use stdout. 118 | var writer *os.File 119 | if cfg.ShouldWriteFile() { 120 | writer, err = os.Create(cfg.OutputFile) 121 | if err != nil { 122 | return fmt.Errorf("failed to create output file: %w", err) 123 | } 124 | defer func() { 125 | if writer != nil { 126 | if cerr := writer.Close(); cerr != nil { 127 | log.Warn().Err(cerr).Msg("Failed to close output file") 128 | } 129 | } 130 | }() 131 | } else { 132 | writer = os.Stdout 133 | } 134 | 135 | // Create a token capturing writer that wraps the actual writer 136 | tokenOpts := &tokens.TokenCounterOptions{} 137 | captureWriter, err := tokens.NewCaptureWriter(writer, tokenOpts) 138 | if err != nil { 139 | return fmt.Errorf("failed to create token counter: %w", err) 140 | } 141 | 142 | // Create a serializer based on the configured format 143 | formatSerializer, err := serializer.NewSerializer(cfg.Format) 144 | if err != nil { 145 | return fmt.Errorf("failed to create serializer: %w", err) 146 | } 147 | 148 | // Prepare redaction info if needed 149 | var redactionInfo *serializer.RedactionInfo 150 | if cfg.RedactSecrets && secretsFound { 151 | redactionInfo = &serializer.RedactionInfo{ 152 | Enabled: true, 153 | Findings: findings, 154 | BaseDir: cfg.TargetDir, 155 | } 156 | } 157 | 158 | // Serialize files to the configured format 159 | if err := formatSerializer.Serialize(captureWriter, cfg.TargetDir, files, cfg.ShowTree, redactionInfo, cfg.LargeFileSizeThreshold, cfg.HighTokenThreshold, cfg.SkipTokenCount); err != nil { 160 | return fmt.Errorf("failed to serialize content: %w", err) 161 | } 162 | 163 | if !cfg.SkipTokenCount { 164 | // Count tokens in the output 165 | if err := captureWriter.CountTokens(); err != nil { 166 | log.Warn().Err(err).Msg("Failed to count tokens in output") 167 | } else { 168 | // Log token count information 169 | p := message.NewPrinter(language.English) 170 | log.Info().Msg(p.Sprintf("Output contains %d tokens", captureWriter.GetTokenCount())) 171 | } 172 | } 173 | 174 | // Log where we wrote results, if we wrote to a file. 175 | if cfg.ShouldWriteFile() { 176 | log.Info().Msgf("File written to %s", cfg.OutputFile) 177 | } 178 | 179 | return nil 180 | } 181 | -------------------------------------------------------------------------------- /internal/core/walker.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | 9 | "github.com/rs/zerolog/log" 10 | gitignore "github.com/sabhiram/go-gitignore" 11 | ) 12 | 13 | // MaxTraversalDepth defines the maximum depth for directory traversal to prevent stack overflow 14 | const MaxTraversalDepth = 256 15 | 16 | // Walker defines an interface for traversing directories and returning a list of file paths. 17 | type Walker interface { 18 | Walk() ([]string, error) 19 | } 20 | 21 | // DefaultWalker is a concrete implementation of Walker that traverses a directory tree 22 | // starting at targetDir, while filtering files based on allowed extensions, ignored path patterns, 23 | // and ignore files (.gitignore and .grimoireignore). It also excludes a specific output file. 24 | type DefaultWalker struct { 25 | // targetDir is the base directory from which we begin walking. 26 | targetDir string 27 | 28 | // outputFile is the absolute path of the file to exclude from the results. 29 | outputFile string 30 | 31 | // allowedFileExtensions is a set of allowed file extensions. 32 | allowedFileExtensions map[string]bool 33 | 34 | // ignoredPathRegexes is a slice of compiled regular expressions for paths that should be ignored. 35 | ignoredPathRegexes []*regexp.Regexp 36 | } 37 | 38 | // NewDefaultWalker constructs and returns a new DefaultWalker configured with the given parameters. 39 | func NewDefaultWalker(targetDir string, allowedFileExtensions map[string]bool, ignoredPathRegexes []*regexp.Regexp, outputFile string) *DefaultWalker { 40 | return &DefaultWalker{ 41 | targetDir: targetDir, 42 | allowedFileExtensions: allowedFileExtensions, 43 | ignoredPathRegexes: ignoredPathRegexes, 44 | outputFile: outputFile, 45 | } 46 | } 47 | 48 | // Walk initiates a recursive traversal starting at targetDir. 49 | // It returns a slice of file paths (relative to targetDir) that meet the specified filtering criteria. 50 | func (dw *DefaultWalker) Walk() ([]string, error) { 51 | var files []string 52 | // Start traversal with no inherited ignore rules. 53 | if err := dw.traverse(dw.targetDir, nil, &files, 0); err != nil { 54 | return nil, fmt.Errorf("directory traversal failed: %w", err) 55 | } 56 | return files, nil 57 | } 58 | 59 | // traverse walks the directory tree starting at the given directory. 60 | // It accumulates ignore rules from any local .gitignore and .grimoireignore files, 61 | // applies the allowed extension and ignored path regex filters, 62 | // and appends any qualifying file paths (relative to targetDir) to the files slice. 63 | func (dw *DefaultWalker) traverse(dir string, inheritedIgnores []*gitignore.GitIgnore, files *[]string, depth int) error { 64 | // Check if maximum depth has been reached 65 | if depth >= MaxTraversalDepth { 66 | return fmt.Errorf("maximum directory depth of %d exceeded at %s", MaxTraversalDepth, dir) 67 | } 68 | 69 | // Start with the ignore rules inherited from parent directories. 70 | currentIgnores := append([]*gitignore.GitIgnore{}, inheritedIgnores...) 71 | 72 | // Define the names of local ignore files to check. 73 | ignoreFilenames := []string{".gitignore", ".grimoireignore"} 74 | 75 | // For each ignore file, if it exists in the current directory, compile and add its rules. 76 | for _, ignoreFilename := range ignoreFilenames { 77 | ignorePath := filepath.Join(dir, ignoreFilename) 78 | if info, err := os.Stat(ignorePath); err == nil && !info.IsDir() { 79 | if gi, err := gitignore.CompileIgnoreFile(ignorePath); err == nil { 80 | currentIgnores = append(currentIgnores, gi) 81 | } else { 82 | log.Warn().Err(err).Msgf("Error parsing ignore file at %s", ignorePath) 83 | } 84 | } 85 | } 86 | 87 | // Read all entries (files and directories) in the current directory. 88 | entries, err := os.ReadDir(dir) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | // Process each entry in the directory. 94 | for _, entry := range entries { 95 | // Compute the full path of the entry. 96 | fullPath := filepath.Join(dir, entry.Name()) 97 | 98 | // Calculate the relative path from targetDir. This will be used for both filtering and output. 99 | relPath, err := filepath.Rel(dw.targetDir, fullPath) 100 | if err != nil { 101 | // If there is an error, fall back to using the full path. 102 | relPath = fullPath 103 | } 104 | // Normalize the relative path to use forward slashes. 105 | relPath = filepath.ToSlash(relPath) 106 | 107 | // Exclude the specific output file from being processed. 108 | if fullPath == dw.outputFile { 109 | continue 110 | } 111 | 112 | // Check if the relative path matches any of the default ignored regex patterns. 113 | skipByPattern := false 114 | for _, r := range dw.ignoredPathRegexes { 115 | if r.MatchString(relPath) { 116 | skipByPattern = true 117 | break 118 | } 119 | } 120 | if skipByPattern { 121 | continue 122 | } 123 | 124 | // Check the cumulative ignore rules (from .gitignore and .grimoireignore files). 125 | skipByGitignore := false 126 | for _, gi := range currentIgnores { 127 | if gi.MatchesPath(relPath) { 128 | skipByGitignore = true 129 | break 130 | } 131 | } 132 | if skipByGitignore { 133 | continue 134 | } 135 | 136 | // If the entry is a directory, recursively traverse it. 137 | if entry.IsDir() { 138 | if err := dw.traverse(fullPath, currentIgnores, files, depth+1); err != nil { 139 | return err 140 | } 141 | } else { 142 | // For files, check whether their extension is allowed. 143 | ext := filepath.Ext(entry.Name()) 144 | if dw.allowedFileExtensions[ext] { 145 | // Append the file's relative path to the list. 146 | *files = append(*files, relPath) 147 | } 148 | } 149 | } 150 | 151 | return nil 152 | } 153 | -------------------------------------------------------------------------------- /internal/secrets/gitleaks.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | _ "embed" 5 | "fmt" 6 | "path/filepath" 7 | 8 | "github.com/BurntSushi/toml" 9 | "github.com/rs/zerolog/log" 10 | "github.com/zricethezav/gitleaks/v8/config" 11 | "github.com/zricethezav/gitleaks/v8/detect" 12 | "github.com/zricethezav/gitleaks/v8/sources" 13 | ) 14 | 15 | //go:embed gitleaks.toml 16 | var DefaultConfig []byte 17 | 18 | // Finding represents a simplified gitleaks finding for easier consumption 19 | type Finding struct { 20 | Description string 21 | Secret string 22 | File string 23 | Line int 24 | } 25 | 26 | // Detector provides functionality to scan files for secrets 27 | type Detector struct { 28 | detector *detect.Detector 29 | } 30 | 31 | // NewDetector creates a new secrets detector using the provided configuration 32 | func NewDetector() (*Detector, error) { 33 | var vc config.ViperConfig 34 | if err := toml.Unmarshal(DefaultConfig, &vc); err != nil { 35 | return nil, fmt.Errorf("failed to unmarshal gitleaks config: %w", err) 36 | } 37 | 38 | cfg, err := vc.Translate() 39 | if err != nil { 40 | return nil, fmt.Errorf("failed to translate gitleaks config: %w", err) 41 | } 42 | 43 | return &Detector{ 44 | detector: detect.NewDetector(cfg), 45 | }, nil 46 | } 47 | 48 | // DetectSecretsInFiles scans the provided file paths for secrets 49 | // Returns a slice of findings and a boolean indicating if any secrets were found 50 | func (d *Detector) DetectSecretsInFiles(filePaths []string) ([]Finding, bool, error) { 51 | if len(filePaths) == 0 { 52 | return nil, false, nil 53 | } 54 | 55 | scanTargetChan := make(chan sources.ScanTarget, len(filePaths)) 56 | for _, path := range filePaths { 57 | // Make sure the path is absolute 58 | absPath, err := filepath.Abs(path) 59 | if err != nil { 60 | log.Warn().Err(err).Msgf("Failed to get absolute path for %s", path) 61 | absPath = path // Fall back to original path 62 | } 63 | scanTargetChan <- sources.ScanTarget{Path: absPath} 64 | } 65 | close(scanTargetChan) 66 | 67 | gitleaksFindings, err := d.detector.DetectFiles(scanTargetChan) 68 | if err != nil { 69 | return nil, false, fmt.Errorf("failed to detect secrets: %w", err) 70 | } 71 | 72 | // Convert findings to our simplified format 73 | findings := make([]Finding, 0, len(gitleaksFindings)) 74 | for _, f := range gitleaksFindings { 75 | findings = append(findings, Finding{ 76 | Description: f.Description, 77 | Secret: f.Secret, 78 | File: f.File, 79 | Line: f.StartLine, 80 | }) 81 | } 82 | 83 | return findings, len(findings) > 0, nil 84 | } 85 | -------------------------------------------------------------------------------- /internal/secrets/gitleaks.toml: -------------------------------------------------------------------------------- 1 | # This gitleaks configuration file was adapted from https://github.com/Bearer/bearer 2 | # Original source: https://github.com/Bearer/bearer/blob/main/pkg/detectors/gitleaks/gitlab_config.toml 3 | # Used under the Elastic License 2.0 (https://www.elastic.co/licensing/elastic-license) 4 | # 5 | # Modifications made: modification of some ids and descriptions 6 | # 7 | # The Elastic License 2.0 permits use, copying, distribution, and derivative works 8 | # subject to certain limitations. See the full license for details. 9 | 10 | title = "gitleaks config" 11 | 12 | [[rules]] 13 | id = "GitLab Personal Access Token" 14 | description = "GitLab Personal Access Token" 15 | regex = '''glpat-[0-9a-zA-Z_\-]{20}''' 16 | tags = ["gitlab", "revocation_type"] 17 | keywords = [ 18 | "glpat", 19 | ] 20 | 21 | [[rules]] 22 | id = "GitLab Runner Registration Token" 23 | description = "GitLab Runner Registration Token" 24 | regex = '''GR1348941[0-9a-zA-Z_\-]{20}''' 25 | tags = ["gitlab"] 26 | keywords = [ 27 | "gr1348941", 28 | ] 29 | 30 | [[rules]] 31 | id = "AWS Access Token" 32 | description = "AWS Access Token" 33 | regex = '''AKIA[0-9A-Z]{16}''' 34 | tags = ["aws", "revocation_type"] 35 | keywords = [ 36 | "akia", 37 | ] 38 | 39 | # Cryptographic keys 40 | [[rules]] 41 | id = "PKCS8 private key" 42 | description = "PKCS8 private key" 43 | regex = '''-----BEGIN PRIVATE KEY-----''' 44 | keywords = [ 45 | "begin private key", 46 | ] 47 | 48 | [[rules]] 49 | id = "RSA private key" 50 | description = "RSA private key" 51 | regex = '''-----BEGIN RSA PRIVATE KEY-----''' 52 | keywords = [ 53 | "begin rsa private key", 54 | ] 55 | 56 | [[rules]] 57 | id = "SSH private key" 58 | description = "SSH private key" 59 | regex = '''-----BEGIN OPENSSH PRIVATE KEY-----''' 60 | keywords = [ 61 | "begin openssh private key", 62 | ] 63 | 64 | [[rules]] 65 | id = "PGP private key" 66 | description = "PGP private key" 67 | regex = '''-----BEGIN PGP PRIVATE KEY BLOCK-----''' 68 | keywords = [ 69 | "begin pgp private key block", 70 | ] 71 | 72 | [[rules]] 73 | id = "Github Personal Access Token" 74 | description = "Github Personal Access Token" 75 | regex = '''ghp_[0-9a-zA-Z]{36}''' 76 | keywords = [ 77 | "ghp_", 78 | ] 79 | 80 | [[rules]] 81 | id = "Github OAuth Access Token" 82 | description = "Github OAuth Access Token" 83 | regex = '''gho_[0-9a-zA-Z]{36}''' 84 | keywords = [ 85 | "gho_", 86 | ] 87 | 88 | [[rules]] 89 | id = "SSH (DSA) private key" 90 | description = "SSH (DSA) private key" 91 | regex = '''-----BEGIN DSA PRIVATE KEY-----''' 92 | keywords = [ 93 | "begin dsa private key", 94 | ] 95 | 96 | [[rules]] 97 | id = "SSH (EC) private key" 98 | description = "SSH (EC) private key" 99 | regex = '''-----BEGIN EC PRIVATE KEY-----''' 100 | keywords = [ 101 | "begin ec private key", 102 | ] 103 | 104 | 105 | [[rules]] 106 | id = "Github App Token" 107 | description = "Github App Token" 108 | regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' 109 | keywords = [ 110 | "ghu_", 111 | "ghs_" 112 | ] 113 | 114 | [[rules]] 115 | id = "Github Refresh Token" 116 | description = "Github Refresh Token" 117 | regex = '''ghr_[0-9a-zA-Z]{76}''' 118 | keywords = [ 119 | "ghr_" 120 | ] 121 | 122 | [[rules]] 123 | id = "Shopify shared secret" 124 | description = "Shopify shared secret" 125 | regex = '''shpss_[a-fA-F0-9]{32}''' 126 | keywords = [ 127 | "shpss_" 128 | ] 129 | 130 | [[rules]] 131 | id = "Shopify access token" 132 | description = "Shopify access token" 133 | regex = '''shpat_[a-fA-F0-9]{32}''' 134 | keywords = [ 135 | "shpat_" 136 | ] 137 | 138 | [[rules]] 139 | id = "Shopify custom app access token" 140 | description = "Shopify custom app access token" 141 | regex = '''shpca_[a-fA-F0-9]{32}''' 142 | keywords = [ 143 | "shpca_" 144 | ] 145 | 146 | [[rules]] 147 | id = "Shopify private app access token" 148 | description = "Shopify private app access token" 149 | regex = '''shppa_[a-fA-F0-9]{32}''' 150 | keywords = [ 151 | "shppa_" 152 | ] 153 | 154 | [[rules]] 155 | id = "Slack token" 156 | description = "Slack token" 157 | regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' 158 | keywords = [ 159 | "xoxb","xoxa","xoxp","xoxr","xoxs", 160 | ] 161 | 162 | [[rules]] 163 | id = "Stripe" 164 | description = "Stripe" 165 | regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}''' 166 | keywords = [ 167 | "sk_test","pk_test","sk_live","pk_live", 168 | ] 169 | 170 | [[rules]] 171 | id = "PyPI upload token" 172 | description = "PyPI upload token" 173 | regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}''' 174 | tags = ["pypi", "revocation_type"] 175 | keywords = [ 176 | "pypi-ageichlwas5vcmc", 177 | ] 178 | 179 | [[rules]] 180 | id = "Google (GCP) Service-account" 181 | description = "Google (GCP) Service-account" 182 | regex = '''\"type\": \"service_account\"''' 183 | 184 | [[rules]] 185 | # demo of this regex not matching passwords in urls that contain env vars: 186 | # https://regex101.com/r/rT9Lv9/6 187 | id = "Password in URL" 188 | description = "Password in URL" 189 | regex = '''[a-zA-Z]{3,10}:\/\/[^$][^:@\/\n]{3,20}:[^$][^:@\n\/]{3,40}@.{1,100}''' 190 | 191 | 192 | [[rules]] 193 | id = "Heroku API Key" 194 | description = "Heroku API Key" 195 | regex = '''(?i)(?:heroku)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:=|\|\|:|<=|=>|:)(?:'|\"|\s|=|\x60){0,5}([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})(?:['|\"|\n|\r|\s|\x60]|$)''' 196 | secretGroup = 1 197 | keywords = [ 198 | "heroku", 199 | ] 200 | 201 | [[rules]] 202 | id = "Slack Webhook" 203 | description = "Slack Webhook" 204 | regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}''' 205 | keywords = [ 206 | "https://hooks.slack.com/services", 207 | ] 208 | 209 | [[rules]] 210 | id = "Twilio API Key" 211 | description = "Twilio API Key" 212 | regex = '''SK[0-9a-fA-F]{32}''' 213 | keywords = [ 214 | "sk", 215 | "twilio" 216 | ] 217 | 218 | [[rules]] 219 | id = "Age secret key" 220 | description = "Age secret key" 221 | regex = '''AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}''' 222 | keywords = [ 223 | "age-secret-key-1", 224 | ] 225 | 226 | [[rules]] 227 | id = "Facebook token" 228 | description = "Facebook token" 229 | regex = '''(?i)(facebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' 230 | secretGroup = 3 231 | keywords = [ 232 | "facebook", 233 | ] 234 | 235 | [[rules]] 236 | id = "Twitter token" 237 | description = "Twitter token" 238 | regex = '''(?i)(twitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{35,44})['\"]''' 239 | secretGroup = 3 240 | keywords = [ 241 | "twitter", 242 | ] 243 | 244 | [[rules]] 245 | id = "Adobe Client ID (Oauth Web)" 246 | description = "Adobe Client ID (Oauth Web)" 247 | regex = '''(?i)(adobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' 248 | secretGroup = 3 249 | keywords = [ 250 | "adobe", 251 | ] 252 | 253 | [[rules]] 254 | id = "Adobe Client Secret" 255 | description = "Adobe Client Secret" 256 | regex = '''(p8e-)(?i)[a-z0-9]{32}''' 257 | keywords = [ 258 | "adobe", 259 | "p8e-," 260 | ] 261 | 262 | [[rules]] 263 | id = "Alibaba AccessKey ID" 264 | description = "Alibaba AccessKey ID" 265 | regex = '''(LTAI)(?i)[a-z0-9]{20}''' 266 | keywords = [ 267 | "ltai", 268 | ] 269 | 270 | [[rules]] 271 | id = "Alibaba Secret Key" 272 | description = "Alibaba Secret Key" 273 | regex = '''(?i)(alibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]''' 274 | secretGroup = 3 275 | keywords = [ 276 | "alibaba", 277 | ] 278 | 279 | [[rules]] 280 | id = "Asana Client ID" 281 | description = "Asana Client ID" 282 | regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{16})['\"]''' 283 | secretGroup = 3 284 | keywords = [ 285 | "asana", 286 | ] 287 | 288 | [[rules]] 289 | id = "Asana Client Secret" 290 | description = "Asana Client Secret" 291 | regex = '''(?i)(asana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]''' 292 | secretGroup = 3 293 | keywords = [ 294 | "asana", 295 | ] 296 | 297 | [[rules]] 298 | id = "Atlassian API token" 299 | description = "Atlassian API token" 300 | regex = '''(?i)(atlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{24})['\"]''' 301 | secretGroup = 3 302 | keywords = [ 303 | "atlassian", 304 | ] 305 | 306 | [[rules]] 307 | id = "Bitbucket client ID" 308 | description = "Bitbucket client ID" 309 | regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{32})['\"]''' 310 | secretGroup = 3 311 | keywords = [ 312 | "bitbucket", 313 | ] 314 | 315 | [[rules]] 316 | id = "Bitbucket client secret" 317 | description = "Bitbucket client secret" 318 | regex = '''(?i)(bitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9_\-]{64})['\"]''' 319 | secretGroup = 3 320 | keywords = [ 321 | "bitbucket", 322 | ] 323 | 324 | [[rules]] 325 | id = "Beamer API token" 326 | description = "Beamer API token" 327 | regex = '''(?i)(beamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](b_[a-z0-9=_\-]{44})['\"]''' 328 | secretGroup = 3 329 | keywords = [ 330 | "beamer", 331 | ] 332 | 333 | [[rules]] 334 | id = "Clojars API token" 335 | description = "Clojars API token" 336 | regex = '''(CLOJARS_)(?i)[a-z0-9]{60}''' 337 | keywords = [ 338 | "clojars", 339 | ] 340 | 341 | [[rules]] 342 | id = "Contentful delivery API token" 343 | description = "Contentful delivery API token" 344 | regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]''' 345 | secretGroup = 3 346 | keywords = [ 347 | "contentful", 348 | ] 349 | 350 | [[rules]] 351 | id = "Contentful preview API token" 352 | description = "Contentful preview API token" 353 | regex = '''(?i)(contentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{43})['\"]''' 354 | secretGroup = 3 355 | keywords = [ 356 | "contentful", 357 | ] 358 | 359 | [[rules]] 360 | id = "Databricks API token" 361 | description = "Databricks API token" 362 | regex = '''dapi[a-h0-9]{32}''' 363 | keywords = [ 364 | "dapi", 365 | "databricks" 366 | ] 367 | 368 | [[rules]] 369 | id = "Discord API key" 370 | description = "Discord API key" 371 | regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]''' 372 | secretGroup = 3 373 | keywords = [ 374 | "discord", 375 | ] 376 | 377 | [[rules]] 378 | id = "Discord client ID" 379 | description = "Discord client ID" 380 | regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([0-9]{18})['\"]''' 381 | secretGroup = 3 382 | keywords = [ 383 | "discord", 384 | ] 385 | 386 | [[rules]] 387 | id = "Discord client secret" 388 | description = "Discord client secret" 389 | regex = '''(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_\-]{32})['\"]''' 390 | secretGroup = 3 391 | keywords = [ 392 | "discord", 393 | ] 394 | 395 | [[rules]] 396 | id = "Doppler API token" 397 | description = "Doppler API token" 398 | regex = '''['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]''' 399 | keywords = [ 400 | "doppler", 401 | ] 402 | 403 | [[rules]] 404 | id = "Dropbox API secret/key" 405 | description = "Dropbox API secret/key" 406 | regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]''' 407 | keywords = [ 408 | "dropbox", 409 | ] 410 | 411 | [[rules]] 412 | id = "Dropbox short lived API token" 413 | description = "Dropbox short lived API token" 414 | regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]''' 415 | keywords = [ 416 | "dropbox", 417 | ] 418 | 419 | [[rules]] 420 | id = "Dropbox long lived API token" 421 | description = "Dropbox long lived API token" 422 | regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]''' 423 | keywords = [ 424 | "dropbox", 425 | ] 426 | 427 | [[rules]] 428 | id = "Duffel API token" 429 | description = "Duffel API token" 430 | regex = '''['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]''' 431 | keywords = [ 432 | "duffel", 433 | ] 434 | 435 | [[rules]] 436 | id = "Dynatrace API token" 437 | description = "Dynatrace API token" 438 | regex = '''['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]''' 439 | keywords = [ 440 | "dt0c01", 441 | ] 442 | 443 | [[rules]] 444 | id = "EasyPost API token" 445 | description = "EasyPost API token" 446 | regex = '''['\"]EZAK(?i)[a-z0-9]{54}['\"]''' 447 | keywords = [ 448 | "ezak", 449 | ] 450 | 451 | 452 | [[rules]] 453 | id = "EasyPost test API token" 454 | description = "EasyPost test API token" 455 | regex = '''['\"]EZTK(?i)[a-z0-9]{54}['\"]''' 456 | keywords = [ 457 | "eztk", 458 | ] 459 | 460 | [[rules]] 461 | id = "Fastly API token" 462 | description = "Fastly API token" 463 | regex = '''(?i)(fastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9\-=_]{32})['\"]''' 464 | secretGroup = 3 465 | keywords = [ 466 | "fastly", 467 | ] 468 | 469 | [[rules]] 470 | id = "Finicity client secret" 471 | description = "Finicity client secret" 472 | regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{20})['\"]''' 473 | secretGroup = 3 474 | keywords = [ 475 | "finicity", 476 | ] 477 | 478 | [[rules]] 479 | id = "Finicity API token" 480 | description = "Finicity API token" 481 | regex = '''(?i)(finicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' 482 | secretGroup = 3 483 | keywords = [ 484 | "finicity", 485 | ] 486 | 487 | [[rules]] 488 | id = "Flutterwave public key" 489 | description = "Flutterwave public key" 490 | regex = '''FLWPUBK_TEST-(?i)[a-h0-9]{32}-X''' 491 | keywords = [ 492 | "FLWPUBK_TEST", 493 | ] 494 | 495 | [[rules]] 496 | id = "Flutterwave secret key" 497 | description = "Flutterwave secret key" 498 | regex = '''FLWSECK_TEST-(?i)[a-h0-9]{32}-X''' 499 | keywords = [ 500 | "FLWSECK_TEST", 501 | ] 502 | 503 | [[rules]] 504 | id = "Flutterwave encrypted key" 505 | description = "Flutterwave encrypted key" 506 | regex = '''FLWSECK_TEST[a-h0-9]{12}''' 507 | keywords = [ 508 | "FLWSECK_TEST", 509 | ] 510 | 511 | [[rules]] 512 | id = "Frame.io API token" 513 | description = "Frame.io API token" 514 | regex = '''fio-u-(?i)[a-z0-9-_=]{64}''' 515 | keywords = [ 516 | "fio-u-", 517 | ] 518 | 519 | [[rules]] 520 | id = "GoCardless API token" 521 | description = "GoCardless API token" 522 | regex = '''['\"]live_(?i)[a-z0-9-_=]{40}['\"]''' 523 | keywords = [ 524 | "gocardless", 525 | ] 526 | 527 | [[rules]] 528 | id = "Grafana API token" 529 | description = "Grafana API token" 530 | regex = '''['\"]eyJrIjoi(?i)[a-z0-9-_=]{72,92}['\"]''' 531 | keywords = [ 532 | "grafana", 533 | ] 534 | 535 | [[rules]] 536 | id = "Hashicorp Terraform user/org API token" 537 | description = "Hashicorp Terraform user/org API token" 538 | regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9-_=]{60,70}['\"]''' 539 | keywords = [ 540 | "atlasv1", 541 | "hashicorp", 542 | "terraform" 543 | ] 544 | 545 | [[rules]] 546 | id = "Hashicorp Vault batch token" 547 | description = "Hashicorp Vault batch token" 548 | regex = '''b\.AAAAAQ[0-9a-zA-Z_-]{156}''' 549 | keywords = [ 550 | "hashicorp", 551 | "AAAAAQ", 552 | "vault" 553 | ] 554 | 555 | [[rules]] 556 | id = "Hubspot API token" 557 | description = "Hubspot API token" 558 | regex = '''(?i)(hubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' 559 | secretGroup = 3 560 | keywords = [ 561 | "hubspot", 562 | ] 563 | 564 | [[rules]] 565 | id = "Intercom API token" 566 | description = "Intercom API token" 567 | regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9=_]{60})['\"]''' 568 | secretGroup = 3 569 | keywords = [ 570 | "intercom", 571 | ] 572 | 573 | [[rules]] 574 | id = "Intercom client secret/ID" 575 | description = "Intercom client secret/ID" 576 | regex = '''(?i)(intercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' 577 | secretGroup = 3 578 | keywords = [ 579 | "intercom", 580 | ] 581 | 582 | [[rules]] 583 | id = "Ionic API token" 584 | description = "Ionic API token" 585 | regex = '''ion_(?i)[a-z0-9]{42}''' 586 | keywords = [ 587 | "ion_", 588 | ] 589 | 590 | [[rules]] 591 | id = "Linear API token" 592 | description = "Linear API token" 593 | regex = '''lin_api_(?i)[a-z0-9]{40}''' 594 | keywords = [ 595 | "lin_api_", 596 | ] 597 | 598 | [[rules]] 599 | id = "Linear client secret/ID" 600 | description = "Linear client secret/ID" 601 | regex = '''(?i)(linear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32})['\"]''' 602 | secretGroup = 3 603 | keywords = [ 604 | "linear", 605 | ] 606 | 607 | [[rules]] 608 | id = "Lob API Key" 609 | description = "Lob API Key" 610 | regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((live|test)_[a-f0-9]{35})['\"]''' 611 | secretGroup = 3 612 | keywords = [ 613 | "lob", 614 | ] 615 | 616 | [[rules]] 617 | id = "Lob Publishable API Key" 618 | description = "Lob Publishable API Key" 619 | regex = '''(?i)(lob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]((test|live)_pub_[a-f0-9]{31})['\"]''' 620 | secretGroup = 3 621 | keywords = [ 622 | "lob", 623 | ] 624 | 625 | [[rules]] 626 | id = "Mailchimp API key" 627 | description = "Mailchimp API key" 628 | regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]''' 629 | secretGroup = 3 630 | keywords = [ 631 | "mailchimp", 632 | ] 633 | 634 | [[rules]] 635 | id = "Mailgun private API token" 636 | description = "Mailgun private API token" 637 | regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]''' 638 | secretGroup = 3 639 | keywords = [ 640 | "mailgun", 641 | ] 642 | 643 | [[rules]] 644 | id = "Mailgun public validation key" 645 | description = "Mailgun public validation key" 646 | regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](pubkey-[a-f0-9]{32})['\"]''' 647 | secretGroup = 3 648 | keywords = [ 649 | "mailgun", 650 | ] 651 | 652 | [[rules]] 653 | id = "Mailgun webhook signing key" 654 | description = "Mailgun webhook signing key" 655 | regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]''' 656 | secretGroup = 3 657 | keywords = [ 658 | "mailgun", 659 | ] 660 | 661 | [[rules]] 662 | id = "Mapbox API token" 663 | description = "Mapbox API token" 664 | regex = '''(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})''' 665 | keywords = [ 666 | "mapbox", 667 | ] 668 | 669 | [[rules]] 670 | id = "messagebird-api-token" 671 | description = "MessageBird API token" 672 | regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{25})['\"]''' 673 | secretGroup = 3 674 | keywords = [ 675 | "messagebird", 676 | ] 677 | 678 | [[rules]] 679 | id = "MessageBird API client ID" 680 | description = "MessageBird API client ID" 681 | regex = '''(?i)(messagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]''' 682 | secretGroup = 3 683 | keywords = [ 684 | "messagebird", 685 | ] 686 | 687 | [[rules]] 688 | id = "New Relic user API Key" 689 | description = "New Relic user API Key" 690 | regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]''' 691 | keywords = [ 692 | "nrak", 693 | ] 694 | 695 | [[rules]] 696 | id = "New Relic user API ID" 697 | description = "New Relic user API ID" 698 | regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]''' 699 | secretGroup = 3 700 | keywords = [ 701 | "newrelic", 702 | ] 703 | 704 | [[rules]] 705 | id = "New Relic ingest browser API token" 706 | description = "New Relic ingest browser API token" 707 | regex = '''['\"](NRJS-[a-f0-9]{19})['\"]''' 708 | keywords = [ 709 | "nrjs", 710 | ] 711 | 712 | [[rules]] 713 | id = "npm access token" 714 | description = "npm access token" 715 | regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]''' 716 | keywords = [ 717 | "npm_", 718 | ] 719 | 720 | [[rules]] 721 | id = "Planetscale password" 722 | description = "Planetscale password" 723 | regex = '''pscale_pw_(?i)[a-z0-9\-_\.]{43}''' 724 | keywords = [ 725 | "pscale_pw_", 726 | ] 727 | 728 | [[rules]] 729 | id = "Planetscale API token" 730 | description = "Planetscale API token" 731 | regex = '''pscale_tkn_(?i)[a-z0-9\-_\.]{43}''' 732 | keywords = [ 733 | "pscale_tkn_", 734 | ] 735 | 736 | [[rules]] 737 | id = "Postman API token" 738 | description = "Postman API token" 739 | regex = '''PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}''' 740 | keywords = [ 741 | "pmak-", 742 | ] 743 | 744 | [[rules]] 745 | id = "Pulumi API token" 746 | description = "Pulumi API token" 747 | regex = '''pul-[a-f0-9]{40}''' 748 | keywords = [ 749 | "pul-", 750 | ] 751 | 752 | [[rules]] 753 | id = "Rubygem API token" 754 | description = "Rubygem API token" 755 | regex = '''rubygems_[a-f0-9]{48}''' 756 | keywords = [ 757 | "rubygems_", 758 | ] 759 | 760 | [[rules]] 761 | id = "Sendgrid API token" 762 | description = "Sendgrid API token" 763 | regex = '''SG\.(?i)[a-z0-9_\-\.]{66}''' 764 | keywords = [ 765 | "sendgrid", 766 | ] 767 | 768 | [[rules]] 769 | id = "Sendinblue API token" 770 | description = "Sendinblue API token" 771 | regex = '''xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}''' 772 | keywords = [ 773 | "xkeysib-", 774 | ] 775 | 776 | [[rules]] 777 | id = "Shippo API token" 778 | description = "Shippo API token" 779 | regex = '''shippo_(live|test)_[a-f0-9]{40}''' 780 | keywords = [ 781 | "shippo_", 782 | ] 783 | 784 | [[rules]] 785 | id = "Linkedin Client secret" 786 | description = "Linkedin Client secret" 787 | regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z]{16})['\"]''' 788 | secretGroup = 3 789 | keywords = [ 790 | "linkedin", 791 | ] 792 | 793 | [[rules]] 794 | id = "Linkedin Client ID" 795 | description = "Linkedin Client ID" 796 | regex = '''(?i)(linkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{14})['\"]''' 797 | secretGroup = 3 798 | keywords = [ 799 | "linkedin", 800 | ] 801 | 802 | [[rules]] 803 | id = "Twitch API token" 804 | description = "Twitch API token" 805 | regex = '''(?i)(twitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{30})['\"]''' 806 | secretGroup = 3 807 | keywords = [ 808 | "twitch", 809 | ] 810 | 811 | [[rules]] 812 | id = "Typeform API token" 813 | description = "Typeform API token" 814 | regex = '''(?i)(typeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(tfp_[a-z0-9\-_\.=]{59})''' 815 | secretGroup = 3 816 | keywords = [ 817 | "typeform", 818 | ] 819 | 820 | [[rules]] 821 | id = "Yandex.Cloud IAM Cookie v1" 822 | description = "Yandex.Cloud IAM Cookie v1" 823 | regex = '''\bc1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2}['|\"|\n|\r|\s|\x60]''' 824 | keywords = [ 825 | "yandex", 826 | ] 827 | 828 | [[rules]] 829 | id = "Yandex.Cloud IAM Token v1" 830 | description = "Yandex.Cloud IAM Token v1" 831 | regex = '''\bt1\.[A-Z0-9a-z_-]+[=]{0,2}\.[A-Z0-9a-z_-]{86}[=]{0,2}['|\"|\n|\r|\s|\x60]''' 832 | keywords = [ 833 | "yandex", 834 | ] 835 | 836 | [[rules]] 837 | id = "Yandex.Cloud IAM API key v1" 838 | description = "Yandex.Cloud IAM API key v1" 839 | regex = '''\bAQVN[A-Za-z0-9_\-]{35,38}['|\"|\n|\r|\s|\x60]''' 840 | keywords = [ 841 | "yandex", 842 | ] 843 | 844 | [[rules]] 845 | id = "Yandex.Cloud AWS API compatible Access Secret" 846 | description = "Yandex.Cloud AWS API compatible Access Secret" 847 | regex = '''\bYC[a-zA-Z0-9_\-]{38}['|\"|\n|\r|\s|\x60]''' 848 | keywords = [ 849 | "yandex", 850 | ] 851 | 852 | [allowlist] 853 | description = "global allow lists" 854 | paths = [ 855 | '''gitleaks.toml''', 856 | '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''' 857 | ] -------------------------------------------------------------------------------- /internal/serializer/detector.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "path/filepath" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | // MinifiedFileThresholds contains configuration values for detecting minified files. 12 | // It now includes general, JS-specific, and CSS-specific thresholds. 13 | type MinifiedFileThresholds struct { 14 | // General thresholds for both file types. 15 | MaxLineLength int // Maximum line length to flag minification. 16 | MaxLinesPerCharRatio float64 // Maximum ratio of non-blank lines to total characters. 17 | MinNonBlankLines int // Minimum number of non-blank lines to perform detection. 18 | MinTotalChars int // Minimum file size to consider for detection. 19 | SingleLineMinLength int // For single-line files: length threshold for minification. 20 | 21 | // JS-specific thresholds. 22 | JS_SingleCharVarStrongThreshold int // If count > this, immediately flag as minified. 23 | JS_SingleCharVarModerateThreshold int // If count > this, add to pattern score. 24 | JS_ShortParamsThreshold int // Number of short parameter clusters to add score. 25 | JS_ChainedMethodsThreshold int // Number of chained method patterns to add score. 26 | JS_PatternScoreThreshold int // Overall pattern score threshold to flag minification. 27 | 28 | // CSS-specific thresholds. 29 | CSS_ClearIndicatorSemicolonCount int // Clear indicator: high semicolon count. 30 | CSS_ClearIndicatorSpaceDivisor int // Clear indicator: space count divisor. 31 | CSS_PatternIndicatorSemicolonCount int // Pattern indicator: semicolon count. 32 | CSS_PatternIndicatorSpaceDivisor int // Pattern indicator: space count divisor. 33 | CSS_NoSpacesAroundBracketsThreshold int // Threshold for missing spaces around brackets. 34 | CSS_NoSpacesAfterColonsThreshold int // Threshold for missing spaces after colons. 35 | CSS_PatternScoreThreshold int // Overall pattern score threshold. 36 | } 37 | 38 | // DefaultMinifiedFileThresholds defines default values for minified file detection. 39 | var DefaultMinifiedFileThresholds = MinifiedFileThresholds{ 40 | // General thresholds. 41 | MaxLineLength: 500, 42 | MaxLinesPerCharRatio: 0.02, 43 | MinNonBlankLines: 5, 44 | MinTotalChars: 200, 45 | SingleLineMinLength: 1000, 46 | 47 | // JS-specific thresholds. 48 | JS_SingleCharVarStrongThreshold: 15, 49 | JS_SingleCharVarModerateThreshold: 8, 50 | JS_ShortParamsThreshold: 3, 51 | JS_ChainedMethodsThreshold: 2, 52 | JS_PatternScoreThreshold: 3, 53 | 54 | // CSS-specific thresholds. 55 | CSS_ClearIndicatorSemicolonCount: 30, 56 | CSS_ClearIndicatorSpaceDivisor: 30, 57 | CSS_PatternIndicatorSemicolonCount: 20, 58 | CSS_PatternIndicatorSpaceDivisor: 20, 59 | CSS_NoSpacesAroundBracketsThreshold: 10, 60 | CSS_NoSpacesAfterColonsThreshold: 10, 61 | CSS_PatternScoreThreshold: 3, 62 | } 63 | 64 | // Precompiled regex patterns for JS heuristics. 65 | var ( 66 | reSingleCharVar = regexp.MustCompile(`\b[a-z]\b=`) 67 | reShortParams = regexp.MustCompile(`\([a-z],[a-z],[a-z](,[a-z])*\)`) 68 | reConsecutiveEnd = regexp.MustCompile(`\){4,}|\]{4,}`) 69 | reChainedMethods = regexp.MustCompile(`\.[a-zA-Z]+\([^)]*\)\.[a-zA-Z]+\([^)]*\)\.[a-zA-Z]+\(`) 70 | reLongOperators = regexp.MustCompile(`[+\-/*&|^]{5,}`) 71 | ) 72 | 73 | // Precompiled regex patterns for CSS heuristics. 74 | var ( 75 | reNoSpacesAroundBrackets = regexp.MustCompile(`[^\s{][{]|[}][^\s}]`) 76 | reNoSpacesAfterColons = regexp.MustCompile(`:[^}\s]`) 77 | ) 78 | 79 | // IsMinifiedFile returns true if the file content appears to be minified. 80 | // It applies general heuristics and delegates to JS- or CSS-specific checks. 81 | func IsMinifiedFile(content, filePath string, thresholds MinifiedFileThresholds) bool { 82 | ext := strings.ToLower(filepath.Ext(filePath)) 83 | if ext != ".js" && ext != ".css" { 84 | return false 85 | } 86 | 87 | // Skip very small files. 88 | if len(content) < thresholds.MinTotalChars { 89 | return false 90 | } 91 | 92 | // Gather general file metrics. 93 | lines := strings.Split(content, "\n") 94 | nonBlankLines, maxLineLength := analyzeLines(lines) 95 | 96 | // Special handling for single-line files. 97 | if nonBlankLines == 1 && len(content) > thresholds.SingleLineMinLength { 98 | if strings.Count(content, "/*") < 3 && strings.Count(content, "//") < 5 { 99 | log.Debug().Str("path", filePath).Msg("Detected single-line minified file") 100 | return true 101 | } 102 | } 103 | 104 | // Apply general heuristics if file is sufficiently large. 105 | if nonBlankLines >= thresholds.MinNonBlankLines { 106 | if maxLineLength > thresholds.MaxLineLength { 107 | return true 108 | } 109 | if float64(nonBlankLines)/float64(len(content)) < thresholds.MaxLinesPerCharRatio { 110 | return true 111 | } 112 | } 113 | 114 | // Delegate file-type–specific checks. 115 | switch ext { 116 | case ".js": 117 | return isMinifiedJS(content, nonBlankLines, maxLineLength, thresholds) 118 | case ".css": 119 | return isMinifiedCSS(content, thresholds) 120 | } 121 | 122 | return false 123 | } 124 | 125 | // analyzeLines computes the number of non-blank lines and the maximum line length. 126 | func analyzeLines(lines []string) (nonBlankLines, maxLineLength int) { 127 | for _, line := range lines { 128 | trimmed := strings.TrimSpace(line) 129 | if len(trimmed) > 0 { 130 | nonBlankLines++ 131 | } 132 | if len(line) > maxLineLength { 133 | maxLineLength = len(line) 134 | } 135 | } 136 | return 137 | } 138 | 139 | // isMinifiedJS applies JavaScript-specific heuristics. 140 | func isMinifiedJS(content string, nonBlankLines, maxLineLength int, thresholds MinifiedFileThresholds) bool { 141 | // Check for a sourceMappingURL which is a strong signal. 142 | if strings.Contains(content, "sourceMappingURL") { 143 | return true 144 | } 145 | 146 | // Only check patterns if content appears suspicious. 147 | shouldCheckPatterns := nonBlankLines < thresholds.MinNonBlankLines || maxLineLength > thresholds.MaxLineLength/2 148 | if !shouldCheckPatterns { 149 | return false 150 | } 151 | 152 | patternScore := 0 153 | 154 | // Heuristic: Many single-character variables. 155 | singleCharCount := len(reSingleCharVar.FindAllString(content, -1)) 156 | if singleCharCount > thresholds.JS_SingleCharVarStrongThreshold { 157 | return true 158 | } else if singleCharCount > thresholds.JS_SingleCharVarModerateThreshold { 159 | patternScore += 2 160 | } 161 | 162 | // Heuristic: Clusters of short parameter names. 163 | if len(reShortParams.FindAllString(content, -1)) > thresholds.JS_ShortParamsThreshold { 164 | patternScore += 2 165 | } 166 | 167 | // Heuristic: Consecutive closing brackets/parentheses. 168 | if reConsecutiveEnd.MatchString(content) { 169 | patternScore++ 170 | } 171 | 172 | // Heuristic: Chained methods without spacing. 173 | if len(reChainedMethods.FindAllString(content, -1)) > thresholds.JS_ChainedMethodsThreshold { 174 | patternScore += 2 175 | } 176 | 177 | // Heuristic: Long strings of operators. 178 | if reLongOperators.MatchString(content) { 179 | patternScore++ 180 | } 181 | 182 | return patternScore >= thresholds.JS_PatternScoreThreshold 183 | } 184 | 185 | // isMinifiedCSS applies CSS-specific heuristics. 186 | func isMinifiedCSS(content string, thresholds MinifiedFileThresholds) bool { 187 | semicolonCount := strings.Count(content, ";") 188 | spaceCount := strings.Count(content, " ") 189 | 190 | // Very clear indicator: high semicolon count and very few spaces. 191 | if semicolonCount > thresholds.CSS_ClearIndicatorSemicolonCount && 192 | spaceCount < len(content)/thresholds.CSS_ClearIndicatorSpaceDivisor { 193 | return true 194 | } 195 | 196 | cssPatternScore := 0 197 | 198 | // Moderate indicator: many semicolons with relatively few spaces. 199 | if semicolonCount > thresholds.CSS_PatternIndicatorSemicolonCount && 200 | spaceCount < len(content)/thresholds.CSS_PatternIndicatorSpaceDivisor { 201 | cssPatternScore += 2 202 | } 203 | 204 | // Check for lack of spacing around brackets. 205 | if len(reNoSpacesAroundBrackets.FindAllString(content, -1)) > thresholds.CSS_NoSpacesAroundBracketsThreshold { 206 | cssPatternScore += 2 207 | } 208 | 209 | // Check for missing spaces after colons. 210 | if len(reNoSpacesAfterColons.FindAllString(content, -1)) > thresholds.CSS_NoSpacesAfterColonsThreshold { 211 | cssPatternScore++ 212 | } 213 | 214 | // Check for !important usage without spacing. 215 | if strings.Count(content, "!important") < strings.Count(content, "!important;") { 216 | cssPatternScore++ 217 | } 218 | 219 | return cssPatternScore >= thresholds.CSS_PatternScoreThreshold 220 | } 221 | -------------------------------------------------------------------------------- /internal/serializer/markdown.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/foresturquhart/grimoire/internal/tokens" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | // MarkdownSerializer provides methods to write multiple files' contents into a 16 | // Markdown-formatted document. Each file is written under an H2 heading, 17 | // and its content is placed inside a fenced code block. 18 | type MarkdownSerializer struct{} 19 | 20 | // NewMarkdownSerializer returns a new instance of MarkdownSerializer. 21 | func NewMarkdownSerializer() *MarkdownSerializer { 22 | return &MarkdownSerializer{} 23 | } 24 | 25 | // Serialize takes a list of file paths relative to baseDir, reads and normalizes 26 | // each file's content, and writes them to writer in Markdown format. If reading 27 | // any file fails, it logs a warning and skips that file. 28 | // If showTree is true, it prepends a directory tree visualization. 29 | // If redactionInfo is not nil, it redacts secrets from the output. 30 | // largeFileSizeThreshold defines the size in bytes above which a file is considered "large" 31 | // and a warning will be logged. 32 | // highTokenThreshold defines the token count above which a file is considered 33 | // to have a high token count and a warning will be logged. 34 | // skipTokenCount indicates whether to skip token counting entirely for warnings. 35 | func (s *MarkdownSerializer) Serialize(writer io.Writer, baseDir string, filePaths []string, showTree bool, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) error { 36 | // Write the header with timestamp 37 | timestamp := time.Now().UTC().Format(time.RFC3339Nano) 38 | header := fmt.Sprintf("This document contains a structured representation of the entire codebase, merging all files into a single Markdown file.\n\nGenerated by Grimoire on: %s\n\n", timestamp) 39 | 40 | if _, err := writer.Write([]byte(header)); err != nil { 41 | return fmt.Errorf("failed to write header: %w", err) 42 | } 43 | 44 | // Write the summary section 45 | summary := "## Summary\n\n" 46 | summary += "This file contains a packed representation of the entire codebase's contents. " 47 | summary += "It is designed to be easily consumable by AI systems for analysis, code review, or other automated processes.\n\n" 48 | summary += "- This file should be treated as read-only. Any changes should be made to the original codebase files.\n" 49 | summary += "- When processing this file, use the file path headings to distinguish between different files.\n" 50 | summary += "- This file may contain sensitive information and should be handled with appropriate care.\n" 51 | 52 | if redactionInfo != nil && redactionInfo.Enabled { 53 | summary += "- Detected secrets have been redacted with the format [REDACTED SECRET: description].\n" 54 | } 55 | 56 | summary += "- Some files may have been excluded based on .gitignore rules and Grimoire's configuration.\n" 57 | 58 | if showTree { 59 | summary += "- The file begins with this summary, followed by the directory structure, and then includes all codebase files.\n\n" 60 | } else { 61 | summary += "- The file begins with this summary, followed by all codebase files.\n\n" 62 | } 63 | 64 | if _, err := writer.Write([]byte(summary)); err != nil { 65 | return fmt.Errorf("failed to write summary: %w", err) 66 | } 67 | 68 | // Add directory tree if requested 69 | if showTree && len(filePaths) > 0 { 70 | treeGen := NewDefaultTreeGenerator() 71 | rootNode := treeGen.GenerateTree(filePaths) 72 | 73 | treeContent := "## Directory Structure\n\n" 74 | treeContent += s.renderTreeAsMarkdownList(rootNode, 0) 75 | treeContent += "\n" 76 | 77 | if _, err := writer.Write([]byte(treeContent)); err != nil { 78 | return fmt.Errorf("failed to write directory tree: %w", err) 79 | } 80 | } 81 | 82 | // Write files heading 83 | filesHeading := "## Files\n\n" 84 | if _, err := writer.Write([]byte(filesHeading)); err != nil { 85 | return fmt.Errorf("failed to write files heading: %w", err) 86 | } 87 | 88 | // Process each file 89 | for i, relPath := range filePaths { 90 | // Write the heading (e.g. ## path/to/file.ext) 91 | heading := fmt.Sprintf("### File: %s\n\n", relPath) 92 | if _, err := writer.Write([]byte(heading)); err != nil { 93 | return fmt.Errorf("failed to write heading for %s: %w", relPath, err) 94 | } 95 | 96 | // Read and normalize file content 97 | content, isLargeFile, err := s.readAndNormalizeContent(baseDir, relPath, redactionInfo, largeFileSizeThreshold, highTokenThreshold, skipTokenCount) 98 | if err != nil { 99 | log.Warn().Err(err).Msgf("Skipping file %s due to read error", relPath) 100 | continue 101 | } 102 | 103 | if isLargeFile { 104 | log.Warn().Msgf("File %s exceeds the large file threshold (%d bytes). Including in output but this may impact performance.", relPath, largeFileSizeThreshold) 105 | } 106 | 107 | // Check if the file is minified (only applicable file type) 108 | if IsMinifiedFile(content, relPath, DefaultMinifiedFileThresholds) { 109 | log.Warn().Msgf("File %s appears to be minified. Consider excluding it to reduce token counts.", relPath) 110 | } 111 | 112 | // Wrap content in fenced code block 113 | formattedContent := fmt.Sprintf("```\n%s\n```", content) 114 | // Add an extra blank line between files, except for the last one 115 | if i < len(filePaths)-1 { 116 | formattedContent += "\n\n" 117 | } 118 | 119 | if _, err := writer.Write([]byte(formattedContent)); err != nil { 120 | return fmt.Errorf("failed to write content for %s: %w", relPath, err) 121 | } 122 | } 123 | 124 | return nil 125 | } 126 | 127 | // renderTreeAsMarkdownList recursively builds a nested Markdown list representation of the tree. 128 | // This is specific to the Markdown serializer's formatting needs. 129 | func (s *MarkdownSerializer) renderTreeAsMarkdownList(node *TreeNode, depth int) string { 130 | if node == nil { 131 | return "" 132 | } 133 | 134 | var builder strings.Builder 135 | 136 | // Skip the root node in output 137 | if node.Name != "" { 138 | // Create indentation based on depth 139 | indent := strings.Repeat(" ", depth) 140 | 141 | // Add list item marker and name 142 | builder.WriteString(indent) 143 | builder.WriteString("- ") 144 | builder.WriteString(node.Name) 145 | 146 | // Add a directory indicator for directories 147 | if node.IsDir { 148 | builder.WriteString("/") 149 | } 150 | 151 | builder.WriteString("\n") 152 | } 153 | 154 | // Process children with incremented depth 155 | for _, child := range node.Children { 156 | childDepth := depth 157 | if node.Name != "" { 158 | // Only increment depth for non-root nodes 159 | childDepth++ 160 | } 161 | builder.WriteString(s.renderTreeAsMarkdownList(child, childDepth)) 162 | } 163 | 164 | return builder.String() 165 | } 166 | 167 | // readAndNormalizeContent reads a file from baseDir/relPath and normalizes its 168 | // content by trimming surrounding whitespace and trailing spaces on each line. 169 | // If redactionInfo is not nil, it redacts secrets from the output. 170 | // It also checks if the file exceeds the large file size threshold and returns a flag if it does. 171 | func (s *MarkdownSerializer) readAndNormalizeContent(baseDir, relPath string, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) (string, bool, error) { 172 | fullPath := filepath.Join(baseDir, relPath) 173 | 174 | // Check file size before reading 175 | fileInfo, err := os.Stat(fullPath) 176 | if err != nil { 177 | return "", false, fmt.Errorf("failed to stat file %s: %w", fullPath, err) 178 | } 179 | 180 | // Check if file exceeds large file threshold 181 | isLargeFile := fileInfo.Size() > largeFileSizeThreshold 182 | 183 | contentBytes, err := os.ReadFile(fullPath) 184 | if err != nil { 185 | return "", false, fmt.Errorf("failed to read file %s: %w", fullPath, err) 186 | } 187 | 188 | // Convert bytes to string and normalize 189 | content := string(contentBytes) 190 | normalizedContent := s.normalizeContent(content) 191 | 192 | // If redaction is enabled, redact any secrets 193 | if redactionInfo != nil && redactionInfo.Enabled { 194 | fileFindings := GetFindingsForFile(redactionInfo, relPath, baseDir) 195 | if len(fileFindings) > 0 { 196 | normalizedContent = RedactSecrets(normalizedContent, fileFindings) 197 | } 198 | } 199 | 200 | // Count tokens for this file and warn if it exceeds the threshold 201 | if !skipTokenCount && highTokenThreshold > 0 { 202 | tokenCount, err := tokens.CountFileTokens(relPath, normalizedContent) 203 | if err != nil { 204 | log.Warn().Err(err).Msgf("Failed to count tokens for file %s", relPath) 205 | } else if tokenCount > highTokenThreshold { 206 | log.Warn().Msgf("File %s has a high token count (%d tokens, threshold: %d). This will consume significant LLM context.", relPath, tokenCount, highTokenThreshold) 207 | } 208 | } 209 | 210 | return normalizedContent, isLargeFile, nil 211 | } 212 | 213 | // normalizeContent trims surrounding whitespace and trailing spaces from each line 214 | // of the input text, then returns the transformed string. 215 | func (s *MarkdownSerializer) normalizeContent(content string) string { 216 | content = strings.TrimSpace(content) 217 | lines := strings.Split(content, "\n") 218 | 219 | for i, line := range lines { 220 | lines[i] = strings.TrimRight(line, " \t") 221 | } 222 | 223 | return strings.Join(lines, "\n") 224 | } 225 | -------------------------------------------------------------------------------- /internal/serializer/serializer.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "path/filepath" 8 | "sort" 9 | "strings" 10 | 11 | "github.com/foresturquhart/grimoire/internal/secrets" 12 | ) 13 | 14 | // RedactionInfo contains information about secrets that need to be redacted. 15 | type RedactionInfo struct { 16 | // Enabled indicates whether redaction is enabled. 17 | Enabled bool 18 | 19 | // Findings contains all the secrets that were detected. 20 | Findings []secrets.Finding 21 | 22 | // BaseDir is the base directory, used to normalize paths. 23 | BaseDir string 24 | } 25 | 26 | // Serializer defines an interface for serializing multiple files into a desired format. 27 | // Implementations should handle the specifics of formatting and output. 28 | type Serializer interface { 29 | // Serialize writes the contents of the specified files, located relative to baseDir, 30 | // into the provided writer in a serialized format. 31 | // If showTree is true, it includes a directory tree visualization. 32 | // If redactionInfo is not nil, secrets should be redacted from the output. 33 | // It returns an error if the serialization process fails. 34 | // largeFileSizeThreshold defines the size in bytes above which a file is considered "large" 35 | // and a warning will be logged. 36 | // highTokenThreshold defines the token count above which a file is considered 37 | // to have a high token count and a warning will be logged. 38 | // skipTokenCount indicates whether to skip token counting entirely for warnings. 39 | Serialize(writer io.Writer, baseDir string, filePaths []string, showTree bool, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) error 40 | } 41 | 42 | // NewSerializer creates serializers based on the specified format string 43 | func NewSerializer(format string) (Serializer, error) { 44 | switch format { 45 | case "md", "markdown": 46 | return NewMarkdownSerializer(), nil 47 | case "xml": 48 | return NewXMLSerializer(), nil 49 | case "txt", "text", "plain", "plaintext": 50 | return NewPlainTextSerializer(), nil 51 | default: 52 | return nil, fmt.Errorf("unsupported format: %s", format) 53 | } 54 | } 55 | 56 | // GetFindingsForFile returns all findings for a specific file 57 | func GetFindingsForFile(redactionInfo *RedactionInfo, filePath string, baseDir string) []secrets.Finding { 58 | if redactionInfo == nil || !redactionInfo.Enabled || len(redactionInfo.Findings) == 0 { 59 | return nil 60 | } 61 | 62 | absPath := filepath.Join(baseDir, filePath) 63 | var fileFindings []secrets.Finding 64 | 65 | for _, finding := range redactionInfo.Findings { 66 | if finding.File == absPath { 67 | fileFindings = append(fileFindings, finding) 68 | } 69 | } 70 | 71 | return fileFindings 72 | } 73 | 74 | // RedactSecrets takes original content and redacts all secrets specified in the findings 75 | func RedactSecrets(content string, findings []secrets.Finding) string { 76 | if len(findings) == 0 { 77 | return content 78 | } 79 | 80 | // Line-based replacement for findings that include line numbers 81 | lineBasedFindings := make([]secrets.Finding, 0) 82 | generalFindings := make([]secrets.Finding, 0) 83 | 84 | // Separate findings into line-based and general 85 | for _, finding := range findings { 86 | if finding.Line > 0 { 87 | lineBasedFindings = append(lineBasedFindings, finding) 88 | } else { 89 | generalFindings = append(generalFindings, finding) 90 | } 91 | } 92 | 93 | // If we have line-based findings, use line-by-line replacement 94 | if len(lineBasedFindings) > 0 { 95 | return redactByLine(content, lineBasedFindings, generalFindings) 96 | } 97 | 98 | // Sort findings by secret length in descending order (longest first) 99 | // This helps avoid substring replacement issues 100 | sort.Slice(generalFindings, func(i, j int) bool { 101 | return len(generalFindings[i].Secret) > len(generalFindings[j].Secret) 102 | }) 103 | 104 | // Handle general replacements 105 | redactedContent := content 106 | for _, finding := range generalFindings { 107 | if finding.Secret == "" { 108 | continue 109 | } 110 | redactionNotice := "[REDACTED SECRET: " + finding.Description + "]" 111 | redactedContent = strings.Replace(redactedContent, finding.Secret, redactionNotice, -1) 112 | } 113 | 114 | return redactedContent 115 | } 116 | 117 | // redactByLine performs redaction on a line-by-line basis, which is more accurate 118 | // when line numbers are available in the findings 119 | func redactByLine(content string, lineBasedFindings []secrets.Finding, generalFindings []secrets.Finding) string { 120 | // Group findings by line number for efficient lookup 121 | findingsByLine := make(map[int][]secrets.Finding) 122 | for _, finding := range lineBasedFindings { 123 | findingsByLine[finding.Line] = append(findingsByLine[finding.Line], finding) 124 | } 125 | 126 | // Sort general findings by length (longest first) 127 | sort.Slice(generalFindings, func(i, j int) bool { 128 | return len(generalFindings[i].Secret) > len(generalFindings[j].Secret) 129 | }) 130 | 131 | var result strings.Builder 132 | scanner := bufio.NewScanner(strings.NewReader(content)) 133 | lineNum := 1 134 | 135 | for scanner.Scan() { 136 | line := scanner.Text() 137 | 138 | // First apply line-specific redactions 139 | if findings, ok := findingsByLine[lineNum]; ok { 140 | // For each line, sort by secret length (longest first) 141 | sort.Slice(findings, func(i, j int) bool { 142 | return len(findings[i].Secret) > len(findings[j].Secret) 143 | }) 144 | 145 | // Apply redactions for this line 146 | for _, finding := range findings { 147 | if finding.Secret == "" { 148 | continue 149 | } 150 | redactionNotice := "[REDACTED SECRET: " + finding.Description + "]" 151 | line = strings.Replace(line, finding.Secret, redactionNotice, -1) 152 | } 153 | } 154 | 155 | // Then apply general redactions that might span multiple lines 156 | for _, finding := range generalFindings { 157 | if finding.Secret == "" { 158 | continue 159 | } 160 | redactionNotice := "[REDACTED SECRET: " + finding.Description + "]" 161 | line = strings.Replace(line, finding.Secret, redactionNotice, -1) 162 | } 163 | 164 | result.WriteString(line) 165 | result.WriteString("\n") 166 | lineNum++ 167 | } 168 | 169 | // Remove the trailing newline if the original content didn't have one 170 | resultStr := result.String() 171 | if !strings.HasSuffix(content, "\n") && strings.HasSuffix(resultStr, "\n") { 172 | resultStr = resultStr[:len(resultStr)-1] 173 | } 174 | 175 | return resultStr 176 | } 177 | -------------------------------------------------------------------------------- /internal/serializer/text.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/foresturquhart/grimoire/internal/tokens" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | // PlainTextSerializer provides methods to write multiple files' contents into a 16 | // plain text formatted document with separator lines for headings and files. 17 | type PlainTextSerializer struct{} 18 | 19 | // NewPlainTextSerializer returns a new instance of PlainTextSerializer. 20 | func NewPlainTextSerializer() *PlainTextSerializer { 21 | return &PlainTextSerializer{} 22 | } 23 | 24 | // Serialize takes a list of file paths relative to baseDir, reads and normalizes 25 | // each file's content, and writes them to writer in plain text format with separators. 26 | // If reading any file fails, it logs a warning and skips that file. 27 | // If showTree is true, it includes a directory tree visualization. 28 | // If redactionInfo is not nil, it redacts secrets from the output. 29 | // largeFileSizeThreshold defines the size in bytes above which a file is considered "large" 30 | // and a warning will be logged. 31 | // highTokenThreshold defines the token count above which a file is considered 32 | // to have a high token count and a warning will be logged. 33 | // skipTokenCount indicates whether to skip token counting entirely for warnings. 34 | func (s *PlainTextSerializer) Serialize(writer io.Writer, baseDir string, filePaths []string, showTree bool, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) error { 35 | // Write the header with timestamp 36 | timestamp := time.Now().UTC().Format(time.RFC3339Nano) 37 | 38 | header := fmt.Sprintf("This document contains a structured representation of the entire codebase, merging all files into a single plain text file.\n\nGenerated by Grimoire on: %s\n\n", timestamp) 39 | 40 | if _, err := writer.Write([]byte(header)); err != nil { 41 | return fmt.Errorf("failed to write header content: %w", err) 42 | } 43 | 44 | if _, err := writer.Write([]byte(s.formatHeading("Summary"))); err != nil { 45 | return fmt.Errorf("failed to write header: %w", err) 46 | } 47 | 48 | // Write the summary section 49 | summary := "This file contains a packed representation of the entire codebase's contents. " 50 | summary += "It is designed to be easily consumable by AI systems for analysis, code review, or other automated processes.\n\n" 51 | summary += "- This file should be treated as read-only. Any changes should be made to the original codebase files.\n" 52 | summary += "- When processing this file, use the file path headings to distinguish between different files.\n" 53 | summary += "- This file may contain sensitive information and should be handled with appropriate care.\n" 54 | 55 | if redactionInfo != nil && redactionInfo.Enabled { 56 | summary += "- Detected secrets have been redacted with the format [REDACTED SECRET: description].\n" 57 | } 58 | 59 | summary += "- Some files may have been excluded based on .gitignore rules and Grimoire's configuration.\n" 60 | 61 | if showTree { 62 | summary += "- The file begins with this summary, followed by the directory structure, and then includes all codebase files.\n\n" 63 | } else { 64 | summary += "- The file begins with this summary, followed by all codebase files.\n\n" 65 | } 66 | 67 | if _, err := writer.Write([]byte(summary)); err != nil { 68 | return fmt.Errorf("failed to write summary: %w", err) 69 | } 70 | 71 | // Add directory tree if requested 72 | if showTree && len(filePaths) > 0 { 73 | if _, err := writer.Write([]byte(s.formatHeading("Directory Structure"))); err != nil { 74 | return fmt.Errorf("failed to write directory tree heading: %w", err) 75 | } 76 | 77 | treeGen := NewDefaultTreeGenerator() 78 | rootNode := treeGen.GenerateTree(filePaths) 79 | 80 | treeContent := s.renderTreeAsPlainText(rootNode, 0) 81 | treeContent += "\n" 82 | 83 | if _, err := writer.Write([]byte(treeContent)); err != nil { 84 | return fmt.Errorf("failed to write directory tree: %w", err) 85 | } 86 | } 87 | 88 | // Write files heading 89 | if _, err := writer.Write([]byte(s.formatHeading("Files"))); err != nil { 90 | return fmt.Errorf("failed to write files heading: %w", err) 91 | } 92 | 93 | // Process each file 94 | for _, relPath := range filePaths { 95 | // Write the file heading 96 | fileHeading := s.formatFileHeading(relPath) 97 | if _, err := writer.Write([]byte(fileHeading)); err != nil { 98 | return fmt.Errorf("failed to write heading for %s: %w", relPath, err) 99 | } 100 | 101 | // Read and normalize file content 102 | content, isLargeFile, err := s.readAndNormalizeContent(baseDir, relPath, redactionInfo, largeFileSizeThreshold, highTokenThreshold, skipTokenCount) 103 | if err != nil { 104 | log.Warn().Err(err).Msgf("Skipping file %s due to read error", relPath) 105 | continue 106 | } 107 | 108 | if isLargeFile { 109 | log.Warn().Msgf("File %s exceeds the large file threshold (%d bytes). Including in output but this may impact performance.", relPath, largeFileSizeThreshold) 110 | } 111 | 112 | // Check if the file is minified (only if applicable file type) 113 | if IsMinifiedFile(content, relPath, DefaultMinifiedFileThresholds) { 114 | log.Warn().Msgf("File %s appears to be minified. Consider excluding it to reduce token counts.", relPath) 115 | } 116 | 117 | // Write content with spacing 118 | if _, err := writer.Write([]byte(content + "\n\n")); err != nil { 119 | return fmt.Errorf("failed to write content for %s: %w", relPath, err) 120 | } 121 | } 122 | 123 | return nil 124 | } 125 | 126 | // formatHeading creates a main section heading with separator lines 127 | func (s *PlainTextSerializer) formatHeading(heading string) string { 128 | separator := strings.Repeat("=", 64) + "\n" 129 | return separator + heading + "\n" + separator + "\n" 130 | } 131 | 132 | // formatFileHeading creates a file heading with shorter separator lines 133 | func (s *PlainTextSerializer) formatFileHeading(path string) string { 134 | separator := strings.Repeat("=", 16) + "\n" 135 | return separator + "File: " + path + "\n" + separator + "\n" 136 | } 137 | 138 | // renderTreeAsPlainText recursively builds a plain text representation of the tree. 139 | func (s *PlainTextSerializer) renderTreeAsPlainText(node *TreeNode, depth int) string { 140 | if node == nil { 141 | return "" 142 | } 143 | 144 | var builder strings.Builder 145 | 146 | // Skip the root node in output 147 | if node.Name != "" { 148 | // Create indentation based on depth 149 | indent := strings.Repeat(" ", depth) 150 | 151 | // Add name and directory indicator if applicable 152 | builder.WriteString(indent) 153 | builder.WriteString(node.Name) 154 | 155 | if node.IsDir { 156 | builder.WriteString("/") 157 | } 158 | 159 | builder.WriteString("\n") 160 | } 161 | 162 | // Process children with incremented depth 163 | for _, child := range node.Children { 164 | childDepth := depth 165 | if node.Name != "" { 166 | // Only increment depth for non-root nodes 167 | childDepth++ 168 | } 169 | builder.WriteString(s.renderTreeAsPlainText(child, childDepth)) 170 | } 171 | 172 | return builder.String() 173 | } 174 | 175 | // readAndNormalizeContent reads a file from baseDir/relPath and normalizes its 176 | // content by trimming surrounding whitespace and trailing spaces on each line. 177 | // If redactionInfo is not nil, it redacts secrets from the output. 178 | // It also checks if the file exceeds the large file size threshold and returns a flag if it does. 179 | func (s *PlainTextSerializer) readAndNormalizeContent(baseDir, relPath string, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) (string, bool, error) { 180 | fullPath := filepath.Join(baseDir, relPath) 181 | 182 | // Check file size before reading 183 | fileInfo, err := os.Stat(fullPath) 184 | if err != nil { 185 | return "", false, fmt.Errorf("failed to stat file %s: %w", fullPath, err) 186 | } 187 | 188 | // Check if file exceeds large file threshold 189 | isLargeFile := fileInfo.Size() > largeFileSizeThreshold 190 | 191 | contentBytes, err := os.ReadFile(fullPath) 192 | if err != nil { 193 | return "", false, fmt.Errorf("failed to read file %s: %w", fullPath, err) 194 | } 195 | 196 | // Convert bytes to string and normalize 197 | content := string(contentBytes) 198 | normalizedContent := s.normalizeContent(content) 199 | 200 | // If redaction is enabled, redact any secrets 201 | if redactionInfo != nil && redactionInfo.Enabled { 202 | fileFindings := GetFindingsForFile(redactionInfo, relPath, baseDir) 203 | if len(fileFindings) > 0 { 204 | normalizedContent = RedactSecrets(normalizedContent, fileFindings) 205 | } 206 | } 207 | 208 | // Count tokens for this file and warn if it exceeds the threshold 209 | if !skipTokenCount && highTokenThreshold > 0 { 210 | tokenCount, err := tokens.CountFileTokens(relPath, normalizedContent) 211 | if err != nil { 212 | log.Warn().Err(err).Msgf("Failed to count tokens for file %s", relPath) 213 | } else if tokenCount > highTokenThreshold { 214 | log.Warn().Msgf("File %s has a high token count (%d tokens, threshold: %d). This will consume significant LLM context.", relPath, tokenCount, highTokenThreshold) 215 | } 216 | } 217 | 218 | return normalizedContent, isLargeFile, nil 219 | } 220 | 221 | // normalizeContent trims surrounding whitespace and trailing spaces from each line 222 | // of the input text, then returns the transformed string. 223 | func (s *PlainTextSerializer) normalizeContent(content string) string { 224 | content = strings.TrimSpace(content) 225 | lines := strings.Split(content, "\n") 226 | 227 | for i, line := range lines { 228 | lines[i] = strings.TrimRight(line, " \t") 229 | } 230 | 231 | return strings.Join(lines, "\n") 232 | } 233 | -------------------------------------------------------------------------------- /internal/serializer/tree_generator.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "path/filepath" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | // TreeGenerator defines an interface for generating a directory tree representation. 10 | type TreeGenerator interface { 11 | // GenerateTree takes a list of file paths and returns a tree structure representation. 12 | GenerateTree(filePaths []string) *TreeNode 13 | } 14 | 15 | // TreeNode represents a node in the directory tree. 16 | type TreeNode struct { 17 | Name string 18 | IsDir bool 19 | Children []*TreeNode 20 | } 21 | 22 | // DefaultTreeGenerator is a concrete implementation of TreeGenerator that creates 23 | // a tree structure from a list of file paths. 24 | type DefaultTreeGenerator struct{} 25 | 26 | // NewDefaultTreeGenerator returns a new instance of DefaultTreeGenerator. 27 | func NewDefaultTreeGenerator() *DefaultTreeGenerator { 28 | return &DefaultTreeGenerator{} 29 | } 30 | 31 | // GenerateTree takes a slice of file paths and builds a tree structure. 32 | // It returns the root node of the tree. 33 | func (g *DefaultTreeGenerator) GenerateTree(filePaths []string) *TreeNode { 34 | if len(filePaths) == 0 { 35 | return &TreeNode{ 36 | Name: "", 37 | IsDir: true, 38 | } 39 | } 40 | 41 | // Sort the paths for consistent output 42 | sort.Strings(filePaths) 43 | 44 | // Build a tree structure 45 | root := &TreeNode{ 46 | Name: "", 47 | IsDir: true, 48 | Children: []*TreeNode{}, 49 | } 50 | 51 | // Map to track nodes by path for efficient lookups 52 | nodeMap := make(map[string]*TreeNode) 53 | nodeMap[""] = root 54 | 55 | // Add all files to the tree 56 | for _, path := range filePaths { 57 | parts := strings.Split(filepath.ToSlash(path), "/") 58 | currentPath := "" 59 | 60 | // Build the tree structure 61 | for i, part := range parts { 62 | isFile := i == len(parts)-1 63 | 64 | // Construct parent path and current path 65 | parentPath := currentPath 66 | if currentPath == "" { 67 | currentPath = part 68 | } else { 69 | currentPath = currentPath + "/" + part 70 | } 71 | 72 | // Check if node already exists 73 | if _, exists := nodeMap[currentPath]; !exists { 74 | // Create new node 75 | newNode := &TreeNode{ 76 | Name: part, 77 | IsDir: !isFile, 78 | Children: []*TreeNode{}, 79 | } 80 | 81 | // Add to parent's children 82 | parentNode := nodeMap[parentPath] 83 | parentNode.Children = append(parentNode.Children, newNode) 84 | 85 | // Sort children by name 86 | sort.Slice(parentNode.Children, func(i, j int) bool { 87 | // Directories come before files 88 | if parentNode.Children[i].IsDir != parentNode.Children[j].IsDir { 89 | return parentNode.Children[i].IsDir 90 | } 91 | // Alphabetical order within same type 92 | return parentNode.Children[i].Name < parentNode.Children[j].Name 93 | }) 94 | 95 | // Add to map 96 | nodeMap[currentPath] = newNode 97 | } 98 | } 99 | } 100 | 101 | return root 102 | } 103 | -------------------------------------------------------------------------------- /internal/serializer/xml.go: -------------------------------------------------------------------------------- 1 | package serializer 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/foresturquhart/grimoire/internal/tokens" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | // XMLSerializer provides methods to write multiple files' contents into an 16 | // XML-formatted document optimized for LLM parsing. 17 | type XMLSerializer struct{} 18 | 19 | // NewXMLSerializer returns a new instance of XMLSerializer. 20 | func NewXMLSerializer() *XMLSerializer { 21 | return &XMLSerializer{} 22 | } 23 | 24 | // Serialize takes a list of file paths relative to baseDir, reads and normalizes 25 | // each file's content, and writes them to writer in a simplified XML format for LLMs. 26 | // If reading any file fails, it logs a warning and skips that file. 27 | // If showTree is true, it includes a plain text directory tree visualization. 28 | // If redactionInfo is not nil, it redacts secrets from the output. 29 | // largeFileSizeThreshold defines the size in bytes above which a file is considered "large" 30 | // and a warning will be logged. 31 | // highTokenThreshold defines the token count above which a file is considered 32 | // to have a high token count and a warning will be logged. 33 | // skipTokenCount indicates whether to skip token counting entirely for warnings. 34 | func (s *XMLSerializer) Serialize(writer io.Writer, baseDir string, filePaths []string, showTree bool, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) error { 35 | // Write header as plain text before XML content 36 | timestamp := time.Now().UTC().Format(time.RFC3339Nano) 37 | header := fmt.Sprintf("This document contains a structured representation of the entire codebase, merging all files into a single XML file.\n\nGenerated by Grimoire on: %s\n\n", timestamp) 38 | 39 | if _, err := writer.Write([]byte(header)); err != nil { 40 | return fmt.Errorf("failed to write header: %w", err) 41 | } 42 | 43 | // Write the summary section 44 | summary := "\n" 45 | summary += "This file contains a packed representation of the entire codebase's contents. " 46 | summary += "It is designed to be easily consumable by AI systems for analysis, code review, or other automated processes.\n\n" 47 | summary += "- This file should be treated as read-only. Any changes should be made to the original codebase files.\n" 48 | summary += "- When processing this file, use the file path attributes to distinguish between different files.\n" 49 | summary += "- This file may contain sensitive information and should be handled with appropriate care.\n" 50 | 51 | if redactionInfo != nil && redactionInfo.Enabled { 52 | summary += "- Detected secrets have been redacted with the format [REDACTED SECRET: description].\n" 53 | } 54 | 55 | summary += "- Some files may have been excluded based on .gitignore rules and Grimoire's configuration.\n" 56 | 57 | if showTree { 58 | summary += "- The file begins with this summary, followed by the directory structure, and then includes all codebase files.\n" 59 | } else { 60 | summary += "- The file begins with this summary, followed by all codebase files.\n" 61 | } 62 | 63 | summary += "\n\n" 64 | 65 | if _, err := writer.Write([]byte(summary)); err != nil { 66 | return fmt.Errorf("failed to write summary: %w", err) 67 | } 68 | 69 | // Add directory tree if requested 70 | if showTree && len(filePaths) > 0 { 71 | if _, err := writer.Write([]byte("\n")); err != nil { 72 | return fmt.Errorf("failed to write directory structure opening tag: %w", err) 73 | } 74 | 75 | treeGen := NewDefaultTreeGenerator() 76 | rootNode := treeGen.GenerateTree(filePaths) 77 | 78 | // Generate plain text tree with indentation 79 | treeContent := s.renderTreeAsPlainText(rootNode, 0) 80 | 81 | if _, err := writer.Write([]byte(treeContent)); err != nil { 82 | return fmt.Errorf("failed to write directory tree content: %w", err) 83 | } 84 | 85 | if _, err := writer.Write([]byte("\n\n")); err != nil { 86 | return fmt.Errorf("failed to write directory structure closing tag: %w", err) 87 | } 88 | } 89 | 90 | // Write files opening tag 91 | if _, err := writer.Write([]byte("\n")); err != nil { 92 | return fmt.Errorf("failed to write files opening tag: %w", err) 93 | } 94 | 95 | // Process each file 96 | for _, relPath := range filePaths { 97 | // Read and normalize file content 98 | content, isLargeFile, err := s.readAndNormalizeContent(baseDir, relPath, redactionInfo, largeFileSizeThreshold, highTokenThreshold, skipTokenCount) 99 | if err != nil { 100 | log.Warn().Err(err).Msgf("Skipping file %s due to read error", relPath) 101 | continue 102 | } 103 | 104 | if isLargeFile { 105 | log.Warn().Msgf("File %s exceeds the large file threshold (%d bytes). Including in output but this may impact performance.", relPath, largeFileSizeThreshold) 106 | } 107 | 108 | // Check if the file is minified (only if applicable file type) 109 | if IsMinifiedFile(content, relPath, DefaultMinifiedFileThresholds) { 110 | log.Warn().Msgf("File %s appears to be minified. Consider excluding it to reduce token counts.", relPath) 111 | } 112 | 113 | // Write file tag with path attribute 114 | fileOpenTag := fmt.Sprintf("\n", relPath) 115 | if _, err := writer.Write([]byte(fileOpenTag)); err != nil { 116 | return fmt.Errorf("failed to write file opening tag for %s: %w", relPath, err) 117 | } 118 | 119 | // Write file content directly inside the file tag 120 | if _, err := writer.Write([]byte(content)); err != nil { 121 | return fmt.Errorf("failed to write content for %s: %w", relPath, err) 122 | } 123 | 124 | // Write file closing tag 125 | if _, err := writer.Write([]byte("\n\n")); err != nil { 126 | return fmt.Errorf("failed to write file closing tag for %s: %w", relPath, err) 127 | } 128 | } 129 | 130 | // Write files closing tag 131 | if _, err := writer.Write([]byte("\n")); err != nil { 132 | return fmt.Errorf("failed to write files closing tag: %w", err) 133 | } 134 | 135 | return nil 136 | } 137 | 138 | // renderTreeAsPlainText recursively builds a plain text representation of the tree with 139 | // indentation for easier LLM parsing. 140 | func (s *XMLSerializer) renderTreeAsPlainText(node *TreeNode, depth int) string { 141 | if node == nil { 142 | return "" 143 | } 144 | 145 | var builder strings.Builder 146 | 147 | // Skip the root node in output 148 | if node.Name != "" { 149 | // Create indentation based on depth 150 | indent := strings.Repeat(" ", depth) 151 | 152 | // Add name and directory indicator if applicable 153 | builder.WriteString(indent) 154 | builder.WriteString(node.Name) 155 | 156 | if node.IsDir { 157 | builder.WriteString("/") 158 | } 159 | 160 | builder.WriteString("\n") 161 | } 162 | 163 | // Process children with incremented depth 164 | for _, child := range node.Children { 165 | childDepth := depth 166 | if node.Name != "" { 167 | // Only increment depth for non-root nodes 168 | childDepth++ 169 | } 170 | builder.WriteString(s.renderTreeAsPlainText(child, childDepth)) 171 | } 172 | 173 | return builder.String() 174 | } 175 | 176 | // readAndNormalizeContent reads a file from baseDir/relPath and normalizes its 177 | // content by trimming surrounding whitespace and trailing spaces on each line. 178 | // If redactionInfo is not nil, it redacts secrets from the output. 179 | // It also checks if the file exceeds the large file size threshold and returns a flag if it does. 180 | func (s *XMLSerializer) readAndNormalizeContent(baseDir, relPath string, redactionInfo *RedactionInfo, largeFileSizeThreshold int64, highTokenThreshold int, skipTokenCount bool) (string, bool, error) { 181 | fullPath := filepath.Join(baseDir, relPath) 182 | 183 | // Check file size before reading 184 | fileInfo, err := os.Stat(fullPath) 185 | if err != nil { 186 | return "", false, fmt.Errorf("failed to stat file %s: %w", fullPath, err) 187 | } 188 | 189 | // Check if file exceeds large file threshold 190 | isLargeFile := fileInfo.Size() > largeFileSizeThreshold 191 | 192 | contentBytes, err := os.ReadFile(fullPath) 193 | if err != nil { 194 | return "", false, fmt.Errorf("failed to read file %s: %w", fullPath, err) 195 | } 196 | 197 | // Convert bytes to string and normalize 198 | content := string(contentBytes) 199 | normalizedContent := s.normalizeContent(content) 200 | 201 | // If redaction is enabled, redact any secrets 202 | if redactionInfo != nil && redactionInfo.Enabled { 203 | fileFindings := GetFindingsForFile(redactionInfo, relPath, baseDir) 204 | if len(fileFindings) > 0 { 205 | normalizedContent = RedactSecrets(normalizedContent, fileFindings) 206 | } 207 | } 208 | 209 | // Count tokens for this file and warn if it exceeds the threshold 210 | if !skipTokenCount && highTokenThreshold > 0 { 211 | tokenCount, err := tokens.CountFileTokens(relPath, normalizedContent) 212 | if err != nil { 213 | log.Warn().Err(err).Msgf("Failed to count tokens for file %s", relPath) 214 | } else if tokenCount > highTokenThreshold { 215 | log.Warn().Msgf("File %s has a high token count (%d tokens, threshold: %d). This will consume significant LLM context.", relPath, tokenCount, highTokenThreshold) 216 | } 217 | } 218 | 219 | return normalizedContent, isLargeFile, nil 220 | } 221 | 222 | // normalizeContent trims surrounding whitespace and trailing spaces from each line 223 | // of the input text, then returns the transformed string. 224 | func (s *XMLSerializer) normalizeContent(content string) string { 225 | content = strings.TrimSpace(content) 226 | lines := strings.Split(content, "\n") 227 | 228 | for i, line := range lines { 229 | lines[i] = strings.TrimRight(line, " \t") 230 | } 231 | 232 | return strings.Join(lines, "\n") 233 | } 234 | -------------------------------------------------------------------------------- /internal/tokens/counter.go: -------------------------------------------------------------------------------- 1 | package tokens 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/tiktoken-go/tokenizer" 8 | ) 9 | 10 | // encoderCache provides a singleton instance of the encoder to avoid repeated initialization 11 | var ( 12 | encoder tokenizer.Codec 13 | encoderOnce sync.Once 14 | encoderErr error 15 | ) 16 | 17 | // getEncoder returns a cached encoder instance to avoid repeated initialization costs 18 | func getEncoder() (tokenizer.Codec, error) { 19 | encoderOnce.Do(func() { 20 | encoder, encoderErr = tokenizer.Get(tokenizer.O200kBase) 21 | }) 22 | return encoder, encoderErr 23 | } 24 | 25 | // CountTokens counts the number of tokens in the provided text using the specified encoder. 26 | // It returns the token count and any error that occurred during counting. 27 | func CountTokens(text string) (int, error) { 28 | enc, err := getEncoder() 29 | if err != nil { 30 | return 0, err 31 | } 32 | 33 | count, err := enc.Count(text) 34 | if err != nil { 35 | return 0, err 36 | } 37 | 38 | return count, nil 39 | } 40 | 41 | // StreamingTokenCounter maintains an incremental token count for streaming content 42 | type StreamingTokenCounter struct { 43 | enc tokenizer.Codec 44 | tokenCount int 45 | mu sync.Mutex 46 | } 47 | 48 | // NewStreamingCounter creates a new streaming token counter 49 | func NewStreamingCounter() (*StreamingTokenCounter, error) { 50 | enc, err := getEncoder() 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return &StreamingTokenCounter{ 56 | enc: enc, 57 | tokenCount: 0, 58 | }, nil 59 | } 60 | 61 | // AddText adds text to the counter and updates the token count 62 | func (c *StreamingTokenCounter) AddText(text string) error { 63 | if text == "" { 64 | return nil 65 | } 66 | 67 | c.mu.Lock() 68 | defer c.mu.Unlock() 69 | 70 | count, err := c.enc.Count(text) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | c.tokenCount += count 76 | return nil 77 | } 78 | 79 | // TokenCount returns the current token count 80 | func (c *StreamingTokenCounter) TokenCount() int { 81 | c.mu.Lock() 82 | defer c.mu.Unlock() 83 | return c.tokenCount 84 | } 85 | 86 | // CountFileTokens counts the tokens in a specific file content and returns 87 | // the count along with any error. This is useful for per-file token analysis. 88 | func CountFileTokens(filePath, content string) (int, error) { 89 | if content == "" { 90 | return 0, nil 91 | } 92 | 93 | // Reuse the shared encoder instance 94 | enc, err := getEncoder() 95 | if err != nil { 96 | return 0, fmt.Errorf("failed to get encoder for file %s: %w", filePath, err) 97 | } 98 | 99 | count, err := enc.Count(content) 100 | if err != nil { 101 | return 0, fmt.Errorf("failed to count tokens for file %s: %w", filePath, err) 102 | } 103 | 104 | return count, nil 105 | } 106 | -------------------------------------------------------------------------------- /internal/tokens/writer.go: -------------------------------------------------------------------------------- 1 | package tokens 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "sync" 7 | ) 8 | 9 | // CaptureWriter is a writer that captures written content while also writing to the underlying writer. 10 | // This allows both writing to the destination (file or stdout) while keeping a copy of the data 11 | // for token counting. 12 | type CaptureWriter struct { 13 | Writer io.Writer // The actual destination writer 14 | Buffer *bytes.Buffer // Buffer that captures a copy of all written content (used for legacy mode) 15 | Counter *StreamingTokenCounter // Incremental token counter (used in streaming mode) 16 | TokenCount int // Stores the counted tokens after processing 17 | mu sync.Mutex // Mutex to protect concurrent writes 18 | chunkSize int // Size of chunks for streaming processing 19 | } 20 | 21 | // TokenCounterOptions configures the behavior of the CaptureWriter 22 | type TokenCounterOptions struct { 23 | ChunkSize int // Size of chunks for streaming (default 4096) 24 | } 25 | 26 | // NewCaptureWriter creates a new CaptureWriter wrapping the provided writer. 27 | func NewCaptureWriter(w io.Writer, opts *TokenCounterOptions) (*CaptureWriter, error) { 28 | cw := &CaptureWriter{ 29 | Writer: w, 30 | Buffer: &bytes.Buffer{}, 31 | } 32 | 33 | if opts != nil { 34 | if opts.ChunkSize > 0 { 35 | cw.chunkSize = opts.ChunkSize 36 | } else { 37 | cw.chunkSize = 4096 // Default chunk size 38 | } 39 | } 40 | 41 | return cw, nil 42 | } 43 | 44 | // Write implements the io.Writer interface, writing data to both the underlying writer 45 | // and handling token counting according to the configured mode. 46 | func (cw *CaptureWriter) Write(p []byte) (n int, err error) { 47 | cw.mu.Lock() 48 | defer cw.mu.Unlock() 49 | 50 | if _, err := cw.Buffer.Write(p); err != nil { 51 | return 0, err 52 | } 53 | 54 | // Write to the actual destination 55 | return cw.Writer.Write(p) 56 | } 57 | 58 | // CountTokens counts the tokens in the captured content and stores the result 59 | // in the TokenCount field. In streaming mode, this just returns the current count. 60 | func (cw *CaptureWriter) CountTokens() error { 61 | cw.mu.Lock() 62 | defer cw.mu.Unlock() 63 | 64 | count, err := CountTokens(cw.Buffer.String()) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | cw.TokenCount = count 70 | return nil 71 | } 72 | 73 | // GetTokenCount returns the current token count 74 | func (cw *CaptureWriter) GetTokenCount() int { 75 | return cw.TokenCount 76 | } 77 | --------------------------------------------------------------------------------