├── .github └── CODEOWNERS ├── .gitignore ├── example.gitignore ├── .context ├── .ai-context.yml ├── generated │ └── generated_context.md └── generate.py ├── src ├── test1.js ├── test3.ts └── test2.tsx └── README.md /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | @temrb -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /example.gitignore: -------------------------------------------------------------------------------- 1 | /**/generated/ 2 | .context/generated/ -------------------------------------------------------------------------------- /.context/.ai-context.yml: -------------------------------------------------------------------------------- 1 | # .ai-context.yml (example) 2 | # This is the configuration file for the AI Context Generator. 3 | # Settings here will be used automatically when you run the script. 4 | 5 | # Specify a project type preset (e.g., python, javascript, go, auto). 6 | # This determines the initial set of files to include and exclude. 7 | preset: javascript 8 | 9 | # A list of directories or files to scan. If this is not specified, 10 | # it defaults to the current directory ["."]. 11 | paths: 12 | - src 13 | 14 | include: 15 | - '*.js' 16 | - '*.tsx' 17 | - '*.ts' 18 | - '*.json' 19 | - '*.html' 20 | - '*.css' 21 | 22 | exclude: 23 | - 'node_modules/**' 24 | - 'dist/**' 25 | - 'build/**' 26 | - '.git/**' 27 | - '*.log' 28 | - '*.md' 29 | 30 | # The name and format of the final output file. 31 | output: generated_context.md 32 | format: markdown # Can be: text, markdown, or json 33 | 34 | # Add custom glob patterns to the preset's lists. 35 | # These are APPENDED to the preset's rules, they don't replace them. 36 | 37 | # Set the maximum file size in megabytes. 38 | # Files larger than this will be skipped. Use 0 for no limit. 39 | max_file_size_mb: 2 40 | 41 | # Enable the experimental minification feature to remove comments and 42 | # extra whitespace from code. 43 | minify: true 44 | 45 | # Enable verbose logging to see details like which files were skipped 46 | # and why. This is useful for debugging your configuration. 47 | verbose: false 48 | -------------------------------------------------------------------------------- /src/test1.js: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Dummy script that exercises a handful of utility functions. 4 | * Run with `node src/test1.js` to see the sample output. 5 | */ 6 | 7 | class Calculator { 8 | add(a, b) { 9 | return a + b; 10 | } 11 | 12 | subtract(a, b) { 13 | return a - b; 14 | } 15 | 16 | multiply(a, b) { 17 | return a * b; 18 | } 19 | 20 | divide(a, b) { 21 | if (b === 0) { 22 | throw new Error('Cannot divide by zero'); 23 | } 24 | 25 | return a / b; 26 | } 27 | } 28 | 29 | const calculator = new Calculator(); 30 | 31 | function assertAlmostEqual(actual, expected, message) { 32 | if (Math.abs(actual - expected) > 1e-9) { 33 | throw new Error(`Assertion failed: ${message}. Expected ${expected}, got ${actual}`); 34 | } 35 | } 36 | 37 | function runCalculatorDemo() { 38 | console.log('Running calculator demo...'); 39 | 40 | const addResult = calculator.add(2, 3); 41 | console.log('2 + 3 =', addResult); 42 | assertAlmostEqual(addResult, 5, 'Addition should work'); 43 | 44 | const subtractResult = calculator.subtract(10, 4); 45 | console.log('10 - 4 =', subtractResult); 46 | assertAlmostEqual(subtractResult, 6, 'Subtraction should work'); 47 | 48 | const multiplyResult = calculator.multiply(6, 7); 49 | console.log('6 * 7 =', multiplyResult); 50 | assertAlmostEqual(multiplyResult, 42, 'Multiplication should work'); 51 | 52 | const divideResult = calculator.divide(12, 3); 53 | console.log('12 / 3 =', divideResult); 54 | assertAlmostEqual(divideResult, 4, 'Division should work'); 55 | 56 | try { 57 | calculator.divide(1, 0); 58 | } catch (error) { 59 | console.log('Expected error for divide by zero:', error.message); 60 | } 61 | 62 | console.log('All calculator checks passed!'); 63 | } 64 | 65 | if (require.main === module) { 66 | runCalculatorDemo(); 67 | } 68 | 69 | module.exports = { 70 | Calculator, 71 | runCalculatorDemo, 72 | }; 73 | -------------------------------------------------------------------------------- /src/test3.ts: -------------------------------------------------------------------------------- 1 | type Metric = { 2 | name: string; 3 | values: number[]; 4 | }; 5 | 6 | type MetricSummary = { 7 | name: string; 8 | count: number; 9 | min: number; 10 | max: number; 11 | average: number; 12 | standardDeviation: number; 13 | }; 14 | 15 | export class MetricAnalyzer { 16 | private readonly metrics: Metric[] = []; 17 | 18 | public addMetric(metric: Metric): void { 19 | let existingIndex = -1; 20 | for (let index = 0; index < this.metrics.length; index += 1) { 21 | if (this.metrics[index].name === metric.name) { 22 | existingIndex = index; 23 | break; 24 | } 25 | } 26 | if (existingIndex >= 0) { 27 | // Merge values by creating a new array to avoid mutating the caller's data. 28 | this.metrics[existingIndex] = { 29 | name: metric.name, 30 | values: this.metrics[existingIndex].values.concat(metric.values), 31 | }; 32 | } else { 33 | this.metrics.push({ 34 | name: metric.name, 35 | values: metric.values.slice(), 36 | }); 37 | } 38 | } 39 | 40 | public summarize(): MetricSummary[] { 41 | return this.metrics.map((metric) => summarizeMetric(metric)); 42 | } 43 | 44 | public clear(): void { 45 | this.metrics.length = 0; 46 | } 47 | } 48 | 49 | export const summarizeMetric = (metric: Metric): MetricSummary => { 50 | if (metric.values.length === 0) { 51 | return { 52 | name: metric.name, 53 | count: 0, 54 | min: 0, 55 | max: 0, 56 | average: 0, 57 | standardDeviation: 0, 58 | }; 59 | } 60 | 61 | let sum = 0; 62 | let min = metric.values[0]; 63 | let max = metric.values[0]; 64 | for (let index = 0; index < metric.values.length; index += 1) { 65 | const value = metric.values[index]; 66 | sum += value; 67 | if (value < min) { 68 | min = value; 69 | } 70 | if (value > max) { 71 | max = value; 72 | } 73 | } 74 | 75 | const count = metric.values.length; 76 | const average = sum / count; 77 | 78 | let squaredErrorSum = 0; 79 | for (let index = 0; index < count; index += 1) { 80 | const error = metric.values[index] - average; 81 | squaredErrorSum += error * error; 82 | } 83 | 84 | const variance = squaredErrorSum / count; 85 | const standardDeviation = Math.sqrt(variance); 86 | 87 | return { 88 | name: metric.name, 89 | count, 90 | min, 91 | max, 92 | average, 93 | standardDeviation, 94 | }; 95 | }; 96 | 97 | export const describeSummaries = (summaries: MetricSummary[]): string => { 98 | if (summaries.length === 0) { 99 | return 'No metrics recorded.'; 100 | } 101 | 102 | const lines: string[] = []; 103 | for (let index = 0; index < summaries.length; index += 1) { 104 | const summary = summaries[index]; 105 | lines.push( 106 | [ 107 | `Metric: ${summary.name}`, 108 | `Count: ${summary.count}`, 109 | `Range: ${summary.min} - ${summary.max}`, 110 | `Average: ${summary.average.toFixed(2)}`, 111 | `Std Dev: ${summary.standardDeviation.toFixed(2)}`, 112 | ].join(' | ') 113 | ); 114 | } 115 | 116 | return lines.join('\n'); 117 | }; 118 | 119 | export const generateSampleMetrics = (): Metric[] => [ 120 | { name: 'response_time_ms', values: [120, 135, 150, 90, 105, 130] }, 121 | { name: 'memory_usage_mb', values: [256, 240, 232, 280, 300] }, 122 | { name: 'cpu_percent', values: [32, 44, 27, 55, 38, 41, 36] }, 123 | ]; 124 | 125 | declare const require: unknown; 126 | declare const module: unknown; 127 | 128 | export function runMetricAnalyzerDemo(): void { 129 | const analyzer = new MetricAnalyzer(); 130 | const metrics = generateSampleMetrics(); 131 | 132 | for (let index = 0; index < metrics.length; index += 1) { 133 | analyzer.addMetric(metrics[index]); 134 | } 135 | 136 | const summaries = analyzer.summarize(); 137 | const report = describeSummaries(summaries); 138 | 139 | console.log('Metric Analyzer Report'); 140 | console.log('======================'); 141 | console.log(report); 142 | } 143 | 144 | if (typeof require !== 'undefined' && typeof module !== 'undefined') { 145 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 146 | const nodeRequire = require as any; 147 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 148 | const nodeModule = module as any; 149 | if (nodeRequire.main === nodeModule) { 150 | runMetricAnalyzerDemo(); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/test2.tsx: -------------------------------------------------------------------------------- 1 | /** @jsx h */ 2 | 3 | type Primitive = string | number | boolean | null | undefined; 4 | 5 | type JSXChild = Primitive | JSXElement; 6 | 7 | type JSXElement = { 8 | type: string; 9 | props: Record & { children?: JSXChild[] }; 10 | }; 11 | 12 | declare const require: unknown; 13 | declare const module: unknown; 14 | 15 | declare global { 16 | namespace JSX { 17 | // eslint-disable-next-line @typescript-eslint/consistent-type-definitions 18 | interface IntrinsicElements { 19 | div: Record; 20 | button: Record; 21 | h1: Record; 22 | p: Record; 23 | ul: Record; 24 | li: Record; 25 | span: Record; 26 | em: Record; 27 | } 28 | } 29 | } 30 | 31 | function h( 32 | type: string, 33 | props: Record | null, 34 | ...children: JSXChild[] 35 | ): JSXElement { 36 | return { 37 | type, 38 | props: { ...(props ?? {}), children }, 39 | }; 40 | } 41 | 42 | type TodoItem = { 43 | id: number; 44 | title: string; 45 | completed: boolean; 46 | }; 47 | 48 | const SAMPLE_TODOS: TodoItem[] = [ 49 | { id: 1, title: 'Write documentation', completed: true }, 50 | { id: 2, title: 'Ship new feature', completed: false }, 51 | { id: 3, title: 'Polish UI', completed: false }, 52 | ]; 53 | 54 | export type TodoDashboardProps = { 55 | filterCompleted?: boolean | null; 56 | }; 57 | 58 | export const TodoDashboard = ({ 59 | filterCompleted = null, 60 | }: TodoDashboardProps): JSXElement => { 61 | const todos = filterTodos(SAMPLE_TODOS, filterCompleted); 62 | const completedCount = SAMPLE_TODOS.filter((todo) => todo.completed).length; 63 | 64 | return ( 65 |
66 |

Todo Dashboard Demo

67 |

Completed tasks: {completedCount}

68 | {renderFilters(filterCompleted)} 69 | {todos.length === 0 ? renderEmptyState() : renderTodoList(todos)} 70 |
71 | ); 72 | }; 73 | 74 | export const renderFilters = (filterCompleted: boolean | null): JSXElement => ( 75 |
76 | 79 | 82 | 85 |
86 | ); 87 | 88 | export const renderEmptyState = (): JSXElement => No todos to display.; 89 | 90 | export const renderTodoList = (todos: TodoItem[]): JSXElement => ( 91 |
    92 | {todos.map((todo) => ( 93 |
  • 94 | {todo.title} 95 | {todo.completed ? '✅' : '🕒'} 96 |
    • 97 | ))} 98 |
99 | ); 100 | 101 | export const filterTodos = ( 102 | todos: TodoItem[], 103 | filterCompleted: boolean | null 104 | ): TodoItem[] => { 105 | if (filterCompleted === null) { 106 | return todos; 107 | } 108 | 109 | return todos.filter((todo) => todo.completed === filterCompleted); 110 | }; 111 | 112 | const indent = (depth: number): string => { 113 | if (depth <= 0) { 114 | return ''; 115 | } 116 | 117 | let value = ''; 118 | for (let index = 0; index < depth; index += 1) { 119 | value += ' '; 120 | } 121 | 122 | return value; 123 | }; 124 | 125 | const trimRight = (value: string): string => value.replace(/[\s\n]+$/, ''); 126 | 127 | export const renderToString = (element: JSXChild, depth = 0): string => { 128 | if (element === null || element === undefined) { 129 | return ''; 130 | } 131 | 132 | if ( 133 | typeof element === 'string' || 134 | typeof element === 'number' || 135 | typeof element === 'boolean' 136 | ) { 137 | return `${indent(depth)}${String(element)}\n`; 138 | } 139 | 140 | const { type, props } = element; 141 | const attrEntries: string[] = []; 142 | for (const key in props) { 143 | if ( 144 | !Object.prototype.hasOwnProperty.call(props, key) || 145 | key === 'children' 146 | ) { 147 | continue; 148 | } 149 | 150 | const value = props[key]; 151 | attrEntries.push(`${key}="${value}"`); 152 | } 153 | const attributes = attrEntries.length > 0 ? ` ${attrEntries.join(' ')}` : ''; 154 | const children = props.children ?? []; 155 | 156 | const opening = `${indent(depth)}<${type}${attributes}>\n`; 157 | const childMarkup = children 158 | .map((child) => renderToString(child, depth + 1)) 159 | .join(''); 160 | const closing = `${indent(depth)}\n`; 161 | 162 | return opening + childMarkup + closing; 163 | }; 164 | 165 | export function runTodoDashboardDemo(): void { 166 | const views = [null, true, false].map((filter) => ({ 167 | filter, 168 | markup: renderToString(), 169 | })); 170 | 171 | for (const { filter, markup } of views) { 172 | const label = filter === null ? 'all' : filter ? 'completed' : 'pending'; 173 | console.log(`\n--- rendering ${label} view ---`); 174 | console.log(trimRight(markup)); 175 | } 176 | } 177 | 178 | // Avoid referencing Node globals when the script is bundled for the browser. 179 | if (typeof require !== 'undefined' && typeof module !== 'undefined') { 180 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 181 | const nodeRequire = require as any; 182 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 183 | const nodeModule = module as any; 184 | if (nodeRequire.main === nodeModule) { 185 | runTodoDashboardDemo(); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AI Context Generator 2 | 3 | ![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg) 4 | ![Python Version](https://img.shields.io/badge/python-3.8+-blue.svg) 5 | [![Version](https://img.shields.io/badge/version-5.0.0-brightgreen.svg)]() 6 | 7 | A small, opinionated CLI tool that packages project source files into a single context file suitable for use with Large Language Models (LLMs). It auto-detects common project types, applies smart include/exclude filters, and can output text, Markdown, or JSON. 8 | 9 | --- 10 | 11 | ## Quick overview 12 | 13 | - Scans your project files and writes a single, easy-to-read context file. 14 | - Keeps all generator tooling in `.context/` and writes outputs to `.context/generated/`. 15 | - Supports presets (Python, JavaScript, Go, Rust, etc.), custom include/exclude patterns, file-size limits, and experimental minification. 16 | 17 | --- 18 | 19 | ## Repo layout (convention) 20 | 21 | ``` 22 | 23 | your-project/ 24 | ├── .context/ 25 | │ ├── generate.py # generator (downloaded or copied) 26 | │ ├── .ai-context.yml # optional config 27 | │ └── generated/ # outputs (gitignored by installer) 28 | ├── src/ 29 | └── ... 30 | 31 | ``` 32 | 33 | ## Install 34 | 35 | Two simple options: copy the `.context` folder from the repo (recommended) or manually download the generator. Run from your **project root**: 36 | 37 | #### MacOS / Linux / WSL / Git Bash 38 | 39 | ```bash 40 | git clone --depth=1 https://github.com/temrb/generate-project-context.git tmp_repo \ 41 | && rm -rf .context \ 42 | && mv tmp_repo/.context ./ \ 43 | && rm -rf tmp_repo \ 44 | && chmod +x .context/generate.py || true 45 | ``` 46 | 47 | - `rm -rf .context` ensures any existing `.context` is replaced cleanly. 48 | - `chmod +x` may fail silently on filesystems that don’t support execute bits — you can always run with `python3 .context/generate.py`. 49 | 50 | #### Windows PowerShell 51 | 52 | ```powershell 53 | git clone --depth=1 https://github.com/temrb/generate-project-context.git tmp_repo 54 | Remove-Item -Recurse -Force .context -ErrorAction SilentlyContinue 55 | Move-Item tmp_repo/.context . -Force 56 | Remove-Item tmp_repo -Recurse -Force 57 | ``` 58 | 59 | - Existing `.context` is removed before copying to avoid merge/nesting issues. 60 | 61 | Both commands copy the `.context` folder (including `generate.py`) into your project. PowerShell doesn’t require `chmod`, but if you later run the generator from a Unix-like shell you can add execute permissions with `chmod +x .context/generate.py`. 62 | 63 | **Tip:** Add generated outputs directory to `.gitignore` if not already ignored: 64 | 65 | ```bash 66 | echo ".context/generated/" >> .gitignore 67 | ``` 68 | 69 | ## Optional deps 70 | 71 | ```bash 72 | pip3 install pyyaml chardet tqdm 73 | ``` 74 | 75 | - `pyyaml` — read `.ai-context.yml` configs 76 | - `chardet` — improve encoding detection 77 | - `tqdm` — progress bars 78 | 79 | The generator runs without them but with reduced functionality (you’ll see warnings). 80 | 81 | --- 82 | 83 | ## Usage 84 | 85 | From your project root: 86 | 87 | ```bash 88 | # Basic (auto-detect project type) 89 | python3 .context/generate.py 90 | 91 | # Scan specific directories 92 | python3 .context/generate.py src/ tests/ 93 | 94 | # Use a preset 95 | python3 .context/generate.py --preset python 96 | 97 | # Output markdown with a custom filename (placed in .context/generated/) 98 | python3 .context/generate.py --output api_docs.md --format markdown 99 | 100 | # Enable experimental minification 101 | python3 .context/generate.py --minify 102 | 103 | # Dry run (list files that would be processed) 104 | python3 .context/generate.py --dry-run --verbose 105 | ``` 106 | 107 | --- 108 | 109 | ## Configuration 110 | 111 | Create `.context/.ai-context.yml` to pin reproducible runs: 112 | 113 | ```yaml 114 | preset: python 115 | paths: 116 | - src 117 | - lib 118 | output: context.md # written to .context/generated/ 119 | include: 120 | - '*.md' 121 | - 'Dockerfile' 122 | exclude: 123 | - '*_test.py' 124 | - 'experiments/*' 125 | max_file_size_mb: 2 126 | format: markdown # text, markdown, json 127 | minify: false 128 | verbose: false 129 | ``` 130 | 131 | **Priority:** CLI options > config file > presets > built-in defaults 132 | 133 | --- 134 | 135 | ## CLI flags (summary) 136 | 137 | - `paths` — directories/files to scan (default: project root) 138 | - `-o, --output` — output filename (default: `.context/generated/context.txt`) 139 | - `--preset` — project preset (auto | python | javascript | java | go | rust | ...) 140 | - `--include` / `--exclude` — glob patterns 141 | - `--max-file-size-mb` — skip large files (default: 1 MB) 142 | - `--format` — `text` | `markdown` | `json` 143 | - `--minify` — experimental comment/whitespace removal 144 | - `--dry-run` — list files without generating 145 | - `-v, --verbose` — verbose logging 146 | - `--version` — show version 147 | 148 | --- 149 | 150 | ## Output formats 151 | 152 | - **Text**: plain, human-readable sections with file separators. 153 | - **Markdown**: syntax-highlighted fenced code blocks per file. 154 | - **JSON**: `metadata` + `files[]` with `path`, `size`, `encoding`, `content`. 155 | 156 | Outputs are placed in `.context/generated/` by default. 157 | 158 | --- 159 | 160 | ## Behavior notes & edge cases 161 | 162 | - Skips common binary extensions (images, archives, compiled artifacts). 163 | - Encoding attempts: UTF-8 → chardet (if installed) → latin-1. 164 | - Skips empty/whitespace-only files. 165 | - Always excludes `.context/` (self-exclusion). 166 | - Minification is experimental — use with caution for production code review. 167 | 168 | --- 169 | 170 | ## Troubleshooting 171 | 172 | - **Config present but not loaded** 173 | 174 | ```text 175 | Warning: Config file .context/.ai-context.yml exists but PyYAML is not installed 176 | ``` 177 | 178 | Fix: `pip install pyyaml` 179 | 180 | - **Garbled text in output** 181 | Fix: `pip install chardet` 182 | 183 | - **No progress bar** 184 | Fix: `pip install tqdm` 185 | 186 | - **Output missing expected files** 187 | 188 | - Ensure include/exclude patterns are correct and relative to the project root. 189 | - Check `--max-file-size-mb` (defaults to 1 MB). 190 | 191 | --- 192 | 193 | ## Contributing 194 | 195 | PRs welcome. Ideas: 196 | 197 | - More accurate language-specific minifiers 198 | - Additional presets and smarter detection 199 | - Improved binary detection and performance tweaks 200 | 201 | --- 202 | 203 | ## License 204 | 205 | MIT 206 | -------------------------------------------------------------------------------- /.context/generated/generated_context.md: -------------------------------------------------------------------------------- 1 | # AI Context Report 2 | 3 | - **Version**: `5.0.0` 4 | - **Project Root**: `/Users/Documents/dev/others/project-context-generator` 5 | - **Generated**: `2025-09-26 18:15:53` 6 | - **Files discovered**: `3` 7 | 8 | --- 9 | 10 | ## `src/test1.js` 11 | 12 | ```js 13 | class Calculator { 14 | add(a, b) { 15 | return a + b; 16 | } 17 | subtract(a, b) { 18 | return a - b; 19 | } 20 | multiply(a, b) { 21 | return a * b; 22 | } 23 | divide(a, b) { 24 | if (b === 0) { 25 | throw new Error('Cannot divide by zero'); 26 | } 27 | return a / b; 28 | } 29 | } 30 | const calculator = new Calculator(); 31 | function assertAlmostEqual(actual, expected, message) { 32 | if (Math.abs(actual - expected) > 1e-9) { 33 | throw new Error( 34 | `Assertion failed: ${message}. Expected ${expected}, got ${actual}` 35 | ); 36 | } 37 | } 38 | function runCalculatorDemo() { 39 | console.log('Running calculator demo...'); 40 | const addResult = calculator.add(2, 3); 41 | console.log('2 + 3 =', addResult); 42 | assertAlmostEqual(addResult, 5, 'Addition should work'); 43 | const subtractResult = calculator.subtract(10, 4); 44 | console.log('10 - 4 =', subtractResult); 45 | assertAlmostEqual(subtractResult, 6, 'Subtraction should work'); 46 | const multiplyResult = calculator.multiply(6, 7); 47 | console.log('6 * 7 =', multiplyResult); 48 | assertAlmostEqual(multiplyResult, 42, 'Multiplication should work'); 49 | const divideResult = calculator.divide(12, 3); 50 | console.log('12 / 3 =', divideResult); 51 | assertAlmostEqual(divideResult, 4, 'Division should work'); 52 | try { 53 | calculator.divide(1, 0); 54 | } catch (error) { 55 | console.log('Expected error for divide by zero:', error.message); 56 | } 57 | console.log('All calculator checks passed!'); 58 | } 59 | if (require.main === module) { 60 | runCalculatorDemo(); 61 | } 62 | module.exports = { 63 | Calculator, 64 | runCalculatorDemo, 65 | }; 66 | ``` 67 | 68 | --- 69 | 70 | ## `src/test2.tsx` 71 | 72 | ```tsx 73 | type Primitive = string | number | boolean | null | undefined; 74 | type JSXChild = Primitive | JSXElement; 75 | type JSXElement = { 76 | type: string; 77 | props: Record & { children?: JSXChild[] }; 78 | }; 79 | declare const require: unknown; 80 | declare const module: unknown; 81 | declare global { 82 | namespace JSX { 83 | interface IntrinsicElements { 84 | div: Record; 85 | button: Record; 86 | h1: Record; 87 | p: Record; 88 | ul: Record; 89 | li: Record; 90 | span: Record; 91 | em: Record; 92 | } 93 | } 94 | } 95 | function h( 96 | type: string, 97 | props: Record | null, 98 | ...children: JSXChild[] 99 | ): JSXElement { 100 | return { 101 | type, 102 | props: { ...(props ?? {}), children }, 103 | }; 104 | } 105 | type TodoItem = { 106 | id: number; 107 | title: string; 108 | completed: boolean; 109 | }; 110 | const SAMPLE_TODOS: TodoItem[] = [ 111 | { id: 1, title: 'Write documentation', completed: true }, 112 | { id: 2, title: 'Ship new feature', completed: false }, 113 | { id: 3, title: 'Polish UI', completed: false }, 114 | ]; 115 | export type TodoDashboardProps = { 116 | filterCompleted?: boolean | null; 117 | }; 118 | export const TodoDashboard = ({ 119 | filterCompleted = null, 120 | }: TodoDashboardProps): JSXElement => { 121 | const todos = filterTodos(SAMPLE_TODOS, filterCompleted); 122 | const completedCount = SAMPLE_TODOS.filter((todo) => todo.completed).length; 123 | return ( 124 |
125 |

Todo Dashboard Demo

126 |

Completed tasks: {completedCount}

127 | {renderFilters(filterCompleted)} 128 | {todos.length === 0 ? renderEmptyState() : renderTodoList(todos)} 129 |
130 | ); 131 | }; 132 | export const renderFilters = (filterCompleted: boolean | null): JSXElement => ( 133 |
134 | 137 | 140 | 143 |
144 | ); 145 | export const renderEmptyState = (): JSXElement => No todos to display.; 146 | export const renderTodoList = (todos: TodoItem[]): JSXElement => ( 147 |
    148 | {todos.map((todo) => ( 149 |
  • 150 | {todo.title} 151 | {todo.completed ? '✅' : '🕒'} 152 |
    • 153 | ))} 154 |
155 | ); 156 | export const filterTodos = ( 157 | todos: TodoItem[], 158 | filterCompleted: boolean | null 159 | ): TodoItem[] => { 160 | if (filterCompleted === null) { 161 | return todos; 162 | } 163 | return todos.filter((todo) => todo.completed === filterCompleted); 164 | }; 165 | const indent = (depth: number): string => { 166 | if (depth <= 0) { 167 | return ''; 168 | } 169 | let value = ''; 170 | for (let index = 0; index < depth; index += 1) { 171 | value += ' '; 172 | } 173 | return value; 174 | }; 175 | const trimRight = (value: string): string => value.replace(/[\s\n]+$/, ''); 176 | export const renderToString = (element: JSXChild, depth = 0): string => { 177 | if (element === null || element === undefined) { 178 | return ''; 179 | } 180 | if ( 181 | typeof element === 'string' || 182 | typeof element === 'number' || 183 | typeof element === 'boolean' 184 | ) { 185 | return `${indent(depth)}${String(element)}\n`; 186 | } 187 | const { type, props } = element; 188 | const attrEntries: string[] = []; 189 | for (const key in props) { 190 | if ( 191 | !Object.prototype.hasOwnProperty.call(props, key) || 192 | key === 'children' 193 | ) { 194 | continue; 195 | } 196 | const value = props[key]; 197 | attrEntries.push(`${key}="${value}"`); 198 | } 199 | const attributes = attrEntries.length > 0 ? ` ${attrEntries.join(' ')}` : ''; 200 | const children = props.children ?? []; 201 | const opening = `${indent(depth)}<${type}${attributes}>\n`; 202 | const childMarkup = children 203 | .map((child) => renderToString(child, depth + 1)) 204 | .join(''); 205 | const closing = `${indent(depth)}\n`; 206 | return opening + childMarkup + closing; 207 | }; 208 | export function runTodoDashboardDemo(): void { 209 | const views = [null, true, false].map((filter) => ({ 210 | filter, 211 | markup: renderToString(), 212 | })); 213 | for (const { filter, markup } of views) { 214 | const label = filter === null ? 'all' : filter ? 'completed' : 'pending'; 215 | console.log(`\n--- rendering ${label} view ---`); 216 | console.log(trimRight(markup)); 217 | } 218 | } 219 | if (typeof require !== 'undefined' && typeof module !== 'undefined') { 220 | const nodeRequire = require as any; 221 | const nodeModule = module as any; 222 | if (nodeRequire.main === nodeModule) { 223 | runTodoDashboardDemo(); 224 | } 225 | } 226 | ``` 227 | 228 | --- 229 | 230 | ## `src/test3.ts` 231 | 232 | ```ts 233 | type Metric = { 234 | name: string; 235 | values: number[]; 236 | }; 237 | type MetricSummary = { 238 | name: string; 239 | count: number; 240 | min: number; 241 | max: number; 242 | average: number; 243 | standardDeviation: number; 244 | }; 245 | export class MetricAnalyzer { 246 | private readonly metrics: Metric[] = []; 247 | public addMetric(metric: Metric): void { 248 | let existingIndex = -1; 249 | for (let index = 0; index < this.metrics.length; index += 1) { 250 | if (this.metrics[index].name === metric.name) { 251 | existingIndex = index; 252 | break; 253 | } 254 | } 255 | if (existingIndex >= 0) { 256 | this.metrics[existingIndex] = { 257 | name: metric.name, 258 | values: this.metrics[existingIndex].values.concat(metric.values), 259 | }; 260 | } else { 261 | this.metrics.push({ 262 | name: metric.name, 263 | values: metric.values.slice(), 264 | }); 265 | } 266 | } 267 | public summarize(): MetricSummary[] { 268 | return this.metrics.map((metric) => summarizeMetric(metric)); 269 | } 270 | public clear(): void { 271 | this.metrics.length = 0; 272 | } 273 | } 274 | export const summarizeMetric = (metric: Metric): MetricSummary => { 275 | if (metric.values.length === 0) { 276 | return { 277 | name: metric.name, 278 | count: 0, 279 | min: 0, 280 | max: 0, 281 | average: 0, 282 | standardDeviation: 0, 283 | }; 284 | } 285 | let sum = 0; 286 | let min = metric.values[0]; 287 | let max = metric.values[0]; 288 | for (let index = 0; index < metric.values.length; index += 1) { 289 | const value = metric.values[index]; 290 | sum += value; 291 | if (value < min) { 292 | min = value; 293 | } 294 | if (value > max) { 295 | max = value; 296 | } 297 | } 298 | const count = metric.values.length; 299 | const average = sum / count; 300 | let squaredErrorSum = 0; 301 | for (let index = 0; index < count; index += 1) { 302 | const error = metric.values[index] - average; 303 | squaredErrorSum += error * error; 304 | } 305 | const variance = squaredErrorSum / count; 306 | const standardDeviation = Math.sqrt(variance); 307 | return { 308 | name: metric.name, 309 | count, 310 | min, 311 | max, 312 | average, 313 | standardDeviation, 314 | }; 315 | }; 316 | export const describeSummaries = (summaries: MetricSummary[]): string => { 317 | if (summaries.length === 0) { 318 | return 'No metrics recorded.'; 319 | } 320 | const lines: string[] = []; 321 | for (let index = 0; index < summaries.length; index += 1) { 322 | const summary = summaries[index]; 323 | lines.push( 324 | [ 325 | `Metric: ${summary.name}`, 326 | `Count: ${summary.count}`, 327 | `Range: ${summary.min} - ${summary.max}`, 328 | `Average: ${summary.average.toFixed(2)}`, 329 | `Std Dev: ${summary.standardDeviation.toFixed(2)}`, 330 | ].join(' | ') 331 | ); 332 | } 333 | return lines.join('\n'); 334 | }; 335 | export const generateSampleMetrics = (): Metric[] => [ 336 | { name: 'response_time_ms', values: [120, 135, 150, 90, 105, 130] }, 337 | { name: 'memory_usage_mb', values: [256, 240, 232, 280, 300] }, 338 | { name: 'cpu_percent', values: [32, 44, 27, 55, 38, 41, 36] }, 339 | ]; 340 | declare const require: unknown; 341 | declare const module: unknown; 342 | export function runMetricAnalyzerDemo(): void { 343 | const analyzer = new MetricAnalyzer(); 344 | const metrics = generateSampleMetrics(); 345 | for (let index = 0; index < metrics.length; index += 1) { 346 | analyzer.addMetric(metrics[index]); 347 | } 348 | const summaries = analyzer.summarize(); 349 | const report = describeSummaries(summaries); 350 | console.log('Metric Analyzer Report'); 351 | console.log('======================'); 352 | console.log(report); 353 | } 354 | if (typeof require !== 'undefined' && typeof module !== 'undefined') { 355 | const nodeRequire = require as any; 356 | const nodeModule = module as any; 357 | if (nodeRequire.main === nodeModule) { 358 | runMetricAnalyzerDemo(); 359 | } 360 | } 361 | ``` 362 | 363 | --- 364 | -------------------------------------------------------------------------------- /.context/generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # generate.py 3 | """AI Context Generator - A structured tool to package project files for LLMs. 4 | 5 | This script collects text files from a project, applies smart filtering, and 6 | generates a single context file for use with Large Language Models. 7 | 8 | Directory Structure: 9 | .context/ # Main context generation directory 10 | ├── generate.py # This script 11 | ├── .ai-context.yml # Configuration file (optional) 12 | └── generated/ # Output directory for generated files 13 | └── context.txt # Generated context file 14 | 15 | Core Principles: 16 | - Structured Organization: All context-related files in .context/ directory 17 | - Convention over Configuration: Smart defaults that work for most projects 18 | - Simplicity: Easy to use with minimal and intuitive CLI 19 | - Performance: Fast, streaming processing with low memory usage 20 | - Universal: Works for any programming language or project type 21 | 22 | Usage Examples: 23 | # Basic usage (auto-detects project type, outputs to .context/generated/context.txt) 24 | python .context/generate.py 25 | 26 | # Specify a directory to scan 27 | python .context/generate.py ../src/ 28 | 29 | # Use a preset and enable minification 30 | python .context/generate.py --preset python --minify 31 | 32 | # Custom output name (still in .context/generated/) 33 | python .context/generate.py --output my_context.md 34 | 35 | # Provide custom patterns (added to preset) 36 | python .context/generate.py --preset python --include "*.toml" --exclude "tests/*" 37 | """ 38 | 39 | from __future__ import annotations 40 | 41 | import argparse 42 | import fnmatch 43 | import io 44 | import json 45 | import logging 46 | import os 47 | import re 48 | import sys 49 | import tempfile 50 | import time 51 | import tokenize 52 | from dataclasses import dataclass, field 53 | from enum import Enum 54 | from pathlib import Path 55 | from typing import Dict, Iterable, List, Optional, Set 56 | 57 | # Optional libs with graceful fallback 58 | try: 59 | import chardet 60 | 61 | HAS_CHARDET = True 62 | except ImportError: 63 | HAS_CHARDET = False 64 | 65 | try: 66 | import yaml 67 | 68 | HAS_YAML = True 69 | except ImportError: 70 | HAS_YAML = False 71 | 72 | try: 73 | from tqdm import tqdm 74 | 75 | HAS_TQDM = True 76 | except ImportError: 77 | HAS_TQDM = False 78 | 79 | __version__ = "5.0.0" 80 | 81 | # --- Constants and Defaults --- 82 | CONTEXT_DIR_NAME = ".context" 83 | GENERATED_DIR_NAME = "generated" 84 | CONFIG_FILE_NAME = ".ai-context.yml" 85 | DEFAULT_OUTPUT_NAME = "context.txt" 86 | DEFAULT_EXCLUDE_PATTERNS = [ 87 | "*/.*", 88 | "dist", 89 | "build", 90 | "node_modules", 91 | "__pycache__", 92 | f"{CONTEXT_DIR_NAME}/*", # Always exclude the .context directory itself 93 | ] 94 | BINARY_FILE_EXTENSIONS = { 95 | ".png", 96 | ".jpg", 97 | ".jpeg", 98 | ".gif", 99 | ".bmp", 100 | ".ico", 101 | ".svg", 102 | ".webp", 103 | ".zip", 104 | ".tar", 105 | ".gz", 106 | ".bz2", 107 | ".7z", 108 | ".rar", 109 | ".pdf", 110 | ".doc", 111 | ".docx", 112 | ".xls", 113 | ".xlsx", 114 | ".ppt", 115 | ".pptx", 116 | ".exe", 117 | ".dll", 118 | ".so", 119 | ".dylib", 120 | ".o", 121 | ".a", 122 | ".lib", 123 | ".mp3", 124 | ".wav", 125 | ".flac", 126 | ".aac", 127 | ".ogg", 128 | ".wma", 129 | ".mp4", 130 | ".mov", 131 | ".avi", 132 | ".mkv", 133 | ".webm", 134 | ".flv", 135 | ".class", 136 | ".jar", 137 | ".pyc", 138 | ".pyo", 139 | ".pyd", 140 | ".db", 141 | ".sqlite", 142 | ".sqlite3", 143 | ".ttf", 144 | ".otf", 145 | ".woff", 146 | ".woff2", 147 | ".eot", 148 | } 149 | ENCODING_SAMPLE_SIZE = 8192 150 | DEFAULT_MAX_FILE_SIZE_MB = 1 151 | 152 | # --- Logging Setup --- 153 | logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stderr) 154 | logger = logging.getLogger(__name__) 155 | 156 | 157 | # --- Path Management --- 158 | class PathManager: 159 | """Manages paths relative to project structure.""" 160 | 161 | @staticmethod 162 | def get_project_root() -> Path: 163 | """Get the project root (parent of .context directory).""" 164 | # If we're running from within .context, go up one level 165 | current = Path.cwd() 166 | if current.name == CONTEXT_DIR_NAME: 167 | return current.parent 168 | # Otherwise check if .context exists in current directory 169 | if (current / CONTEXT_DIR_NAME).exists(): 170 | return current 171 | # Otherwise assume current directory is project root 172 | return current 173 | 174 | @staticmethod 175 | def get_context_dir() -> Path: 176 | """Get the .context directory path.""" 177 | project_root = PathManager.get_project_root() 178 | return project_root / CONTEXT_DIR_NAME 179 | 180 | @staticmethod 181 | def get_generated_dir() -> Path: 182 | """Get the .context/generated directory path.""" 183 | return PathManager.get_context_dir() / GENERATED_DIR_NAME 184 | 185 | @staticmethod 186 | def get_config_path() -> Path: 187 | """Get the path to .ai-context.yml in .context directory.""" 188 | return PathManager.get_context_dir() / CONFIG_FILE_NAME 189 | 190 | @staticmethod 191 | def ensure_generated_dir() -> Path: 192 | """Ensure the generated directory exists.""" 193 | generated_dir = PathManager.get_generated_dir() 194 | generated_dir.mkdir(parents=True, exist_ok=True) 195 | return generated_dir 196 | 197 | @staticmethod 198 | def resolve_project_path(path: Path) -> Path: 199 | """Resolve a path relative to the project root.""" 200 | project_root = PathManager.get_project_root() 201 | if path.is_absolute(): 202 | return path 203 | return (project_root / path).resolve() 204 | 205 | @staticmethod 206 | def rel_to_project_root(path: Path) -> str: 207 | """Get path relative to project root for display.""" 208 | try: 209 | project_root = PathManager.get_project_root() 210 | return str(path.relative_to(project_root)) 211 | except Exception: 212 | return str(path) 213 | 214 | 215 | # --- Core Data Structures --- 216 | class OutputFormat(Enum): 217 | TEXT = "text" 218 | JSON = "json" 219 | MARKDOWN = "markdown" 220 | 221 | 222 | @dataclass 223 | class Config: 224 | """Configuration with structure-aware defaults.""" 225 | 226 | paths: List[Path] = field(default_factory=lambda: [PathManager.get_project_root()]) 227 | output: Path = field( 228 | default_factory=lambda: PathManager.get_generated_dir() / DEFAULT_OUTPUT_NAME 229 | ) 230 | preset: str = "auto" 231 | include: List[str] = field(default_factory=list) 232 | exclude: List[str] = field(default_factory=list) 233 | exclude_common: bool = True 234 | max_file_size_mb: float = DEFAULT_MAX_FILE_SIZE_MB 235 | output_format: OutputFormat = OutputFormat.TEXT 236 | minify: bool = False 237 | verbose: bool = False 238 | dry_run: bool = False 239 | 240 | @property 241 | def max_file_size_bytes(self) -> int: 242 | if self.max_file_size_mb <= 0: 243 | return float("inf") 244 | return int(self.max_file_size_mb * 1024 * 1024) 245 | 246 | 247 | @dataclass 248 | class ProcessedFile: 249 | """Represents a file that has been processed.""" 250 | 251 | path: Path 252 | content: Optional[str] = None 253 | skipped: bool = False 254 | reason: Optional[str] = None 255 | size: int = 0 256 | encoding: str = "unknown" 257 | 258 | 259 | # --- Project Intelligence --- 260 | class ProjectDetector: 261 | """Auto-detects project type to apply smart configuration presets.""" 262 | 263 | PRESETS = { 264 | "python": { 265 | "include": [ 266 | "*.py", 267 | "requirements*.txt", 268 | "pyproject.toml", 269 | "setup.py", 270 | "*.pyx", 271 | "*.pyi", 272 | ], 273 | "exclude": [ 274 | "*.pyc", 275 | "*.pyo", 276 | "*.pyd", 277 | "*.egg-info/*", 278 | "venv/*", 279 | ".venv/*", 280 | "__pycache__/*", 281 | ], 282 | }, 283 | "javascript": { 284 | "include": [ 285 | "*.js", 286 | "*.jsx", 287 | "*.ts", 288 | "*.tsx", 289 | "*.mjs", 290 | "*.cjs", 291 | "package.json", 292 | "tsconfig.json", 293 | "*.vue", 294 | "*.svelte", 295 | "*.html", 296 | "*.css", 297 | "*.scss", 298 | "*.less", 299 | ], 300 | "exclude": [ 301 | "*.log", 302 | "package-lock.json", 303 | "yarn.lock", 304 | "pnpm-lock.yaml", 305 | "*.map", 306 | ], 307 | }, 308 | "java": { 309 | "include": [ 310 | "*.java", 311 | "*.xml", 312 | "*.properties", 313 | "pom.xml", 314 | "build.gradle*", 315 | "*.kt", 316 | ], 317 | "exclude": ["*.class", "*.jar", "target/*", "build/*"], 318 | }, 319 | "csharp": { 320 | "include": ["*.cs", "*.csproj", "*.sln", "*.xaml", "*.config", "*.resx"], 321 | "exclude": ["bin/*", "obj/*", "*.dll", "*.exe"], 322 | }, 323 | "ruby": { 324 | "include": ["*.rb", "*.erb", "*.rake", "Gemfile", "Rakefile", "*.ru"], 325 | "exclude": ["Gemfile.lock", "vendor/*"], 326 | }, 327 | "go": { 328 | "include": ["*.go", "go.mod", "go.sum", "*.proto"], 329 | "exclude": ["vendor/*"], 330 | }, 331 | "rust": { 332 | "include": ["*.rs", "Cargo.toml", "Cargo.lock"], 333 | "exclude": ["target/*"], 334 | }, 335 | "php": { 336 | "include": ["*.php", "composer.json", "*.blade.php"], 337 | "exclude": ["vendor/*", "composer.lock"], 338 | }, 339 | "cpp": { 340 | "include": [ 341 | "*.cpp", 342 | "*.cc", 343 | "*.cxx", 344 | "*.h", 345 | "*.hpp", 346 | "*.hxx", 347 | "CMakeLists.txt", 348 | "*.cmake", 349 | ], 350 | "exclude": ["*.o", "*.obj", "build/*", "cmake-build-*/*"], 351 | }, 352 | "swift": { 353 | "include": ["*.swift", "Package.swift", "*.xcodeproj/*", "*.xcworkspace/*"], 354 | "exclude": ["*.xcuserdata/*", "build/*", "DerivedData/*"], 355 | }, 356 | } 357 | 358 | INDICATORS = { 359 | "python": [ 360 | "requirements.txt", 361 | "requirements-dev.txt", 362 | "pyproject.toml", 363 | "setup.py", 364 | "Pipfile", 365 | ], 366 | "javascript": ["package.json", "node_modules"], 367 | "java": ["pom.xml", "build.gradle", "build.gradle.kts"], 368 | "csharp": ["*.csproj", "*.sln"], 369 | "ruby": ["Gemfile", "Rakefile"], 370 | "go": ["go.mod"], 371 | "rust": ["Cargo.toml"], 372 | "php": ["composer.json"], 373 | "cpp": ["CMakeLists.txt", "Makefile"], 374 | "swift": ["Package.swift", "*.xcodeproj"], 375 | } 376 | 377 | @classmethod 378 | def detect(cls, path: Path) -> str: 379 | """Detects the project type by looking for key indicator files.""" 380 | for name, indicators in cls.INDICATORS.items(): 381 | for indicator in indicators: 382 | if "*" in indicator: 383 | # Handle glob patterns 384 | if list(path.glob(indicator)): 385 | logger.info(f"✓ Detected {name.capitalize()} project.") 386 | return name 387 | else: 388 | if (path / indicator).exists(): 389 | logger.info(f"✓ Detected {name.capitalize()} project.") 390 | return name 391 | 392 | logger.info("✓ Could not auto-detect project type, using generic settings.") 393 | return "generic" 394 | 395 | @classmethod 396 | def get_preset_config(cls, name: str) -> Dict: 397 | """Returns the configuration for a given preset name.""" 398 | return cls.PRESETS.get(name, {}) 399 | 400 | 401 | # --- File Filtering and Collection --- 402 | class SmartFilter: 403 | """Intelligent file filtering with caching and fast-path excludes.""" 404 | 405 | def __init__(self, config: Config): 406 | self.exclude_common = config.exclude_common 407 | self.include_patterns = [self._compile_glob(p) for p in config.include] 408 | self.exclude_patterns = [self._compile_glob(p) for p in config.exclude] 409 | self.base_exclude = [self._compile_glob(p) for p in DEFAULT_EXCLUDE_PATTERNS] 410 | self._cache: Dict[Path, bool] = {} 411 | 412 | @staticmethod 413 | def _compile_glob(pattern: str) -> re.Pattern: 414 | """Compiles a glob pattern into a regex.""" 415 | return re.compile(fnmatch.translate(pattern)) 416 | 417 | def _matches(self, path_str: str, patterns: List[re.Pattern]) -> bool: 418 | """Checks if a string matches any of the compiled patterns.""" 419 | return any(p.match(path_str) for p in patterns) 420 | 421 | def should_process(self, path: Path) -> bool: 422 | """Determines if a file or directory should be processed.""" 423 | 424 | key = path.resolve() 425 | if key in self._cache: 426 | return self._cache[key] 427 | 428 | # Always exclude .context directory 429 | try: 430 | rel_path = PathManager.rel_to_project_root(path) 431 | if rel_path.startswith(CONTEXT_DIR_NAME): 432 | self._cache[key] = False 433 | return False 434 | except Exception: 435 | pass 436 | 437 | # Use posix path for consistent pattern matching 438 | path_str = str(path.as_posix()) 439 | 440 | # Fast path for common binary extensions 441 | if path.suffix.lower() in BINARY_FILE_EXTENSIONS: 442 | self._cache[key] = False 443 | return False 444 | 445 | # Fast path for common excluded directories 446 | if self.exclude_common: 447 | if path.name.startswith(".") or self._matches(path_str, self.base_exclude): 448 | self._cache[key] = False 449 | return False 450 | 451 | # Custom exclude patterns 452 | if self._matches(path_str, self.exclude_patterns): 453 | self._cache[key] = False 454 | return False 455 | 456 | # If include patterns are specified, a file must match at least one 457 | if ( 458 | self.include_patterns 459 | and path.is_file() 460 | and not self._matches(path.name, self.include_patterns) 461 | ): 462 | self._cache[key] = False 463 | return False 464 | 465 | self._cache[key] = True 466 | return True 467 | 468 | 469 | class FileCollector: 470 | """Walks directories and collects files based on filter criteria.""" 471 | 472 | def __init__(self, config: Config, file_filter: SmartFilter): 473 | self.config = config 474 | self.filter = file_filter 475 | 476 | def collect(self) -> Iterable[Path]: 477 | """Collects all files that should be processed.""" 478 | seen_paths: Set[Path] = set() 479 | 480 | for start_path in self.config.paths: 481 | # Resolve paths relative to project root 482 | start_path = PathManager.resolve_project_path(start_path) 483 | 484 | if not start_path.exists(): 485 | logger.warning( 486 | f"Warning: Path does not exist and will be skipped: {start_path}" 487 | ) 488 | continue 489 | 490 | if start_path.is_file(): 491 | if self.filter.should_process(start_path): 492 | resolved_path = start_path.resolve() 493 | if resolved_path not in seen_paths: 494 | seen_paths.add(resolved_path) 495 | yield resolved_path 496 | else: 497 | for root, dirs, files in os.walk( 498 | start_path, topdown=True, followlinks=False 499 | ): 500 | root_path = Path(root) 501 | # Filter directories in-place 502 | dirs[:] = [ 503 | d for d in dirs if self.filter.should_process(root_path / d) 504 | ] 505 | 506 | for name in files: 507 | file_path = root_path / name 508 | if self.filter.should_process(file_path): 509 | resolved_path = file_path.resolve() 510 | if resolved_path not in seen_paths: 511 | seen_paths.add(resolved_path) 512 | yield resolved_path 513 | 514 | 515 | # --- File Processing and Minification --- 516 | class Minifier: 517 | """Language-aware minification logic.""" 518 | 519 | @staticmethod 520 | def minify(content: str, path: Path) -> str: 521 | """Removes comments and extraneous whitespace based on file type.""" 522 | ext = path.suffix.lower() 523 | if ext in {".py"}: 524 | return Minifier._minify_python(content) 525 | elif ext in {".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs"}: 526 | return Minifier._minify_javascript(content) 527 | elif ext in {".css", ".scss", ".less"}: 528 | return Minifier._minify_css(content) 529 | return Minifier._minify_generic(content) 530 | 531 | @staticmethod 532 | def _minify_generic(content: str) -> str: 533 | """A simple generic minifier for comments and empty lines.""" 534 | lines = content.splitlines() 535 | res = [] 536 | 537 | for i, line in enumerate(lines): 538 | if i == 0 and line.startswith("#!"): 539 | res.append(line) 540 | continue 541 | 542 | # Remove single line comments (naive but safer) 543 | newline = re.sub(r"//.*$", "", line) 544 | newline = re.sub(r"#(?!!).*$", "", newline) 545 | 546 | if newline.strip(): 547 | res.append(newline.rstrip()) 548 | 549 | # Simple block comment removal 550 | result = "\n".join(res) 551 | result = re.sub(r"/\*.*?\*/", "", result, flags=re.DOTALL) 552 | 553 | return result 554 | 555 | @staticmethod 556 | def _minify_python(content: str) -> str: 557 | """Safely removes only comments from Python code using the tokenize module.""" 558 | try: 559 | tokens = tokenize.generate_tokens(io.StringIO(content).readline) 560 | return tokenize.untokenize( 561 | tok for tok in tokens if tok.type != tokenize.COMMENT 562 | ) 563 | except (tokenize.TokenError, IndentationError): 564 | return Minifier._minify_generic(content) 565 | 566 | @staticmethod 567 | def _minify_javascript(content: str) -> str: 568 | """Minify JavaScript/TypeScript code.""" 569 | # Remove single-line comments 570 | content = re.sub(r"//.*$", "", content, flags=re.MULTILINE) 571 | # Remove multi-line comments 572 | content = re.sub(r"/\*.*?\*/", "", content, flags=re.DOTALL) 573 | # Remove empty lines 574 | lines = [line.rstrip() for line in content.splitlines() if line.strip()] 575 | return "\n".join(lines) 576 | 577 | @staticmethod 578 | def _minify_css(content: str) -> str: 579 | """Minify CSS code.""" 580 | # Remove comments 581 | content = re.sub(r"/\*.*?\*/", "", content, flags=re.DOTALL) 582 | # Remove unnecessary whitespace 583 | content = re.sub(r"\s+", " ", content) 584 | content = re.sub(r"\s*([{}:;,])\s*", r"\1", content) 585 | return content.strip() 586 | 587 | 588 | class FileProcessor: 589 | """Handles reading, decoding, and processing a single file.""" 590 | 591 | def __init__(self, config: Config): 592 | self.config = config 593 | 594 | def process_file(self, path: Path) -> ProcessedFile: 595 | """Processes a single file.""" 596 | try: 597 | size = path.stat().st_size 598 | if ( 599 | self.config.max_file_size_bytes != float("inf") 600 | and size > self.config.max_file_size_bytes 601 | ): 602 | return ProcessedFile( 603 | path, 604 | skipped=True, 605 | reason=f"Exceeds max size ({size} > {self.config.max_file_size_bytes})", 606 | size=size, 607 | ) 608 | 609 | with path.open("rb") as f: 610 | # Read a small chunk to detect binary content and encoding 611 | chunk = f.read(ENCODING_SAMPLE_SIZE) 612 | if b"\x00" in chunk: 613 | return ProcessedFile( 614 | path, skipped=True, reason="Binary file detected", size=size 615 | ) 616 | 617 | encoding = self._detect_encoding(chunk) 618 | 619 | # Read the full content 620 | f.seek(0) 621 | try: 622 | content = f.read().decode(encoding, errors="replace") 623 | except Exception as e: 624 | return ProcessedFile( 625 | path, 626 | skipped=True, 627 | reason=f"Could not decode with {encoding}: {e}", 628 | size=size, 629 | ) 630 | 631 | # Apply minification if requested 632 | if self.config.minify: 633 | content = Minifier.minify(content, path) 634 | 635 | if not content.strip(): 636 | reason = ( 637 | "Empty after minification" 638 | if self.config.minify 639 | else "Empty or whitespace-only file" 640 | ) 641 | return ProcessedFile(path, skipped=True, reason=reason, size=size) 642 | 643 | return ProcessedFile(path, content=content, size=size, encoding=encoding) 644 | 645 | except Exception as e: 646 | return ProcessedFile(path, skipped=True, reason=f"Error reading file: {e}") 647 | 648 | @staticmethod 649 | def _detect_encoding(chunk: bytes) -> str: 650 | """Detects file encoding from a sample chunk.""" 651 | # First try UTF-8 652 | try: 653 | chunk.decode("utf-8") 654 | return "utf-8" 655 | except UnicodeDecodeError: 656 | pass 657 | 658 | # Then try chardet if available 659 | if HAS_CHARDET: 660 | result = chardet.detect(chunk) 661 | if result.get("encoding") and result["confidence"] > 0.7: 662 | return result["encoding"] 663 | 664 | # Fallback to latin-1 665 | return "latin-1" 666 | 667 | 668 | # --- Output Formatting and Writing --- 669 | class Formatter: 670 | """Base class for output formatters.""" 671 | 672 | def __init__(self, config: Config): 673 | self.config = config 674 | 675 | def write_header(self, file_stream, file_count: int): 676 | pass 677 | 678 | def write_file(self, file_stream, processed_file: ProcessedFile): 679 | raise NotImplementedError 680 | 681 | def write_footer(self, file_stream): 682 | pass 683 | 684 | 685 | class TextFormatter(Formatter): 686 | """Formats output as a simple, human-readable text file.""" 687 | 688 | def write_header(self, file_stream, file_count: int): 689 | project_root = PathManager.get_project_root() 690 | header = ( 691 | f"AI Context Report (v{__version__})\n" 692 | f"{'=' * 50}\n" 693 | f"Project Root: {project_root.resolve()}\n" 694 | f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}\n" 695 | f"Files discovered: {file_count}\n" 696 | f"{'=' * 50}\n\n" 697 | ) 698 | file_stream.write(header) 699 | 700 | def write_file(self, file_stream, processed_file: ProcessedFile): 701 | rel_path = PathManager.rel_to_project_root(processed_file.path) 702 | separator = f"\n\n{'=' * 20} File: {rel_path} {'=' * 20}\n\n" 703 | file_stream.write(separator) 704 | file_stream.write(processed_file.content) 705 | 706 | 707 | class MarkdownFormatter(Formatter): 708 | """Formats output as a structured Markdown file.""" 709 | 710 | def write_header(self, file_stream, file_count: int): 711 | project_root = PathManager.get_project_root() 712 | header = ( 713 | f"# AI Context Report\n\n" 714 | f"- **Version**: `{__version__}`\n" 715 | f"- **Project Root**: `{project_root.resolve()}`\n" 716 | f"- **Generated**: `{time.strftime('%Y-%m-%d %H:%M:%S')}`\n" 717 | f"- **Files discovered**: `{file_count}`\n\n" 718 | f"---\n\n" 719 | ) 720 | file_stream.write(header) 721 | 722 | def write_file(self, file_stream, processed_file: ProcessedFile): 723 | rel_path = PathManager.rel_to_project_root(processed_file.path) 724 | ext = processed_file.path.suffix.lstrip(".").lower() or "text" 725 | 726 | file_header = f"## `{rel_path}`\n\n" 727 | code_block = f"```{ext}\n{processed_file.content}\n```\n\n---\n\n" 728 | file_stream.write(file_header) 729 | file_stream.write(code_block) 730 | 731 | 732 | class JsonFormatter(Formatter): 733 | """Formats output as a JSON object.""" 734 | 735 | def __init__(self, config: Config): 736 | super().__init__(config) 737 | self._is_first_file = True 738 | 739 | def write_header(self, file_stream, file_count: int): 740 | project_root = PathManager.get_project_root() 741 | file_stream.write("{\n") 742 | file_stream.write(' "metadata": {\n') 743 | file_stream.write(f' "version": "{__version__}",\n') 744 | project_root_str = json.dumps(str(project_root.resolve())) 745 | file_stream.write(f' "project_root": {project_root_str},\n') 746 | file_stream.write( 747 | f' "generated_at": "{time.strftime("%Y-%m-%d %H:%M:%S")}",\n' 748 | ) 749 | file_stream.write(f' "file_count": {file_count}\n') 750 | file_stream.write(" },\n") 751 | file_stream.write(' "files": [\n') 752 | 753 | def write_file(self, file_stream, processed_file: ProcessedFile): 754 | rel_path = PathManager.rel_to_project_root(processed_file.path) 755 | file_data = { 756 | "path": rel_path, 757 | "size": processed_file.size, 758 | "encoding": processed_file.encoding, 759 | "content": processed_file.content, 760 | } 761 | if not self._is_first_file: 762 | file_stream.write(",\n") 763 | 764 | json_string = json.dumps(file_data, indent=4) 765 | indented_json = " " + json_string.replace("\n", "\n ") 766 | file_stream.write(indented_json) 767 | self._is_first_file = False 768 | 769 | def write_footer(self, file_stream): 770 | file_stream.write("\n ]\n}\n") 771 | 772 | 773 | class StreamingWriter: 774 | """Writes processed files to the output stream.""" 775 | 776 | def __init__(self, config: Config): 777 | self.config = config 778 | self.formatter = self._get_formatter() 779 | 780 | def _get_formatter(self) -> Formatter: 781 | if self.config.output_format == OutputFormat.MARKDOWN: 782 | return MarkdownFormatter(self.config) 783 | if self.config.output_format == OutputFormat.JSON: 784 | return JsonFormatter(self.config) 785 | return TextFormatter(self.config) 786 | 787 | def write(self, files: Iterable[Path]): 788 | """Processes and writes files in a streaming fashion.""" 789 | processor = FileProcessor(self.config) 790 | 791 | # Collect and sort paths 792 | file_paths = sorted(list(files)) 793 | 794 | # Exclude the output file itself 795 | try: 796 | out_resolved = self.config.output.resolve() 797 | file_paths = [p for p in file_paths if p.resolve() != out_resolved] 798 | except (OSError, RuntimeError, ValueError): 799 | pass 800 | 801 | total_files = len(file_paths) 802 | 803 | if self.config.dry_run: 804 | logger.info( 805 | f"DRY RUN: Would process {total_files} files and write to {self.config.output}" 806 | ) 807 | for path in file_paths: 808 | logger.info(f" - {PathManager.rel_to_project_root(path)}") 809 | return 810 | 811 | # Ensure output directory exists 812 | PathManager.ensure_generated_dir() 813 | 814 | processed_count = 0 815 | skipped_count = 0 816 | 817 | progress_bar = None 818 | if HAS_TQDM and sys.stderr.isatty(): 819 | progress_bar = tqdm(total=total_files, desc="Processing files", unit="file") 820 | 821 | try: 822 | out_suffix = self.config.output.suffix or ".txt" 823 | with tempfile.NamedTemporaryFile( 824 | "w", 825 | delete=False, 826 | encoding="utf-8", 827 | dir=self.config.output.parent, 828 | prefix=".tmp_", 829 | suffix=out_suffix, 830 | ) as tmp: 831 | tmp_path = Path(tmp.name) 832 | 833 | self.formatter.write_header(tmp, total_files) 834 | 835 | for path in file_paths: 836 | if progress_bar: 837 | progress_bar.update(1) 838 | 839 | result = processor.process_file(path) 840 | 841 | if result.skipped: 842 | skipped_count += 1 843 | if self.config.verbose: 844 | logger.info( 845 | f"Skipped: {PathManager.rel_to_project_root(path)} (Reason: {result.reason})" 846 | ) 847 | continue 848 | 849 | self.formatter.write_file(tmp, result) 850 | processed_count += 1 851 | 852 | self.formatter.write_footer(tmp) 853 | 854 | # Atomically move temp file to final destination 855 | os.replace(tmp_path, self.config.output) 856 | 857 | except Exception as e: 858 | if "tmp_path" in locals() and tmp_path.exists(): 859 | try: 860 | tmp_path.unlink() 861 | except OSError: 862 | pass 863 | raise e 864 | finally: 865 | if progress_bar: 866 | progress_bar.close() 867 | 868 | logger.info("✓ Generation complete.") 869 | logger.info(f" - Output file: {self.config.output.resolve()}") 870 | logger.info(f" - Files processed: {processed_count}") 871 | logger.info(f" - Files skipped: {skipped_count}") 872 | 873 | 874 | # --- Main Orchestrator --- 875 | class Generator: 876 | """Orchestrates the context generation process.""" 877 | 878 | def __init__(self, config: Config): 879 | self.config = self._resolve_config(config) 880 | 881 | def _resolve_config(self, cli_config: Config) -> Config: 882 | """Merges CLI config, file config, and presets.""" 883 | file_conf = self._load_config_from_file() 884 | 885 | # Priority: CLI > File > Preset > Default 886 | final_config = Config() 887 | 888 | # Apply preset 889 | preset_name = cli_config.preset or file_conf.get("preset", "auto") 890 | if preset_name == "auto": 891 | project_root = PathManager.get_project_root() 892 | preset_name = ProjectDetector.detect(project_root) 893 | 894 | preset_conf = ProjectDetector.get_preset_config(preset_name) 895 | 896 | # Combine includes and excludes 897 | includes = preset_conf.get("include", []) 898 | includes.extend(file_conf.get("include", [])) 899 | includes.extend(cli_config.include) 900 | 901 | excludes = preset_conf.get("exclude", []) 902 | excludes.extend(file_conf.get("exclude", [])) 903 | excludes.extend(cli_config.exclude) 904 | 905 | final_config.include = list(dict.fromkeys(includes)) 906 | final_config.exclude = list(dict.fromkeys(excludes)) 907 | 908 | # Handle paths - resolve relative to project root 909 | if cli_config.paths != [PathManager.get_project_root()]: 910 | final_config.paths = cli_config.paths 911 | else: 912 | config_paths = file_conf.get("paths", ["."]) 913 | final_config.paths = [Path(p) for p in config_paths] 914 | 915 | # Handle output 916 | default_output = PathManager.get_generated_dir() / DEFAULT_OUTPUT_NAME 917 | if cli_config.output != default_output: 918 | # CLI specified output 919 | if cli_config.output.parent == Path("."): 920 | # Just a filename, put it in generated dir 921 | final_config.output = ( 922 | PathManager.get_generated_dir() / cli_config.output.name 923 | ) 924 | else: 925 | final_config.output = cli_config.output 926 | else: 927 | # Use file config or default 928 | config_output = file_conf.get("output", DEFAULT_OUTPUT_NAME) 929 | if Path(config_output).parent == Path("."): 930 | final_config.output = PathManager.get_generated_dir() / config_output 931 | else: 932 | final_config.output = Path(config_output) 933 | 934 | # Other settings 935 | final_config.max_file_size_mb = ( 936 | cli_config.max_file_size_mb 937 | if cli_config.max_file_size_mb != DEFAULT_MAX_FILE_SIZE_MB 938 | else file_conf.get("max_file_size_mb", DEFAULT_MAX_FILE_SIZE_MB) 939 | ) 940 | final_config.output_format = ( 941 | cli_config.output_format 942 | if cli_config.output_format != OutputFormat.TEXT 943 | else OutputFormat(file_conf.get("format", OutputFormat.TEXT.value)) 944 | ) 945 | final_config.minify = cli_config.minify or file_conf.get("minify", False) 946 | final_config.verbose = cli_config.verbose or file_conf.get("verbose", False) 947 | final_config.dry_run = cli_config.dry_run 948 | 949 | return final_config 950 | 951 | def _load_config_from_file(self) -> Dict: 952 | """Loads configuration from .context/.ai-context.yml if it exists.""" 953 | config_path = PathManager.get_config_path() 954 | 955 | if config_path.exists(): 956 | if not HAS_YAML: 957 | logger.warning( 958 | f"Warning: Config file {config_path} exists but PyYAML is not installed. " 959 | "Install it with: pip install pyyaml" 960 | ) 961 | return {} 962 | try: 963 | with config_path.open("r", encoding="utf-8") as f: 964 | data = yaml.safe_load(f) 965 | if data and isinstance(data, dict): 966 | logger.info( 967 | f"✓ Loaded configuration from {PathManager.rel_to_project_root(config_path)}" 968 | ) 969 | return data 970 | return {} 971 | except Exception as e: 972 | logger.warning( 973 | f"Warning: Could not read config file {config_path}: {e}" 974 | ) 975 | return {} 976 | 977 | def run(self): 978 | """Executes the file collection, processing, and writing pipeline.""" 979 | start_time = time.monotonic() 980 | 981 | if self.config.verbose: 982 | logger.setLevel(logging.DEBUG) 983 | logger.info("Verbose mode enabled.") 984 | logger.debug(f"Resolved config: {self.config}") 985 | 986 | logger.info(f"Starting AI Context Generator (v{__version__})...") 987 | logger.info(f"Project root: {PathManager.get_project_root()}") 988 | 989 | if self.config.minify: 990 | logger.info( 991 | "Note: Minification is experimental and may not work perfectly for all languages." 992 | ) 993 | 994 | smart_filter = SmartFilter(self.config) 995 | collector = FileCollector(self.config, smart_filter) 996 | files_to_process = collector.collect() 997 | 998 | writer = StreamingWriter(self.config) 999 | writer.write(files_to_process) 1000 | 1001 | duration = time.monotonic() - start_time 1002 | logger.info(f"Finished in {duration:.2f} seconds.") 1003 | 1004 | 1005 | # --- Command-Line Interface --- 1006 | def create_arg_parser() -> argparse.ArgumentParser: 1007 | parser = argparse.ArgumentParser( 1008 | description="AI Context Generator: A structured tool to package project files for LLMs.", 1009 | formatter_class=argparse.RawTextHelpFormatter, 1010 | epilog=""" 1011 | Directory Structure: 1012 | .context/ # Main directory for context generation 1013 | ├── generate.py # This script 1014 | ├── .ai-context.yml # Configuration file (optional) 1015 | └── generated/ # Output directory 1016 | └── context.txt # Generated context file 1017 | 1018 | Examples: 1019 | python .context/generate.py # Auto-detect and generate 1020 | python .context/generate.py ../src/ # Scan specific directory 1021 | python .context/generate.py --preset python # Use Python preset 1022 | python .context/generate.py --output api.md # Custom output name 1023 | """, 1024 | ) 1025 | 1026 | parser.add_argument( 1027 | "paths", 1028 | nargs="*", 1029 | help="Directories or files to include (default: project root)", 1030 | ) 1031 | parser.add_argument( 1032 | "-o", 1033 | "--output", 1034 | help=f"Output file name (default: {DEFAULT_OUTPUT_NAME} in .context/generated/)", 1035 | ) 1036 | parser.add_argument( 1037 | "--preset", 1038 | choices=["auto"] + list(ProjectDetector.PRESETS.keys()), 1039 | default=None, 1040 | help="Use a preset configuration for a specific project type", 1041 | ) 1042 | parser.add_argument( 1043 | "--include", 1044 | nargs="+", 1045 | help="Glob patterns for files to include (adds to presets)", 1046 | ) 1047 | parser.add_argument( 1048 | "--exclude", 1049 | nargs="+", 1050 | help="Glob patterns for files/directories to exclude (adds to presets)", 1051 | ) 1052 | parser.add_argument( 1053 | "--max-file-size-mb", 1054 | type=float, 1055 | default=DEFAULT_MAX_FILE_SIZE_MB, 1056 | help=f"Maximum size for a single file in MB (default: {DEFAULT_MAX_FILE_SIZE_MB}MB)", 1057 | ) 1058 | parser.add_argument( 1059 | "--format", 1060 | choices=[f.value for f in OutputFormat], 1061 | default=None, 1062 | help="The output format (default: text)", 1063 | ) 1064 | parser.add_argument( 1065 | "--minify", 1066 | action="store_true", 1067 | help="[EXPERIMENTAL] Remove comments and whitespace from code", 1068 | ) 1069 | parser.add_argument( 1070 | "--dry-run", 1071 | action="store_true", 1072 | help="List files that would be processed without creating output", 1073 | ) 1074 | parser.add_argument( 1075 | "-v", 1076 | "--verbose", 1077 | action="store_true", 1078 | help="Enable verbose logging", 1079 | ) 1080 | parser.add_argument( 1081 | "--version", 1082 | action="version", 1083 | version=f"%(prog)s {__version__}", 1084 | ) 1085 | 1086 | return parser 1087 | 1088 | 1089 | def main(): 1090 | parser = create_arg_parser() 1091 | args = parser.parse_args() 1092 | 1093 | try: 1094 | # Parse paths 1095 | if args.paths: 1096 | paths = [Path(p) for p in args.paths] 1097 | else: 1098 | paths = [PathManager.get_project_root()] 1099 | 1100 | # Parse output 1101 | if args.output: 1102 | output_path = Path(args.output) 1103 | # If just a filename, put it in generated directory 1104 | if output_path.parent == Path("."): 1105 | output_path = PathManager.get_generated_dir() / output_path 1106 | else: 1107 | output_path = PathManager.get_generated_dir() / DEFAULT_OUTPUT_NAME 1108 | 1109 | # Parse format 1110 | output_format = OutputFormat(args.format) if args.format else OutputFormat.TEXT 1111 | 1112 | config = Config( 1113 | paths=paths, 1114 | output=output_path, 1115 | preset=args.preset, 1116 | include=args.include or [], 1117 | exclude=args.exclude or [], 1118 | max_file_size_mb=args.max_file_size_mb, 1119 | output_format=output_format, 1120 | minify=args.minify, 1121 | verbose=args.verbose, 1122 | dry_run=args.dry_run, 1123 | ) 1124 | 1125 | generator = Generator(config) 1126 | generator.run() 1127 | 1128 | except KeyboardInterrupt: 1129 | logger.info("\nOperation cancelled by user.") 1130 | sys.exit(130) 1131 | except Exception as e: 1132 | # Avoid referencing args if an exception happens before args is defined. 1133 | verbose_flag = False 1134 | if "args" in locals(): 1135 | try: 1136 | verbose_flag = bool(getattr(args, "verbose", False)) 1137 | except Exception: 1138 | verbose_flag = False 1139 | logger.error(f"An unexpected error occurred: {e}", exc_info=verbose_flag) 1140 | sys.exit(1) 1141 | 1142 | 1143 | if __name__ == "__main__": 1144 | main() 1145 | --------------------------------------------------------------------------------