├── CNAME ├── LICENSE ├── README.md ├── index.html └── script.js /CNAME: -------------------------------------------------------------------------------- 1 | dropnread.io -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Project Flattener 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Project Flattener 2 | 3 | A web-based tool that helps prepare source code for Large Language Models (LLMs) by combining multiple files into a single text file. Perfect for when you need to share your codebase with AI models like GPT-4, Claude, or Gemini. 4 | 5 | ## Features 6 | 7 | - 🗂️ Drag & drop folder support 8 | - 📏 Token limit awareness for various LLMs 9 | - 🧹 Comment stripping for common programming languages 10 | - ⚙️ Configurable file/folder exclusion patterns 11 | - 📊 Real-time token counting 12 | - 💾 Binary file handling options 13 | 14 | ## Supported Models 15 | 16 | - GPT-4 (128K tokens) 17 | - Claude 2 (100K tokens) 18 | - Gemini Experimental 1206 (2M tokens) 19 | - Gemini 1.5 Pro (1M tokens) 20 | 21 | ## Usage 22 | 23 | 1. Open `index.html` in a modern browser (Chrome/Edge recommended) 24 | 2. Select your target LLM model 25 | 3. Configure options: 26 | - Strip comments to reduce token usage 27 | - Include/exclude binary files 28 | - Adjust file exclusion patterns 29 | 4. Drag a folder or use the "Select Directory" button 30 | 5. Copy the processed output to clipboard 31 | 32 | ## Comment Stripping Support 33 | 34 | Supports comment removal for: 35 | - JavaScript/TypeScript 36 | - Python 37 | - PHP 38 | - Java 39 | - C/C++ 40 | - Ruby 41 | - Go 42 | - Rust 43 | - Shell scripts 44 | - And more... 45 | 46 | ## Default Exclusions 47 | 48 | Comes with sensible defaults for excluding: 49 | - Version control directories (.git, .svn) 50 | - Dependencies (node_modules, vendor) 51 | - Build outputs (dist, build) 52 | - IDE files (.idea, .vscode) 53 | - Compiled files (*.exe, *.pyc) 54 | - And many more... 55 | 56 | ## Browser Support 57 | 58 | - Chrome/Chromium (recommended) 59 | - Edge 60 | - Other modern browsers with File System Access API support 61 | 62 | ## Local Development 63 | 64 | 1. Clone the repository 65 | 2. Open `index.html` in your browser 66 | 3. No build process or dependencies required 67 | 68 | ## License 69 | 70 | MIT License - See LICENSE file for details -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Project Flattener 6 | 7 | 8 | 40 | 41 | 42 | 43 | 48 | 49 |
50 |

Project Flattener

51 | 52 |
53 |

54 | This tool helps prepare source code for Large Language Models (LLMs) by combining multiple files into a single text. 55 | Drop a folder or click Select Directory to start. 56 |

57 |
58 |

• Model Selection: Choose your target LLM to ensure the output fits within its token limit

59 |

• Strip Comments: Remove code comments to reduce token usage

60 |

• Include Binary Files: List binary files in the output (without their contents)

61 |

• Filters: Exclude files/folders that match specific patterns

62 |
63 |
64 | 65 |
66 |
Drop folder or file here
67 | 68 | 69 | 76 |
77 | 78 |
79 | 80 | 86 | 87 |
88 |

Model Settings

89 |
90 | 91 | 100 |
101 |
102 | 106 |

Removes comments from common programming languages to reduce token count

107 |
108 |
109 | 113 |

Lists binary files with their paths (no content)

114 |
115 |
116 | 117 |
118 |

Filters

119 |
120 | 121 | 172 |
173 |
174 | 175 |
176 |

Output Format

177 |
178 | 179 | 185 |
186 | Available variables: 187 |
188 | {path} - File path 189 |
190 | {filename} - File name 191 |
192 | {content} - File content 193 |
194 | {newline} - New line character 195 |
196 |
197 |
198 | 199 | 210 | 211 | 240 |
241 | 242 | 243 | 244 | 245 | 246 | 247 | -------------------------------------------------------------------------------- /script.js: -------------------------------------------------------------------------------- 1 | document.addEventListener('DOMContentLoaded', () => { 2 | const dropZone = document.querySelector('.drop-zone'); 3 | const loader = document.querySelector('.loader'); 4 | const currentFile = document.querySelector('.current-file'); 5 | const resultContainer = document.querySelector('.result-container'); 6 | const resultTextarea = document.getElementById('result'); 7 | const excludePatternsText = document.getElementById('exclude-patterns'); 8 | const copyBtn = document.getElementById('copy-btn'); 9 | const dropStatus = document.getElementById('drop-status'); 10 | const warningDiv = document.getElementById('size-warning'); 11 | const warningTitle = document.getElementById('warning-title'); 12 | const warningMessage = document.getElementById('warning-message'); 13 | 14 | let result = ''; 15 | let processedFilesCount = 0; 16 | 17 | // Add these constants at the top 18 | let MAX_TOKENS = 128000; // GPT-4's limit 19 | let CHARS_PER_TOKEN = 4; // Rough estimation 20 | let MAX_CHARS = MAX_TOKENS * CHARS_PER_TOKEN; 21 | 22 | // Add near other constants 23 | const COMMENT_PATTERNS = { 24 | // Single line and multi-line comment patterns for different languages 25 | '.js': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 26 | '.jsx': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 27 | '.ts': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 28 | '.tsx': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 29 | '.py': [/#.*$/gm, /'''[\s\S]*?'''/g, /"""[\s\S]*?"""/g], 30 | '.php': [/\/\/.*$/gm, /#.*$/gm, /\/\*[\s\S]*?\*\//g], 31 | '.java': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 32 | '.c': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 33 | '.cpp': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 34 | '.h': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 35 | '.hpp': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 36 | '.cs': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 37 | '.rb': [/#.*$/gm, /=begin[\s\S]*?=end/g], 38 | '.sh': [/#.*$/gm], 39 | '.bash': [/#.*$/gm], 40 | '.go': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 41 | '.rs': [/\/\/.*$/gm, /\/\*[\s\S]*?\*\//g], 42 | }; 43 | 44 | // Add with other constants at the top 45 | const stripCommentsCheckbox = document.getElementById('strip-comments'); 46 | const includeBinaryCheckbox = document.getElementById('include-binary'); 47 | 48 | // Add a button for directory selection 49 | const selectDirBtn = document.createElement('button'); 50 | selectDirBtn.textContent = 'Select Directory'; 51 | selectDirBtn.className = 'px-4 py-2 bg-blue-500 text-white rounded-md hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 transition-colors duration-200'; 52 | dropZone.appendChild(selectDirBtn); 53 | 54 | const fallbackInput = document.getElementById('fallbackInput'); 55 | 56 | selectDirBtn.addEventListener('click', async () => { 57 | try { 58 | if (window.showDirectoryPicker) { 59 | const dirHandle = await window.showDirectoryPicker(); 60 | await processDirectoryHandle(dirHandle); 61 | } else { 62 | // Safari (fallback) 63 | fallbackInput.click(); 64 | } 65 | } catch (error) { 66 | console.error('Error selecting directory:', error); 67 | dropStatus.textContent = 'Error selecting directory'; 68 | } 69 | }); 70 | 71 | // For fallback 72 | fallbackInput.addEventListener('change', async (e) => { 73 | const files = e.target.files; 74 | if (!files.length) { 75 | dropStatus.textContent = 'No files selected (fallback).'; 76 | return; 77 | } 78 | await processFallbackFiles(files); 79 | }); 80 | 81 | 82 | // Drag and drop handlers 83 | dropZone.addEventListener('dragover', (e) => { 84 | e.preventDefault(); 85 | dropZone.classList.add('dragover'); 86 | }); 87 | 88 | dropZone.addEventListener('dragleave', () => { 89 | dropZone.classList.remove('dragover'); 90 | }); 91 | 92 | dropZone.addEventListener('drop', async (e) => { 93 | e.preventDefault(); 94 | dropZone.classList.remove('dragover'); 95 | 96 | const items = e.dataTransfer.items; 97 | if (items.length === 0) { 98 | dropStatus.textContent = 'No files dropped'; 99 | return; 100 | } 101 | 102 | // Try to get directory handle from dropped items 103 | try { 104 | const item = items[0]; 105 | if (item.kind === 'file') { 106 | const handle = await item.getAsFileSystemHandle(); 107 | if (handle.kind === 'directory') { 108 | await processDirectoryHandle(handle); 109 | } else { 110 | // Handle single file 111 | const file = await handle.getFile(); 112 | result = await file.text(); 113 | displayResult(); 114 | } 115 | } 116 | } catch (error) { 117 | console.error('Error processing drop:', error); 118 | dropStatus.textContent = 'Error processing dropped items. Try using the Select Directory button instead.'; 119 | } 120 | }); 121 | 122 | async function processDirectoryHandle(dirHandle, path = '') { 123 | const excludePatterns = excludePatternsText.value 124 | .split('\n') 125 | .filter(pattern => pattern.trim()) 126 | .map(pattern => new RegExp(pattern.trim() 127 | .replace(/\./g, '\\.') 128 | .replace(/\*/g, '.*'))); 129 | 130 | loader.classList.remove('hidden'); 131 | result = ''; 132 | resultContainer.classList.add('hidden'); 133 | warningDiv.classList.add('hidden'); // Hide any previous warnings 134 | processedFilesCount = 0; 135 | document.getElementById('files-counter').textContent = '0'; 136 | 137 | // Scroll loader into view 138 | loader.scrollIntoView({ 139 | behavior: 'smooth', 140 | block: 'start' 141 | }); 142 | 143 | try { 144 | const hitLimit = await readDirectory(dirHandle, excludePatterns); 145 | if (!hitLimit) { 146 | displayResult(); 147 | } 148 | } catch (error) { 149 | console.error('Error processing directory:', error); 150 | dropStatus.textContent = 'Error processing directory'; 151 | } 152 | } 153 | 154 | async function processFallbackFiles(files) { 155 | const excludePatterns = excludePatternsText.value 156 | .split('\n') 157 | .filter(pattern => pattern.trim()) 158 | .map(pattern => new RegExp(pattern.trim() 159 | .replace(/\./g, '\\.') 160 | .replace(/\*/g, '.*'))); 161 | 162 | loader.classList.remove('hidden'); 163 | result = ''; 164 | resultContainer.classList.add('hidden'); 165 | warningDiv.classList.add('hidden'); 166 | processedFilesCount = 0; 167 | document.getElementById('files-counter').textContent = '0'; 168 | 169 | loader.scrollIntoView({ 170 | behavior: 'smooth', 171 | block: 'start' 172 | }); 173 | 174 | try { 175 | for (const file of files) { 176 | let fullPath = file.webkitRelativePath || file.name; 177 | currentFile.textContent = fullPath; 178 | 179 | if (excludePatterns.some(pattern => pattern.test(fullPath))) { 180 | continue; 181 | } 182 | 183 | processedFilesCount++; 184 | document.getElementById('files-counter').textContent = processedFilesCount; 185 | 186 | const ext = '.' + file.name.split('.').pop().toLowerCase(); 187 | const isText = isTextFile(file.name); 188 | 189 | if (isText) { 190 | let content = await file.text(); 191 | if (stripCommentsCheckbox.checked && COMMENT_PATTERNS[ext]) { 192 | content = stripComments(content, ext); 193 | } 194 | 195 | const pattern = document.getElementById('format-pattern').value; 196 | const newContent = pattern 197 | .replace('{path}', fullPath.replace(file.name, '')) 198 | .replace('{filename}', file.name) 199 | .replace('{content}', content) 200 | .replace(/{newline}/g, '\n'); 201 | 202 | if ((result.length + newContent.length) > MAX_CHARS) { 203 | result = ''; 204 | showSizeWarning(); 205 | return; 206 | } 207 | result += newContent; 208 | } else { 209 | if (includeBinaryCheckbox.checked) { 210 | const newContent = `// File: ${fullPath}\n\n`; 211 | if ((result.length + newContent.length) > MAX_CHARS) { 212 | result = ''; 213 | showSizeWarning(); 214 | return; 215 | } 216 | result += newContent; 217 | } 218 | } 219 | } 220 | displayResult(); 221 | } catch (error) { 222 | console.error('Error processing fallback files:', error); 223 | dropStatus.textContent = 'Error processing fallback files'; 224 | } 225 | } 226 | 227 | // Add this after the constants 228 | const TEXT_FILE_EXTENSIONS = new Set([ 229 | // Web files 230 | '.html', '.css', '.js', '.jsx', '.ts', '.tsx', '.json', '.xml', '.svg', '.md', '.mdx', 231 | // Config files 232 | '.env', '.yml', '.yaml', '.toml', '.ini', '.conf', '.config', 233 | // Programming languages 234 | '.py', '.java', '.cpp', '.c', '.h', '.hpp', '.cs', '.php', '.rb', '.go', '.rs', '.swift', 235 | '.kt', '.kts', '.scala', '.sh', '.bash', '.pl', '.pm', '.r', '.lua', '.sql', 236 | // Documentation 237 | '.txt', '.rtf', '.csv', '.log', '.readme', 238 | // Other text files 239 | '.gitignore', '.dockerignore', '.editorconfig' 240 | ]); 241 | 242 | // Add this helper function 243 | function isTextFile(filename) { 244 | const ext = '.' + filename.split('.').pop().toLowerCase(); 245 | return TEXT_FILE_EXTENSIONS.has(ext); 246 | } 247 | 248 | // Add this function 249 | function stripComments(content, extension) { 250 | const patterns = COMMENT_PATTERNS[extension]; 251 | if (!patterns) return content; 252 | 253 | let result = content; 254 | for (const pattern of patterns) { 255 | result = result.replace(pattern, ''); 256 | } 257 | 258 | // Remove empty lines and normalize spacing 259 | return result 260 | .split('\n') 261 | .filter(line => line.trim()) 262 | .join('\n') 263 | .replace(/\n{3,}/g, '\n\n'); 264 | } 265 | 266 | // Update the readDirectory function 267 | async function readDirectory(dirHandle, excludePatterns, path = '') { 268 | for await (const entry of dirHandle.values()) { 269 | const fullPath = path + entry.name; 270 | 271 | if (excludePatterns.some(pattern => pattern.test(fullPath))) { 272 | continue; 273 | } 274 | 275 | currentFile.textContent = fullPath; 276 | 277 | try { 278 | if (entry.kind === 'file') { 279 | processedFilesCount++; 280 | document.getElementById('files-counter').textContent = processedFilesCount; 281 | 282 | if (isTextFile(entry.name)) { 283 | const file = await entry.getFile(); 284 | let content = await file.text(); 285 | 286 | if (stripCommentsCheckbox.checked) { 287 | const ext = '.' + entry.name.split('.').pop().toLowerCase(); 288 | if (COMMENT_PATTERNS[ext]) { 289 | content = stripComments(content, ext); 290 | } 291 | } 292 | 293 | const pattern = document.getElementById('format-pattern').value; 294 | const newContent = pattern 295 | .replace('{path}', path) 296 | .replace('{filename}', entry.name) 297 | .replace('{content}', content) 298 | .replace(/{newline}/g, '\n'); 299 | 300 | if ((result.length + newContent.length) > MAX_CHARS) { 301 | result = ''; 302 | warningDiv.classList.remove('hidden'); 303 | warningTitle.textContent = '⚠️ Project Too Large'; 304 | warningMessage.textContent = 305 | `Processing stopped: Project would exceed the ${MAX_TOKENS.toLocaleString()} token limit. ` + 306 | `Try excluding more files or using a model with a larger context window.`; 307 | loader.classList.add('hidden'); 308 | 309 | // Scroll warning into view 310 | warningDiv.scrollIntoView({ 311 | behavior: 'smooth', 312 | block: 'start' 313 | }); 314 | 315 | return true; // Indicate we hit the limit 316 | } 317 | 318 | result += newContent; 319 | } else if (includeBinaryCheckbox.checked) { // Only process binary files if checkbox is checked 320 | const newContent = `// File: ${fullPath}\n\n`; 321 | if ((result.length + newContent.length) > MAX_CHARS) { 322 | result = ''; 323 | warningDiv.classList.remove('hidden'); 324 | warningTitle.textContent = '⚠️ Project Too Large'; 325 | warningMessage.textContent = 326 | `Processing stopped: Project would exceed the ${MAX_TOKENS.toLocaleString()} token limit. ` + 327 | `Try excluding more files or using a model with a larger context window.`; 328 | loader.classList.add('hidden'); 329 | 330 | // Scroll warning into view 331 | warningDiv.scrollIntoView({ 332 | behavior: 'smooth', 333 | block: 'start' 334 | }); 335 | 336 | return true; // Indicate we hit the limit 337 | } 338 | result += newContent; 339 | } 340 | } else if (entry.kind === 'directory') { 341 | if (await readDirectory(entry, excludePatterns, `${fullPath}/`)) { 342 | return true; // Propagate the limit hit up the call stack 343 | } 344 | } 345 | } catch (error) { 346 | console.error(`Error processing ${fullPath}:`, error); 347 | dropStatus.textContent = `Error processing: ${fullPath}`; 348 | } 349 | } 350 | return false; // Indicate we didn't hit the limit 351 | } 352 | 353 | function displayResult() { 354 | loader.classList.add('hidden'); 355 | resultContainer.classList.remove('hidden'); 356 | resultTextarea.value = result; 357 | updateStats(); 358 | 359 | // Scroll to result container with smooth animation 360 | resultContainer.scrollIntoView({ 361 | behavior: 'smooth', 362 | block: 'start' 363 | }); 364 | } 365 | 366 | function updateStats() { 367 | const text = resultTextarea.value; 368 | const chars = text.length; 369 | const words = text.split(/\s+/).filter(Boolean).length; 370 | const tokens = Math.ceil(chars / CHARS_PER_TOKEN); 371 | 372 | document.getElementById('char-count').textContent = `${chars.toLocaleString()} / ${MAX_CHARS.toLocaleString()}`; 373 | document.getElementById('word-count').textContent = words.toLocaleString(); 374 | document.getElementById('token-count').textContent = `${tokens.toLocaleString()} / ${MAX_TOKENS.toLocaleString()}`; 375 | 376 | // Add warning class if close to limit 377 | const tokenCount = document.getElementById('token-count'); 378 | if (tokens > MAX_TOKENS * 0.9) { 379 | tokenCount.classList.add('text-red-500'); 380 | } else if (tokens > MAX_TOKENS * 0.75) { 381 | tokenCount.classList.add('text-yellow-500'); 382 | } else { 383 | tokenCount.classList.remove('text-red-500', 'text-yellow-500'); 384 | } 385 | } 386 | 387 | copyBtn.addEventListener('click', () => { 388 | navigator.clipboard.writeText(resultTextarea.value); 389 | copyBtn.textContent = 'Copied!'; 390 | setTimeout(() => { 391 | copyBtn.textContent = 'Copy to Clipboard'; 392 | }, 2000); 393 | }); 394 | 395 | // Add near the top with other constants 396 | MAX_TOKENS = 128000; 397 | const modelSelect = document.getElementById('model-select'); 398 | 399 | modelSelect.addEventListener('change', () => { 400 | MAX_TOKENS = parseInt(modelSelect.value); 401 | MAX_CHARS = MAX_TOKENS * CHARS_PER_TOKEN; 402 | updateStats(); // Update display with new limits 403 | }); 404 | }); --------------------------------------------------------------------------------