├── .github └── FUNDING.yml ├── .gitignore ├── README.md ├── assets ├── github.svg └── icon.png ├── config.json ├── main.py ├── main_cmd.py ├── requirements.txt └── src ├── config ├── __init__.py └── config.py ├── core ├── __init__.py ├── doc_generator.py ├── file_merger.py └── file_processor.py ├── ui ├── __init__.py ├── app.py ├── backend.py └── styles.py └── utils ├── __init__.py ├── gitignore_parser.py └── logging_config.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [easydevv] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: # Replace with a single Buy Me a Coffee username 14 | thanks_dev: # Replace with a single thanks.dev username 15 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig 2 | # Created by https://www.toptal.com/developers/gitignore/api/windows,visualstudiocode,python,test 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=windows,visualstudiocode,python,test 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | cover/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | .pybuilder/ 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | # For a library or package, you might want to ignore these files since the code is 92 | # intended to run in multiple environments; otherwise, check them in: 93 | # .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # poetry 103 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 104 | # This is especially recommended for binary packages to ensure reproducibility, and is more 105 | # commonly ignored for libraries. 106 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 107 | #poetry.lock 108 | 109 | # pdm 110 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 111 | #pdm.lock 112 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 113 | # in version control. 114 | # https://pdm.fming.dev/#use-with-ide 115 | .pdm.toml 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # mypy 147 | .mypy_cache/ 148 | .dmypy.json 149 | dmypy.json 150 | 151 | # Pyre type checker 152 | .pyre/ 153 | 154 | # pytype static type analyzer 155 | .pytype/ 156 | 157 | # Cython debug symbols 158 | cython_debug/ 159 | 160 | # PyCharm 161 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 162 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 163 | # and can be added to the global gitignore or merged into this file. For a more nuclear 164 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 165 | #.idea/ 166 | 167 | ### Python Patch ### 168 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 169 | poetry.toml 170 | 171 | # ruff 172 | .ruff_cache/ 173 | 174 | # LSP config files 175 | pyrightconfig.json 176 | 177 | ### Test ### 178 | ### Ignore all files that could be used to test your code and 179 | ### you wouldn't want to push 180 | 181 | # Reference https://en.wikipedia.org/wiki/Metasyntactic_variable 182 | 183 | # Most common 184 | *foo 185 | *bar 186 | *fubar 187 | *foobar 188 | *baz 189 | 190 | # Less common 191 | *qux 192 | *quux 193 | *bongo 194 | *bazola 195 | *ztesch 196 | 197 | # UK, Australia 198 | *wibble 199 | *wobble 200 | *wubble 201 | *flob 202 | *blep 203 | *blah 204 | *boop 205 | *beep 206 | 207 | # Japanese 208 | *hoge 209 | *piyo 210 | *fuga 211 | *hogera 212 | *hogehoge 213 | 214 | # Portugal, Spain 215 | *fulano 216 | *sicrano 217 | *beltrano 218 | *mengano 219 | *perengano 220 | *zutano 221 | 222 | # France, Italy, the Netherlands 223 | *toto 224 | *titi 225 | *tata 226 | *tutu 227 | *pipppo 228 | *pluto 229 | *paperino 230 | *aap 231 | *noot 232 | *mies 233 | 234 | # Other names that would make sense 235 | *tests 236 | *testsdir 237 | *testsfile 238 | *testsfiles 239 | *test 240 | *testdir 241 | *testfile 242 | *testfiles 243 | *testing 244 | *testingdir 245 | *testingfile 246 | *testingfiles 247 | *temp 248 | *tempdir 249 | *tempfile 250 | *tempfiles 251 | *tmp 252 | *tmpdir 253 | *tmpfile 254 | *tmpfiles 255 | *lol 256 | 257 | ### VisualStudioCode ### 258 | .vscode 259 | !.vscode/settings.json 260 | !.vscode/tasks.json 261 | !.vscode/launch.json 262 | !.vscode/extensions.json 263 | !.vscode/*.code-snippets 264 | 265 | # Local History for Visual Studio Code 266 | .history/ 267 | 268 | # Built Visual Studio Code Extensions 269 | *.vsix 270 | 271 | ### VisualStudioCode Patch ### 272 | # Ignore all local history of files 273 | .history 274 | .ionide 275 | 276 | ### Windows ### 277 | # Windows thumbnail cache files 278 | Thumbs.db 279 | Thumbs.db:encryptable 280 | ehthumbs.db 281 | ehthumbs_vista.db 282 | 283 | # Dump file 284 | *.stackdump 285 | 286 | # Folder config file 287 | [Dd]esktop.ini 288 | 289 | # Recycle Bin used on file shares 290 | $RECYCLE.BIN/ 291 | 292 | # Windows Installer files 293 | *.cab 294 | *.msi 295 | *.msix 296 | *.msm 297 | *.msp 298 | 299 | # Windows shortcuts 300 | *.lnk 301 | 302 | # End of https://www.toptal.com/developers/gitignore/api/windows,visualstudiocode,python,test 303 | 304 | # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) 305 | 306 | test* 307 | .output-md/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | project-to-markdown-logo 4 |

5 | 6 | # Project To Markdown 7 | 8 | **Project To Markdown** is a Python application that converts project files into structured markdown format, optimizing documentation for AI analysis. Designed to enhance the performance of advanced language models like **GPT and Claude**, it improves the efficiency of AI Retrieval-Augmented Generation (RAG) systems. 9 | 10 | The tool offers both GUI and CLI, making it accessible to various users. By transforming content into an AI-friendly format, Project To Markdown enables AI assistants to provide more accurate and context-aware responses when queried about project details. 11 | 12 | ## Demo 13 | 14 | Use GUI version to generate markdown results. Then, attach these results to GPT or Claude Chat to get more accurate answers to your questions. 15 | 16 | ![project-to-markdown-preview](https://github.com/user-attachments/assets/cb02da9e-0b68-4f69-8ceb-a1cc28784b2e) 17 | 18 | https://github.com/user-attachments/assets/8049c059-d0fe-4994-80bf-3e6f01e70879 19 | 20 | 21 | ## Recommended Use Cases 22 | 23 | 1. Document your entire project in markdown format and have it reviewed by AI. Use it to solve complex problems that span multiple files. This approach is easier as it doesn't require learning new AI tools and can be directly applied to existing AI chat interfaces. 24 | 25 | 2. Document the 'examples' folder of new or lesser-known open-source repositories. AI models don't always have the most up-to-date information, so this method can yield higher quality responses. When there's a lot of practical example data, it can significantly accelerate app development. 26 | 27 | ## Features 28 | 29 | - Convert project files to markdown, split by subfolder. 30 | - Merge all files into a single markdown file (optional) 31 | - Add timestamps to generated markdown filenames (optional) 32 | - Generate a folder structure of the project (optional) 33 | - GUI for easy interaction 34 | - CLI for automation and scripting 35 | - Respects .gitignore rules 36 | 37 | ## Installation 38 | 39 | 1. Clone the repository: 40 | ``` 41 | git clone https://github.com/easydevv/project-to-markdown.git 42 | cd project-to-markdown 43 | ``` 44 | 45 | 2. Set up a virtual environment: 46 | ``` 47 | python -m venv .venv 48 | ``` 49 | ``` 50 | .venv\scripts\activate 51 | ``` 52 | 53 | 3. Install the required dependencies: 54 | ``` 55 | pip install -r requirements.txt 56 | ``` 57 | 58 | ## Usage 59 | 60 | ### GUI Version 61 | 62 | To run the GUI version of the application: 63 | 64 | ``` 65 | python main.py 66 | ``` 67 | or 68 | ``` 69 | flet run 70 | ``` 71 | 72 | The GUI allows you to: 73 | - Select a project directory 74 | - Choose conversion options 75 | - Start the conversion process 76 | - View the conversion log 77 | 78 | ### CLI Version 79 | 80 | To run the command-line version: 81 | 82 | ``` 83 | python main_cmd.py PROJECT_PATH [OPTIONS] 84 | ``` 85 | 86 | Options: 87 | - `--onefile`: Merge all files into a single markdown file 88 | - `--timestamp`: Add timestamps to generated markdown filenames 89 | - `--no-tree`: Do not generate a folder structure file 90 | 91 | Example: 92 | ``` 93 | python main_cmd.py /path/to/your/project --onefile --timestamp 94 | ``` 95 | 96 | ## Configuration 97 | 98 | The application uses a `config.json` file to store default settings. You can modify this file to change the default behavior: 99 | 100 | ``` 101 | { 102 | "exclude_types": ["md", "log", "pyc", ...], 103 | "exclude_folders": [".output-md", "__pycache__", ...], 104 | "output_folder": ".output-md", 105 | "max_workers": 4 106 | } 107 | ``` 108 | 109 | - `exclude_types`: File extensions to exclude from conversion 110 | - `exclude_folders`: Folders to exclude from conversion 111 | - `output_folder`: Default folder for generated markdown files 112 | - `max_workers`: (not working yet) Maximum number of concurrent workers for file processing 113 | -------------------------------------------------------------------------------- /assets/github.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EasyDevv/project-to-markdown/d091d9d03340da8f1c49a9dbc39c850624aac716/assets/icon.png -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude_types": [ 3 | "md", 4 | "log", 5 | "pyc", 6 | "pyo", 7 | "pyd", 8 | "dll", 9 | "so", 10 | "exe", 11 | "jpg", 12 | "jpeg", 13 | "png", 14 | "gif", 15 | "svg", 16 | "webp", 17 | "tiff", 18 | "tif", 19 | "psd" 20 | ], 21 | "exclude_folders": [ 22 | ".output-md", 23 | ".vscode", 24 | ".venv", 25 | "__pycache__", 26 | ".gitignore", 27 | "node_modules", 28 | ".git", 29 | "image", 30 | "images" 31 | ], 32 | "output_folder": ".output-md", 33 | "max_workers": 4 34 | } -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | 3 | from src.ui import App 4 | 5 | 6 | def main(): 7 | app = App() 8 | ft.app(target=app.main, assets_dir="assets") 9 | 10 | 11 | if __name__ == "__main__": 12 | main() 13 | -------------------------------------------------------------------------------- /main_cmd.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import logging 4 | from pathlib import Path 5 | 6 | from src.core import FileMerger 7 | from src.utils import get_logger, setup_logging 8 | 9 | setup_logging(console_level=logging.INFO) 10 | logger = get_logger(__name__) 11 | 12 | 13 | def get_project_path(): 14 | while True: 15 | path = input("Enter the project folder path: ").strip() 16 | if path: 17 | return Path(path) 18 | print("Please enter a valid path.") 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser( 23 | description="Merge project files into a single markdown file." 24 | ) 25 | parser.add_argument( 26 | "project_path", 27 | type=Path, 28 | help="Path to the project folder", 29 | ) 30 | parser.add_argument( 31 | "--onefile", 32 | action="store_true", 33 | help="Integrate all subdirectories into a single document.", 34 | ) 35 | parser.add_argument( 36 | "--timestamp", 37 | action="store_true", 38 | help="Add a timestamp to the project file name.", 39 | ) 40 | parser.add_argument( 41 | "--no-tree", 42 | action="store_false", 43 | help="Generate a project tree structure file.", 44 | ) 45 | 46 | args = parser.parse_args() 47 | 48 | path = args.project_path or get_project_path() 49 | 50 | merger = FileMerger( 51 | project_path=path, 52 | merge_onefile=args.onefile, 53 | enable_timestamp=args.timestamp, 54 | enable_folder_structure=args.no_tree, 55 | logger=logger, 56 | ) 57 | asyncio.run(merger.merge_files()) 58 | 59 | 60 | if __name__ == "__main__": 61 | main() 62 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flet==0.23.2 2 | aiofiles==24.1.0 3 | rich==13.8.0 -------------------------------------------------------------------------------- /src/config/__init__.py: -------------------------------------------------------------------------------- 1 | from .config import Config 2 | 3 | __all__ = ["Config"] 4 | -------------------------------------------------------------------------------- /src/config/config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import json 3 | import logging 4 | 5 | 6 | class Config: 7 | """ 8 | A class for managing application settings. 9 | """ 10 | 11 | def __init__( 12 | self, logger=logging.getLogger(), config_file: Path = Path("config.json") 13 | ): 14 | """ 15 | Initialize the Config class. 16 | 17 | :param logger: Logger object 18 | :param config_file: Path to the configuration file 19 | """ 20 | self.logger = logger 21 | self.config_file = config_file 22 | self.exclude_types = [ 23 | "md", 24 | "log", 25 | "pyc", 26 | "pyo", 27 | "pyd", 28 | "dll", 29 | "so", 30 | "exe", 31 | "jpg", 32 | "jpeg", 33 | "png", 34 | "gif", 35 | "svg", 36 | "webp", 37 | "tiff", 38 | "tif", 39 | "psd", 40 | ] 41 | self.exclude_folders = [ 42 | ".output-md", 43 | ".vscode", 44 | ".venv", 45 | "__pycache__", 46 | ".gitignore", 47 | "node_modules", 48 | ".git", 49 | "image", 50 | "images", 51 | ] 52 | self.output_dir = ".output-md" 53 | self.max_workers = 4 54 | 55 | self.load_config() 56 | 57 | def load_config(self): 58 | """ 59 | Load settings from the configuration file. If the file doesn't exist, create it with default values. 60 | """ 61 | if self.config_file.exists(): 62 | with self.config_file.open("r", encoding="utf-8-sig") as f: 63 | data = json.load(f) 64 | self.exclude_types = data.get("exclude_types", self.exclude_types) 65 | self.exclude_folders = data.get("exclude_folders", self.exclude_folders) 66 | self.output_dir = data.get("output_folder", self.output_dir) 67 | self.max_workers = data.get("max_workers", self.max_workers) 68 | self.logger.info("Configuration loaded successfully") 69 | else: 70 | self.logger.warning( 71 | "Configuration file not found, creating with default values" 72 | ) 73 | self.save_config() 74 | 75 | def save_config(self): 76 | """ 77 | Save the current settings to the configuration file. 78 | """ 79 | with self.config_file.open("w", encoding="utf-8-sig") as f: 80 | json.dump( 81 | { 82 | "exclude_types": self.exclude_types, 83 | "exclude_folders": self.exclude_folders, 84 | "output_folder": self.output_dir, 85 | "max_workers": self.max_workers, 86 | }, 87 | f, 88 | indent=4, 89 | ) 90 | self.logger.info("Configuration saved successfully") 91 | -------------------------------------------------------------------------------- /src/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .file_merger import FileMerger 2 | from .doc_generator import ( 3 | generate_content, 4 | generate_tree_structure, 5 | generate_onefile_content, 6 | ) 7 | from .file_processor import FileProcessor 8 | 9 | __all__ = [ 10 | "FileMerger", 11 | "generate_content", 12 | "generate_tree_structure", 13 | "generate_onefile_content", 14 | "FileProcessor", 15 | ] 16 | -------------------------------------------------------------------------------- /src/core/doc_generator.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pathlib import Path 3 | from typing import List, Tuple 4 | 5 | 6 | async def generate_tree_structure(files: List[Path], project_path: Path) -> str: 7 | """ 8 | Asynchronously generates a complete document content including the tree structure of given files with emojis. 9 | 10 | :param files: List of files to generate the tree structure for 11 | :param project_path: Root path of the project 12 | :return: Generated document content string 13 | """ 14 | 15 | async def add_to_tree(current_path: Path, depth: int): 16 | items = sorted(current_path.iterdir(), key=lambda x: (x.is_file(), x.name)) 17 | for i, item in enumerate(items): 18 | rel_path = item.relative_to(project_path) 19 | is_last = i == len(items) - 1 20 | prefix = "└── " if is_last else "├── " 21 | 22 | if item.is_file() and item in files: 23 | tree.append(f"{indent * depth}{prefix}📄 {rel_path.name}") 24 | elif item.is_dir(): 25 | if any(file.is_relative_to(item) for file in files): 26 | tree.append(f"{indent * depth}{prefix}📂 {rel_path.name}") 27 | await add_to_tree(item, depth + 1) 28 | 29 | tree = ["📦 root"] 30 | indent = " " 31 | await add_to_tree(project_path, 1) 32 | 33 | tree_str = "\n".join(tree) 34 | 35 | content = [] 36 | 37 | content.append("# Project Structure") 38 | content.append(f"```\n{tree_str}\n```") 39 | 40 | return "\n".join(content) 41 | 42 | 43 | async def generate_content( 44 | folder_path: Path, files: List[Tuple[Path, str, str]] 45 | ) -> str: 46 | """ 47 | Asynchronously generates content for each folder. 48 | 49 | :param folder_path: Path of the folder 50 | :param files: List of files in the folder (file path, extension, file content) 51 | :param project_path: Root path of the project 52 | :return: Generated document content 53 | """ 54 | folder_name = Path(folder_path) 55 | 56 | async def process_file(file_path: Path, extension: str, file_content: str) -> str: 57 | content = [] 58 | content.append(f"## {folder_name / file_path.name}") 59 | content.append(f"```{extension}") 60 | content.append(file_content.strip()) 61 | content.append("```\n") 62 | return "\n".join(content) 63 | 64 | content = [f"# {folder_name} Contents"] 65 | 66 | # Process each file asynchronously 67 | file_contents = await asyncio.gather( 68 | *[ 69 | process_file(file_path, extension, file_content) 70 | for file_path, extension, file_content in sorted(files) 71 | ] 72 | ) 73 | 74 | content.extend(file_contents) 75 | 76 | return "\n".join(content) 77 | 78 | 79 | async def generate_onefile_content( 80 | folder_path: Path, files: List[Tuple[Path, str, str]] 81 | ) -> str: 82 | """Generate content for a single file containing all processed files asynchronously. 83 | 84 | :param folder_path: Path of the folder 85 | :param files: List of files in the folder (file path, extension, file content) 86 | :param project_path: Root path of the project 87 | :return: Generated document content 88 | """ 89 | content = [f"# {folder_path.name} Project Contents"] 90 | 91 | async def process_file(file_path: Path, extension: str, file_content: str) -> str: 92 | relative_path = file_path.relative_to(folder_path) 93 | return f"## {relative_path}\n```{extension}\n{file_content.strip()}\n```\n" 94 | 95 | # Process each file asynchronously 96 | file_contents = await asyncio.gather( 97 | *[ 98 | process_file(file_path, extension, file_content) 99 | for file_path, extension, file_content in sorted(files) 100 | ] 101 | ) 102 | 103 | content.extend(file_contents) 104 | return "\n".join(content) 105 | -------------------------------------------------------------------------------- /src/core/file_merger.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from datetime import datetime 4 | from pathlib import Path 5 | from typing import Dict, List, Optional, Tuple 6 | 7 | import aiofiles 8 | 9 | from src.config import Config 10 | from src.utils import GitIgnoreParser 11 | 12 | from .doc_generator import ( 13 | generate_content, 14 | generate_tree_structure, 15 | generate_onefile_content, 16 | ) 17 | from .file_processor import FileProcessor 18 | 19 | 20 | class MergeException(Exception): 21 | """Represents an exception that occurs during file merging.""" 22 | 23 | 24 | class FileMerger: 25 | """A class for merging project files.""" 26 | 27 | def __init__( 28 | self, 29 | project_path: Path, 30 | merge_onefile: bool, 31 | enable_timestamp: bool, 32 | enable_folder_structure: bool, 33 | logger: logging.Logger, 34 | ): 35 | """Initialize the FileMerger class.""" 36 | config = Config(logger=logger) 37 | 38 | self.project_path = project_path.resolve() 39 | self.exclude_types = set(config.exclude_types) 40 | self.onefile = merge_onefile 41 | self.enable_timestamp = enable_timestamp 42 | self.enable_folder_structure = enable_folder_structure 43 | self.logger = logger 44 | 45 | self.gitignore_parser = GitIgnoreParser(self.project_path) 46 | self.file_processor = FileProcessor() 47 | self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") 48 | self.excluded_folders = set(config.exclude_folders) 49 | 50 | self.filtered_files: List[Path] = [] 51 | 52 | self.output_dir = self.project_path / config.output_dir 53 | self.output_dir.mkdir(parents=True, exist_ok=True) 54 | 55 | async def initialize(self): 56 | """Initialize FileMerger. Initializes GitIgnoreParser.""" 57 | await self.gitignore_parser.initialize() 58 | 59 | def _generate_onefile_filename(self) -> Path: 60 | """Generate the name for the single file.""" 61 | output_filename = f"{self.project_path.name}_codes.md" 62 | return self.output_dir / output_filename 63 | 64 | def _generate_tree_structure_filename(self) -> Path: 65 | """Generate the name for the tree structure file.""" 66 | output_filename = f"{self.project_path.name}_structure.md" 67 | return self.output_dir / output_filename 68 | 69 | def _generate_output_path(self, folder_path: Optional[Path] = None) -> Path: 70 | """Generate the path for the output file.""" 71 | project_name = self.project_path.name 72 | 73 | folder_part = "root" if folder_path == Path(".") else str(folder_path) 74 | folder_part = folder_part.replace("/", "-").replace("\\", "-") 75 | output_filename = f"{project_name}_{folder_part}" 76 | 77 | if self.enable_timestamp: 78 | output_filename += f"-{self.timestamp}" 79 | 80 | output_filename += ".md" 81 | 82 | return self.output_dir / output_filename 83 | 84 | async def _filter_files(self): 85 | """Filter files within the project, excluding specified folders.""" 86 | self.filtered_files = [ 87 | file_path 88 | for file_path in self.project_path.rglob("*") 89 | if await self._should_process_file(file_path) 90 | ] 91 | self.logger.info(f"Filtered files count: {len(self.filtered_files)}") 92 | 93 | async def _should_process_file(self, file_path: Path) -> bool: 94 | """Check if the given file should be processed.""" 95 | return ( 96 | file_path.is_file() 97 | and file_path.suffix[1:] not in self.exclude_types 98 | and not await self.gitignore_parser.is_ignored(file_path) 99 | and not self._is_excluded_folder(file_path) 100 | ) 101 | 102 | def _is_excluded_folder(self, path: Path) -> bool: 103 | """Check if the given path is in an excluded folder.""" 104 | return any(excluded in path.parts for excluded in self.excluded_folders) 105 | 106 | async def _process_files(self) -> List[Tuple[Path, str, str]]: 107 | """Process all filtered files.""" 108 | tasks = [ 109 | self._process_file_wrapper(file_path) for file_path in self.filtered_files 110 | ] 111 | processed_files = [result for result in await asyncio.gather(*tasks) if result] 112 | self.logger.info(f"Processed: {len(processed_files)} files") 113 | return processed_files 114 | 115 | async def _process_file_wrapper( 116 | self, file_path: Path 117 | ) -> Optional[Tuple[Path, str, str]]: 118 | """A wrapper function for processing individual files.""" 119 | try: 120 | result = await self.file_processor.process_file( 121 | file_path, self.project_path 122 | ) 123 | self.logger.debug( 124 | f"Processed: {file_path}, content length: {len(result[2]) if result else 0}" 125 | ) 126 | return result 127 | except Exception as e: 128 | self.logger.error(f"Error processing file {file_path}: {str(e)}") 129 | return None 130 | 131 | async def _categorize_files( 132 | self, processed_files: List[Tuple[Path, str, str]] 133 | ) -> Dict[Path, List[Tuple[Path, str, str]]]: 134 | """Categorize processed files by folder asynchronously.""" 135 | categorized = {} 136 | 137 | async def categorize_file(file_path: Path, extension: str, content: str): 138 | folder = file_path.parent 139 | if folder not in categorized: 140 | categorized[folder] = [] 141 | categorized[folder].append((file_path, extension, content)) 142 | 143 | await asyncio.gather( 144 | *( 145 | categorize_file(file_path, extension, content) 146 | for file_path, extension, content in processed_files 147 | ) 148 | ) 149 | 150 | return categorized 151 | 152 | async def _write_multiple_files( 153 | self, categorized_files: Dict[Path, List[Tuple[Path, str, str]]] 154 | ) -> List[Path]: 155 | """Write content to multiple files based on folder structure.""" 156 | output_files = [] 157 | for folder_path, files in categorized_files.items(): 158 | try: 159 | output_file = self._generate_output_path(folder_path) 160 | 161 | content = await generate_content(folder_path, files) 162 | async with aiofiles.open( 163 | output_file, "w", encoding="utf-8" 164 | ) as out_file: 165 | await out_file.write(content) 166 | output_files.append(output_file) 167 | self.logger.info(f"Created file: {output_file}") 168 | except Exception as e: 169 | self.logger.error(f"Error processing folder {folder_path}: {str(e)}") 170 | return output_files 171 | 172 | async def _write_onefile(self, processed_files) -> Path: 173 | """Write the content to a single file.""" 174 | output_file = self._generate_onefile_filename() 175 | 176 | folder_path = Path(".") 177 | content = await generate_onefile_content(folder_path, processed_files) 178 | # content = await generate_onefile_content(processed_files) 179 | 180 | async with aiofiles.open(output_file, "w", encoding="utf-8") as out_file: 181 | await out_file.write(content) 182 | self.logger.info(f"Created single file: {output_file}") 183 | return output_file 184 | 185 | async def _generate_tree_structure(self) -> Optional[Path]: 186 | """Generate and write the tree structure file if enabled.""" 187 | if not self.enable_folder_structure: 188 | return None 189 | 190 | try: 191 | tree_structure = await generate_tree_structure( 192 | files=self.filtered_files, project_path=self.project_path 193 | ) 194 | tree_output_file = self._generate_tree_structure_filename() 195 | async with aiofiles.open( 196 | tree_output_file, "w", encoding="utf-8" 197 | ) as tree_file: 198 | await tree_file.write(tree_structure) 199 | self.logger.info(f"Created structure file: {tree_output_file}") 200 | return tree_output_file 201 | except Exception as e: 202 | self.logger.error(f"Error generating structure: {str(e)}") 203 | return None 204 | 205 | async def merge_files(self) -> List[Path]: 206 | """ 207 | Main function for merging files. 208 | 209 | :param progress_callback: Callback function to report progress 210 | :return: Paths of generated output files 211 | """ 212 | try: 213 | await self.initialize() 214 | await self._filter_files() 215 | processed_files = await self._process_files() 216 | 217 | output_files = [] 218 | 219 | if self.onefile: 220 | output_file = await self._write_onefile(processed_files) 221 | output_files.append(output_file) 222 | else: 223 | categorized_files = await self._categorize_files(processed_files) 224 | output_files.extend(await self._write_multiple_files(categorized_files)) 225 | 226 | tree_structure_file = await self._generate_tree_structure() 227 | if tree_structure_file: 228 | output_files.append(tree_structure_file) 229 | 230 | return output_files 231 | 232 | except Exception as e: 233 | self.logger.error(f"Unexpected error during file merging: {str(e)}") 234 | raise MergeException(str(e)) 235 | -------------------------------------------------------------------------------- /src/core/file_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | from typing import Optional, Tuple 4 | 5 | import aiofiles 6 | 7 | 8 | class FileProcessor: 9 | @staticmethod 10 | def get_file_extension(filename: Path) -> str: 11 | return filename.suffix[1:] 12 | 13 | @staticmethod 14 | async def read_file_content(file_path: Path) -> Optional[str]: 15 | try: 16 | async with aiofiles.open(file_path, "r") as f: 17 | return await f.read() 18 | except UnicodeDecodeError: 19 | logging.warning(f"Unable to read {file_path} as UTF-8. Skipping.") 20 | except PermissionError: 21 | logging.error(f"Permission denied: Unable to read {file_path}") 22 | except Exception as e: 23 | logging.error(f"Unexpected error reading {file_path}: {str(e)}") 24 | return None 25 | 26 | async def process_file( 27 | self, file_path: Path, project_path: Path 28 | ) -> Tuple[Path, str, Optional[str]]: 29 | relative_path = file_path.relative_to(project_path) 30 | extension = self.get_file_extension(file_path) 31 | content = await self.read_file_content(file_path) 32 | return relative_path, extension, content 33 | -------------------------------------------------------------------------------- /src/ui/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import App 2 | from .backend import Backend 3 | from .styles import styles 4 | 5 | 6 | __all__ = ["App", "Backend", "styles"] 7 | -------------------------------------------------------------------------------- /src/ui/app.py: -------------------------------------------------------------------------------- 1 | import flet as ft 2 | from pathlib import Path 3 | from queue import Queue 4 | from threading import Thread 5 | from .backend import Backend 6 | from .styles import styles 7 | 8 | # GitHub repository URL 9 | GITHUB_URL = "https://github.com/easydevv/project-to-markdown" 10 | 11 | 12 | class App: 13 | def __init__(self): 14 | self.backend = Backend(log_callback=self.log_callback) 15 | self.log_queue = Queue() 16 | self.log_thread = Thread(target=self.process_log_queue, daemon=True) 17 | self.log_thread.start() 18 | 19 | def log_callback(self, message): 20 | self.log_queue.put(message) 21 | 22 | def process_log_queue(self): 23 | while True: 24 | message = self.log_queue.get() 25 | if message == "STOP": 26 | break 27 | if hasattr(self, "log_view") and hasattr(self, "page"): 28 | self.log_view.controls.append( 29 | ft.Text(message, color=styles.colors.text) 30 | ) 31 | self.page.update() 32 | 33 | def main(self, page: ft.Page): 34 | self.page = page 35 | page.title = "Project To Markdown" 36 | page.padding = 20 37 | page.window.width = 700 38 | page.theme_mode = ft.ThemeMode.DARK 39 | 40 | def pick_directory_result(e: ft.FilePickerResultEvent): 41 | if e.path: 42 | self.backend.set_project_path(Path(e.path)) 43 | project_path_text.value = str(self.backend.project_path) 44 | merge_button.disabled = False 45 | else: 46 | project_path_text.value = "" 47 | merge_button.disabled = True 48 | page.update() 49 | 50 | def on_project_path_change(e): 51 | path = e.control.value.strip() 52 | is_valid_path = path != "" and Path(path).exists() and Path(path).is_dir() 53 | merge_button.disabled = not is_valid_path 54 | if is_valid_path: 55 | self.backend.set_project_path(Path(path)) 56 | page.update() 57 | 58 | pick_directory_dialog = ft.FilePicker(on_result=pick_directory_result) 59 | page.overlay.append(pick_directory_dialog) 60 | 61 | header = self.create_header() 62 | project_path_text = self.create_project_path_text(on_project_path_change) 63 | pick_directory_button = self.create_pick_directory_button(pick_directory_dialog) 64 | options = self.create_options() 65 | progress_bar = ft.ProgressBar(visible=False, color=styles.colors.primary) 66 | status_text = ft.Text(color=styles.colors.text) 67 | merge_button = self.create_merge_button(progress_bar, page) 68 | 69 | options_container = self.create_options_container(options) 70 | file_section = self.create_file_section( 71 | project_path_text, 72 | pick_directory_button, 73 | merge_button, 74 | progress_bar, 75 | status_text, 76 | ) 77 | log_container = self.create_log_container() 78 | 79 | page.add( 80 | header, 81 | ft.Divider(height=styles.divider.height, color=styles.divider.color), 82 | ft.Row( 83 | [file_section, options_container], 84 | alignment=ft.MainAxisAlignment.START, 85 | expand=True, 86 | spacing=20, 87 | ), 88 | ft.Divider(height=styles.divider.height, color=styles.divider.color), 89 | log_container, 90 | ) 91 | 92 | def create_header(self): 93 | github_icon = ft.Image( 94 | src="github.svg", 95 | width=28, 96 | height=28, 97 | ) 98 | github_button = ft.Container( 99 | content=github_icon, 100 | on_click=lambda _: self.open_github_url(), 101 | ) 102 | return ft.Row( 103 | [ 104 | ft.Text( 105 | "Project To Markdown", 106 | size=24, 107 | weight=ft.FontWeight.BOLD, 108 | color=styles.title.color, 109 | ), 110 | github_button, 111 | ], 112 | alignment=ft.MainAxisAlignment.SPACE_BETWEEN, 113 | ) 114 | 115 | def open_github_url(self): 116 | import webbrowser 117 | 118 | webbrowser.open(GITHUB_URL) 119 | 120 | def create_project_path_text(self, on_change): 121 | return ft.TextField( 122 | label="Project Path", 123 | on_change=on_change, 124 | prefix_icon=ft.icons.SEARCH, 125 | dense=True, 126 | multiline=True, 127 | expand=True, 128 | border_color=styles.colors.border, 129 | border_radius=styles.textfield.border_radius, 130 | label_style=styles.textfield.label_style, 131 | text_style=styles.textfield.text_style, 132 | bgcolor=styles.textfield.bgcolor, 133 | ) 134 | 135 | def create_pick_directory_button(self, pick_directory_dialog): 136 | return ft.ElevatedButton( 137 | "Pick Directory", 138 | icon=ft.icons.FOLDER_OPEN, 139 | on_click=lambda _: pick_directory_dialog.get_directory_path(), 140 | style=styles.button.style, 141 | ) 142 | 143 | def create_options(self): 144 | return ft.Column( 145 | [ 146 | ft.Checkbox( 147 | label="onefile", 148 | tooltip="Merge into one markdown file", 149 | value=self.backend.merge_onefile, 150 | on_change=lambda e: self.backend.set_merge_onefile(e.control.value), 151 | ), 152 | ft.Checkbox( 153 | label="timestamp", 154 | tooltip="Add timestamp to each markdown filename", 155 | value=self.backend.enable_timestamp, 156 | on_change=lambda e: self.backend.set_enable_timestamp( 157 | e.control.value 158 | ), 159 | ), 160 | ft.Checkbox( 161 | label="tree", 162 | tooltip="Generate folder structure", 163 | value=self.backend.enable_folder_structure, 164 | on_change=lambda e: self.backend.set_enable_folder_structure( 165 | e.control.value 166 | ), 167 | ), 168 | ] 169 | ) 170 | 171 | def create_merge_button(self, progress_bar, page): 172 | async def merge_files(e): 173 | if not self.backend.project_path: 174 | self.log_callback("Please select a project directory first.") 175 | page.update() 176 | return 177 | 178 | progress_bar.visible = True 179 | e.control.disabled = True 180 | page.update() 181 | 182 | try: 183 | await self.backend.merge_files() 184 | except Exception as ex: 185 | error_message = f"Error: {str(ex)}" 186 | self.log_callback(error_message) 187 | 188 | progress_bar.visible = False 189 | e.control.disabled = False 190 | page.update() 191 | 192 | merge_button = ft.ElevatedButton( 193 | "Merge Files", 194 | on_click=merge_files, 195 | disabled=True, 196 | style=styles.button.style, 197 | ) 198 | 199 | return merge_button 200 | 201 | def create_options_container(self, options): 202 | return ft.Container( 203 | content=ft.Column( 204 | [ 205 | ft.Text( 206 | "Options", 207 | size=styles.title.size, 208 | weight=styles.title.weight, 209 | color=styles.title.color, 210 | ), 211 | options, 212 | ] 213 | ), 214 | padding=styles.container.padding, 215 | border_radius=styles.container.border_radius, 216 | border=styles.container.border, 217 | ) 218 | 219 | def create_file_section( 220 | self, 221 | project_path_text, 222 | pick_directory_button, 223 | merge_button, 224 | progress_bar, 225 | status_text, 226 | ): 227 | return ft.Container( 228 | content=ft.Column( 229 | [ 230 | ft.Text( 231 | "File Selection", 232 | size=styles.title.size, 233 | weight=styles.title.weight, 234 | color=styles.title.color, 235 | ), 236 | ft.Divider(height=5, color=styles.divider.color), 237 | project_path_text, 238 | ft.Row([pick_directory_button, merge_button]), 239 | progress_bar, 240 | status_text, 241 | ], 242 | horizontal_alignment=ft.CrossAxisAlignment.START, 243 | ), 244 | expand=True, 245 | padding=styles.container.padding, 246 | border_radius=styles.container.border_radius, 247 | border=styles.container.border, 248 | ) 249 | 250 | def create_log_container(self): 251 | self.log_view = ft.ListView(expand=1, spacing=10, padding=20, auto_scroll=True) 252 | return ft.Container( 253 | content=ft.Column( 254 | [ 255 | ft.Text( 256 | "Log", 257 | size=styles.title.size, 258 | weight=styles.title.weight, 259 | color=styles.title.color, 260 | ), 261 | ft.Container( 262 | content=self.log_view, 263 | height=200, 264 | border=styles.container.border, 265 | border_radius=styles.container.border_radius, 266 | padding=10, 267 | ), 268 | ] 269 | ), 270 | padding=styles.container.padding, 271 | border_radius=styles.container.border_radius, 272 | border=styles.container.border, 273 | ) 274 | -------------------------------------------------------------------------------- /src/ui/backend.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | from typing import Callable 4 | 5 | from src.core.file_merger import FileMerger 6 | from src.utils.logging_config import get_logger, setup_logging 7 | 8 | 9 | class Backend: 10 | def __init__(self, log_callback: Callable[[str], None]): 11 | self.project_path = None 12 | self.merge_onefile = False 13 | self.enable_timestamp = False 14 | self.enable_folder_structure = True 15 | self.log_callback = log_callback 16 | self.setup_logger() 17 | 18 | def setup_logger(self): 19 | setup_logging(console_level=logging.INFO) 20 | self.logger = get_logger(__name__) 21 | self.logger.addHandler(CallbackHandler(self.log_callback)) 22 | 23 | def set_project_path(self, path: Path): 24 | self.project_path = path 25 | self.logger.info(f"Selected directory: {self.project_path}") 26 | 27 | def set_merge_onefile(self, value: bool): 28 | self.merge_onefile = value 29 | self.logger.info(f"Merge into one file set to: {value}") 30 | 31 | def set_enable_timestamp(self, value: bool): 32 | self.enable_timestamp = value 33 | self.logger.info(f"Enable timestamp set to: {value}") 34 | 35 | def set_enable_folder_structure(self, value: bool): 36 | self.enable_folder_structure = value 37 | self.logger.info(f"Generate folder structure set to: {value}") 38 | 39 | async def merge_files(self): 40 | if not self.project_path: 41 | raise ValueError("Project path not set") 42 | 43 | self.logger.info("Starting file merge process") 44 | merger = FileMerger( 45 | project_path=self.project_path, 46 | merge_onefile=self.merge_onefile, 47 | enable_timestamp=self.enable_timestamp, 48 | enable_folder_structure=self.enable_folder_structure, 49 | logger=self.logger, 50 | ) 51 | 52 | try: 53 | output_files = await merger.merge_files() 54 | self.logger.info( 55 | f"Merged project files into {len(output_files)} markdown file(s)." 56 | ) 57 | return output_files 58 | except Exception as ex: 59 | self.logger.error(f"Error during file merge: {str(ex)}") 60 | raise 61 | 62 | 63 | class CallbackHandler(logging.Handler): 64 | def __init__(self, callback: Callable[[str], None]): 65 | super().__init__() 66 | self.callback = callback 67 | 68 | def emit(self, record): 69 | log_entry = self.format(record) 70 | self.callback(log_entry) 71 | -------------------------------------------------------------------------------- /src/ui/styles.py: -------------------------------------------------------------------------------- 1 | from types import SimpleNamespace 2 | import flet as ft 3 | 4 | # Color scheme 5 | BACKGROUND_COLOR = "#141414" 6 | SURFACE_COLOR = "#1E1E1E" 7 | PRIMARY_COLOR = ft.colors.BLUE 8 | BORDER_COLOR = "#333333" 9 | TEXT_COLOR = "white70" 10 | 11 | PADDING = 15 12 | BORDER_RADIUS = 8 13 | 14 | styles = SimpleNamespace( 15 | colors=SimpleNamespace( 16 | background=BACKGROUND_COLOR, 17 | surface=SURFACE_COLOR, 18 | primary=PRIMARY_COLOR, 19 | border=BORDER_COLOR, 20 | text=TEXT_COLOR, 21 | ), 22 | container=SimpleNamespace( 23 | padding=PADDING, 24 | border_radius=BORDER_RADIUS, 25 | border=ft.border.all(1, BORDER_COLOR), 26 | ), 27 | title=SimpleNamespace( 28 | size=18, 29 | weight=ft.FontWeight.BOLD, 30 | color="#fafafa", 31 | ), 32 | text=SimpleNamespace( 33 | color=TEXT_COLOR, 34 | ), 35 | button=SimpleNamespace( 36 | style=ft.ButtonStyle( 37 | shape={ 38 | ft.MaterialState.DEFAULT: ft.RoundedRectangleBorder( 39 | radius=BORDER_RADIUS 40 | ), 41 | ft.MaterialState.HOVERED: ft.RoundedRectangleBorder( 42 | radius=BORDER_RADIUS * 2 43 | ), 44 | }, 45 | color={ 46 | ft.MaterialState.DEFAULT: SURFACE_COLOR, 47 | }, 48 | bgcolor={ 49 | ft.MaterialState.DEFAULT: ft.colors.GREY_100, 50 | ft.MaterialState.HOVERED: ft.colors.GREY_600, 51 | ft.MaterialState.DISABLED: ft.colors.GREY_800, 52 | }, 53 | ) 54 | ), 55 | textfield=SimpleNamespace( 56 | border_color=BORDER_COLOR, 57 | border_radius=BORDER_RADIUS, 58 | label_style=ft.TextStyle(color=TEXT_COLOR), 59 | text_style=ft.TextStyle(color=TEXT_COLOR), 60 | bgcolor="transparent", 61 | ), 62 | divider=SimpleNamespace( 63 | height=10, 64 | color="transparent", 65 | ), 66 | ) 67 | -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .logging_config import get_logger, setup_logging 2 | from .gitignore_parser import GitIgnoreParser 3 | 4 | __all__ = ["get_logger", "setup_logging", "GitIgnoreParser"] 5 | -------------------------------------------------------------------------------- /src/utils/gitignore_parser.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import logging 3 | from pathlib import Path 4 | from typing import List 5 | 6 | import aiofiles 7 | 8 | 9 | class GitIgnoreParser: 10 | def __init__(self, base_dir: Path): 11 | self.base_dir = base_dir 12 | self.ignore_patterns = [] 13 | logging.basicConfig(level=logging.DEBUG) 14 | 15 | async def initialize(self): 16 | self.ignore_patterns = await self._parse_gitignore() 17 | 18 | async def _parse_gitignore(self) -> List[str]: 19 | gitignore_path = self.base_dir / ".gitignore" 20 | if not gitignore_path.exists(): 21 | logging.warning(f"No .gitignore file found in {self.base_dir}") 22 | return [] 23 | 24 | try: 25 | async with aiofiles.open(gitignore_path, "r") as f: 26 | content = await f.read() 27 | patterns = [ 28 | line.strip() 29 | for line in content.splitlines() 30 | if line.strip() and not line.startswith("#") 31 | ] 32 | logging.debug(f"Parsed {len(patterns)} patterns from .gitignore") 33 | return patterns 34 | except Exception as e: 35 | logging.error(f"Error reading .gitignore file: {str(e)}") 36 | return [] 37 | 38 | async def is_ignored(self, path: Path) -> bool: 39 | relative_path = path.relative_to(self.base_dir) 40 | str_path = str(relative_path).replace("\\", "/") # Normalize path separators 41 | 42 | for pattern in self.ignore_patterns: 43 | if self._match_pattern(str_path, pattern): 44 | logging.debug(f"File {path} matched pattern {pattern}") 45 | return True 46 | 47 | logging.debug(f"File {path} is not ignored") 48 | return False 49 | 50 | def _match_pattern(self, path: str, pattern: str) -> bool: 51 | if pattern.startswith("*"): 52 | return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch( 53 | path.split("/")[-1], pattern 54 | ) 55 | elif pattern.startswith("/"): 56 | return fnmatch.fnmatch(path, pattern[1:]) or fnmatch.fnmatch( 57 | path, "*/" + pattern[1:] 58 | ) 59 | elif pattern.endswith("/"): 60 | return fnmatch.fnmatch(path + "/", pattern) or fnmatch.fnmatch( 61 | path, pattern + "*" 62 | ) 63 | else: 64 | return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch( 65 | path, "*/" + pattern 66 | ) 67 | -------------------------------------------------------------------------------- /src/utils/logging_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | from rich.logging import RichHandler 4 | from rich.console import Console 5 | from rich.theme import Theme 6 | 7 | 8 | def setup_logging(log_file: Path = Path(".log"), console_level=logging.INFO): 9 | """ 10 | Set up a centralized logging system. 11 | 12 | :param log_file: Log file path 13 | :param console_level: Logging level for console output 14 | """ 15 | # Rich console configuration 16 | console = Console(theme=Theme({"logging.level": "bold"})) 17 | 18 | # Log format configuration 19 | log_format = "%(message)s" 20 | date_format = "[%X]" 21 | 22 | # Rich handler configuration 23 | rich_handler = RichHandler( 24 | console=console, 25 | enable_link_path=False, 26 | markup=True, 27 | rich_tracebacks=True, 28 | tracebacks_show_locals=True, 29 | level=console_level, # Set console logging level 30 | ) 31 | 32 | # File handler configuration 33 | file_handler = logging.FileHandler(log_file) 34 | file_handler.setFormatter( 35 | logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 36 | ) 37 | file_handler.setLevel(logging.DEBUG) # Keep all logs in the file 38 | 39 | # Root logger configuration 40 | logging.basicConfig( 41 | level=logging.DEBUG, # Keep root logger at DEBUG to capture all logs 42 | format=log_format, 43 | datefmt=date_format, 44 | handlers=[rich_handler, file_handler], 45 | ) 46 | 47 | 48 | def get_logger(name: str) -> logging.Logger: 49 | """ 50 | Returns a logger with the specified name. 51 | 52 | :param name: Logger name 53 | :return: Logger instance set up 54 | """ 55 | return logging.getLogger(name) 56 | --------------------------------------------------------------------------------