├── .github └── workflows │ └── main.yml ├── .gitignore ├── README.md ├── build └── lib │ └── fastsymapi │ ├── __init__.py │ ├── logging.py │ ├── sql_db │ ├── __init__.py │ ├── crud.py │ ├── database.py │ └── models.py │ └── symbols.py ├── fastsymapi ├── __init__.py ├── logging.py ├── sql_db │ ├── __init__.py │ ├── crud.py │ ├── database.py │ └── models.py └── symbols.py ├── fastsymapi_tests.py ├── requirements.txt └── setup.py /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: FastSymApi CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - "**" 7 | pull_request: 8 | branches: 9 | - main 10 | types: 11 | - closed 12 | 13 | permissions: 14 | contents: write # Grant write permissions to the contents 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python 3.12 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: 3.12 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | pip install -r requirements.txt 29 | - name: Run tests 30 | run: | 31 | pytest fastsymapi_tests.py 32 | - name: Build project 33 | run: | 34 | python setup.py bdist_wheel 35 | - name: Start Application 36 | run: | 37 | nohup uvicorn fastsymapi:app --host 0.0.0.0 --port 8000 & 38 | timeout-minutes: 1 39 | - name: Health Check 40 | run: | 41 | for i in {1..10}; do 42 | if curl -s http://localhost:8000/health | grep "ok"; then 43 | echo "Application started successfully" 44 | exit 0 45 | fi 46 | sleep 3 47 | done 48 | echo "Application failed to start" 49 | exit 1 50 | release: 51 | if: github.ref == 'refs/heads/main' 52 | needs: build 53 | runs-on: ubuntu-latest 54 | steps: 55 | - uses: actions/checkout@v2 56 | - name: Set up Python 3.12 57 | uses: actions/setup-python@v2 58 | with: 59 | python-version: 3.12 60 | - name: Install dependencies 61 | run: | 62 | python -m pip install --upgrade pip 63 | pip install -r requirements.txt 64 | - name: Build project 65 | run: | 66 | python setup.py bdist_wheel 67 | - name: Extract version 68 | id: extract_version 69 | run: | 70 | echo "VERSION=$(python setup.py --version)" >> $GITHUB_ENV 71 | - name: Check if Tag Exists 72 | id: check_tag 73 | run: | 74 | if git rev-parse "v${{ env.VERSION }}" >/dev/null 2>&1; then 75 | echo "TAG_EXISTS=true" >> $GITHUB_ENV 76 | else 77 | echo "TAG_EXISTS=false" >> $GITHUB_ENV 78 | fi 79 | - name: Create Tag 80 | if: ${{ env.TAG_EXISTS == false }} 81 | continue-on-error: true 82 | run: | 83 | git config --global user.email "30161177+P1tt1cus@users.noreply.github.com" 84 | git config --global user.name "P1tt1cus" 85 | git tag v${{ env.VERSION }} 86 | git push origin v${{ env.VERSION }} 87 | - name: Create GitHub Release 88 | if: ${{ env.TAG_EXISTS == false }} 89 | id: create_release 90 | uses: actions/create-release@v1 91 | env: 92 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 93 | with: 94 | tag_name: v${{ env.VERSION }} 95 | release_name: Release v${{ env.VERSION }} 96 | draft: false 97 | prerelease: false 98 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .fsymapi/** 2 | *.pdb 3 | *.dll 4 | *.pd_ 5 | *.pyc 6 | *.db 7 | .fsa 8 | .fastsymapi 9 | .fapi 10 | fastsymapi.egg-info 11 | dist 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastSymApi 2 | 3 | The FastSymApi server is a Fast API server designed for debugging and development environments. It allows users to download and cache symbols from Microsoft, Google, and Mozilla symbol servers. Additionally, users can easily add support for other symbol servers. 4 | 5 | When clients connect to FastSymApi and attempt to download a symbol, the server first checks if the symbol exists within its `./fastsymapi/symbols` cache. If found, the server returns the symbol; otherwise, it responds with a status `404` and proceeds to download the symbol. On subsequent requests, if the symbol is already downloaded and cached, the server returns it, either compressed using GZIP or decompressed based on the presence of the Accept-Encoding: gzip header. GZIP compression reduces bandwidth usage and improves download speed for clients. 6 | 7 | FastSymApi has been tested and works with the following tools: 8 | 9 | - x64dbg 10 | - WinDbg 11 | - Symchk 12 | 13 | Supports the following symbol servers: 14 | 15 | - 16 | - 17 | - 18 | 19 | ## Setup FastSymApi 20 | 21 | Clone the repository 22 | 23 | ``` 24 | git clone https://github.com/P1tt1cus/FastSymApi 25 | ``` 26 | 27 | Install the requirements 28 | 29 | ``` 30 | pip install requirements.txt 31 | ``` 32 | 33 | Start the server 34 | 35 | ``` 36 | uvicorn fastsymapi:app --host 0.0.0.0 --port 80 37 | ``` 38 | 39 | Debug Mode 40 | 41 | ``` 42 | uvicorn fastsymapi:app --reload 43 | ``` 44 | 45 | ## Run Tests 46 | 47 | ``` 48 | pytest fastsymapi_tests.py 49 | ``` 50 | 51 | ## Configure x64dbg 52 | 53 | **options** >> **preferences** >> **misc** 54 | 55 | Symbol store 56 | 57 | ``` 58 | http://FastSymApiServerIp/ 59 | ``` 60 | 61 | ## Configure WinDbg 62 | 63 | ``` 64 | .sympath srv*C:\symbols*http://FastSymApiServerIp/ 65 | ``` 66 | -------------------------------------------------------------------------------- /build/lib/fastsymapi/__init__.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from fastsymapi.sql_db import models 3 | from fastsymapi.logging import logger 4 | from fastsymapi.sql_db.database import engine 5 | from fastsymapi.symbols import sym 6 | 7 | 8 | def create_app(): 9 | """ Create the application context """ 10 | 11 | # Create the database tables 12 | models.base.metadata.create_all(bind=engine) 13 | 14 | # instantiate FastAPI 15 | app = FastAPI() 16 | 17 | # Symbol API 18 | app.include_router(sym) 19 | 20 | logger.info("Starting FastSymApi server...") 21 | 22 | return app 23 | 24 | 25 | app = create_app() 26 | -------------------------------------------------------------------------------- /build/lib/fastsymapi/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger("uvicorn") 4 | 5 | logger.setLevel("DEBUG") -------------------------------------------------------------------------------- /build/lib/fastsymapi/sql_db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/P1tt1cus/FastSymApi/bb4242c0b081afa34766ba7be5d313b2349cd539/build/lib/fastsymapi/sql_db/__init__.py -------------------------------------------------------------------------------- /build/lib/fastsymapi/sql_db/crud.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from fastsymapi.sql_db import models 3 | 4 | def find_pdb_entry(db: Session, guid: str, pdbfile: str): 5 | """ Find a PDB entry """ 6 | return db.query(models.SymbolEntry).filter(models.SymbolEntry.guid == guid, 7 | models.SymbolEntry.pdbfile == pdbfile).first() 8 | 9 | def find_still_downloading(db: Session): 10 | """ Return all still downloading PDB entries """ 11 | return db.query(models.SymbolEntry).filter(models.SymbolEntry.downloading == True).all() 12 | 13 | def create_pdb_entry(db: Session, guid: str, pdbname: str, pdbfile: str, found: bool = False): 14 | """ Create a new PDB entry """ 15 | pdb_entry = models.SymbolEntry(pdbname=pdbname, guid=guid, pdbfile=pdbfile, found=found) 16 | db.add(pdb_entry) 17 | db.commit() 18 | db.refresh(pdb_entry) 19 | return pdb_entry 20 | 21 | def modify_pdb_entry(db: Session, pdbentry: models.SymbolEntry) -> None: 22 | """ Modify a PDB entry """ 23 | db.add(pdbentry) 24 | db.commit() 25 | db.refresh(pdbentry) -------------------------------------------------------------------------------- /build/lib/fastsymapi/sql_db/database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | SQLALCHEMY_DATABASE_URL = "sqlite:///./fsymapi.db" 6 | 7 | engine = create_engine( 8 | SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread":False}, 9 | pool_size=20, 10 | max_overflow=30 11 | ) 12 | 13 | session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine) 14 | 15 | base = declarative_base() 16 | 17 | def get_db() -> sessionmaker: 18 | db = session_local() 19 | try: 20 | yield db 21 | finally: 22 | db.close() -------------------------------------------------------------------------------- /build/lib/fastsymapi/sql_db/models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Integer, String, Boolean, UniqueConstraint 2 | from fastsymapi.sql_db.database import base 3 | 4 | class SymbolEntry(base): 5 | __tablename__ = "symbolentry" 6 | id = Column(Integer, primary_key=True, index=True, unique=True) 7 | guid = Column(String, index=True) 8 | pdbname = Column(String, index=True) 9 | pdbfile = Column(String, index=True) 10 | downloading = Column(Boolean, index=True, default=False) 11 | found = Column(Boolean, index=True, default=False) 12 | 13 | # Adds a unique constraint on the guid, pdbfile 14 | __table_args__ = (UniqueConstraint('guid', 'pdbfile'),) 15 | 16 | -------------------------------------------------------------------------------- /build/lib/fastsymapi/symbols.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, Response, BackgroundTasks, Request 2 | from fastapi.responses import FileResponse, StreamingResponse 3 | from fastapi.encoders import jsonable_encoder 4 | from fastsymapi.sql_db.database import get_db, session_local 5 | from fastsymapi.sql_db import crud, models 6 | from fastsymapi.logging import logger 7 | import requests 8 | import click 9 | from sqlalchemy.orm import Session 10 | import os 11 | import shutil 12 | import gzip 13 | 14 | sym = APIRouter() 15 | 16 | CHUNK_SIZE = 1024*1024*2 17 | 18 | SYMBOL_PATH = os.path.join(os.path.dirname(__file__), "symbols") 19 | 20 | SYM_URLS = [ 21 | "http://msdl.microsoft.com/download/symbols", 22 | "http://chromium-browser-symsrv.commondatastorage.googleapis.com", 23 | "http://symbols.mozilla.org", 24 | "http://symbols.mozilla.org/try" 25 | ] 26 | 27 | 28 | def download_symbol(pdbentry: models.SymbolEntry, db: Session) -> None: 29 | """ Iterate over SYM_URLs looking for the requested PDB file """ 30 | 31 | # Iterate over the symbol server URLs 32 | for sym_url in SYM_URLS: 33 | 34 | # Check if symbol exists on the server 35 | symbol_url = sym_url + \ 36 | f"/{pdbentry.pdbname}/{pdbentry.guid}/{pdbentry.pdbfile}" 37 | resp = requests.get(symbol_url, stream=True) 38 | 39 | # If the symbol was found download it 40 | if resp.status_code == 200: 41 | pdbentry.found = True 42 | download_and_save_symbol(pdbentry, resp, db) 43 | break 44 | 45 | # Unable to find PDB at any of the Symbol Servers 46 | else: 47 | logger.debug(f"Could not find symbol: { 48 | symbol_url} {resp.status_code}") 49 | 50 | # Set the PDB entry to 'finished' downloading 51 | pdbentry.downloading = False 52 | crud.modify_pdb_entry(db, pdbentry) 53 | 54 | 55 | def download_and_save_symbol(pdbentry, resp, db): 56 | """ Download the symbol and save it to disk """ 57 | 58 | # Notify that the download is beginning 59 | logger.warning(f"Downloading... {pdbentry.guid} {pdbentry.pdbfile}") 60 | 61 | # Create the PDB directory with GUID if it does not exist 62 | pdb_file_path = os.path.join(SYMBOL_PATH, pdbentry.pdbname, pdbentry.guid) 63 | if not os.path.exists(pdb_file_path): 64 | os.makedirs(pdb_file_path) 65 | 66 | # Logic that identifies whether its a gzip or not 67 | content_encoding = resp.headers.get("Content-Encoding", "") 68 | is_gzip_supported = "gzip" in content_encoding.lower() 69 | 70 | # Create the PDB file and iterate over it writing the chunks 71 | pdb_tmp_file_path = os.path.join( 72 | pdb_file_path, "tmp_"+pdbentry.pdbfile+".gzip") 73 | 74 | # if the file is already compressed, just write the raw bytes 75 | if is_gzip_supported: 76 | pdbfile_handle = open(pdb_tmp_file_path, 'wb') 77 | # else, we must compress it ourselves 78 | else: 79 | pdbfile_handle = gzip.open(pdb_tmp_file_path, 'wb') 80 | 81 | # Get the size of the PDB buffer being downloaded 82 | pdb_size = get_pdb_size(resp) 83 | if pdb_size is None: 84 | pdbentry.downloading = False 85 | crud.modify_pdb_entry(db, pdbentry) 86 | return 87 | 88 | # Dpwnload percentage calculation 89 | downloaded = 0 90 | percent = 0 91 | last_logged_percent = -1 92 | while downloaded < pdb_size: 93 | remaining = pdb_size - downloaded 94 | chunk = resp.raw.read(min(CHUNK_SIZE, remaining)) 95 | pdbfile_handle.write(chunk) 96 | downloaded += len(chunk) 97 | percent = int((downloaded / pdb_size) * 100) 98 | if percent // 5 > last_logged_percent: # Log every 5% 99 | last_logged_percent = percent // 5 100 | logger.warning(f"Downloading... {pdbentry.guid} { 101 | pdbentry.pdbfile} {percent}%") 102 | 103 | # Close the file handle 104 | pdbfile_handle.close() 105 | 106 | # Finished downloading PDB 107 | logger.info(f"Successfully downloaded... { 108 | pdbentry.guid} {pdbentry.pdbfile}") 109 | 110 | # If the file is already a gzip, there is no need to compress it 111 | pdb_file_path = os.path.join(pdb_file_path, pdbentry.pdbfile+".gzip") 112 | shutil.move(pdb_tmp_file_path, pdb_file_path) 113 | 114 | 115 | def get_pdb_size(resp): 116 | """ Get the size of the PDB buffer being downloaded """ 117 | 118 | for header in ["Content-Length", "x-goog-stored-content-length"]: 119 | if resp.headers.get(header): 120 | return int(resp.headers[header]) 121 | 122 | # Output an error stating the content-length could not be found. 123 | logger.error(f"Could not get content length from server: { 124 | resp.status_code}") 125 | return None 126 | 127 | 128 | def get_symbol(pdbname: str, pdbfile: str, guid: str, background_tasks: BackgroundTasks, db: Session, is_gzip_supported: bool): 129 | pdb_file_path = os.path.join(SYMBOL_PATH, pdbname, guid, pdbfile+".gzip") 130 | 131 | if not os.path.isfile(pdb_file_path): 132 | pdbentry = crud.find_pdb_entry(db, guid, pdbfile) 133 | if not pdbentry: 134 | pdbentry = crud.create_pdb_entry(db, guid, pdbname, pdbfile) 135 | if pdbentry.downloading: 136 | return Response(status_code=404) 137 | pdbentry.downloading = True 138 | crud.modify_pdb_entry(db, pdbentry) 139 | background_tasks.add_task(download_symbol, pdbentry, db) 140 | return Response(status_code=404) 141 | 142 | pdbentry = crud.find_pdb_entry(db, guid, pdbfile) 143 | if not pdbentry: 144 | pdbentry = crud.create_pdb_entry(db, guid, pdbname, pdbfile, True) 145 | 146 | if is_gzip_supported: 147 | logger.debug("Returning gzip compressed stream...") 148 | return FileResponse(pdb_file_path, headers={"content-encoding": "gzip"}, media_type="application/octet-stream") 149 | 150 | def stream_decompressed_data(chunk_size=CHUNK_SIZE): 151 | with gzip.open(pdb_file_path, 'rb') as gzip_file: 152 | while True: 153 | chunk = gzip_file.read(chunk_size) 154 | if not chunk: 155 | break 156 | yield chunk 157 | 158 | logger.debug("Returning decompressed stream...") 159 | return StreamingResponse(stream_decompressed_data(), media_type="application/octet-stream") 160 | 161 | 162 | @sym.get("/{pdbname}/{guid}/{pdbfile}") 163 | @sym.get("/download/symbols/{pdbname}/{guid}/{pdbfile}") 164 | async def get_symbol_api(pdbname: str, guid: str, pdbfile: str, request: Request, background_tasks: BackgroundTasks, db: Session = Depends(get_db)): 165 | accept_encoding = request.headers.get("Accept-Encoding", "") 166 | is_gzip_supported = "gzip" in accept_encoding.lower() 167 | return get_symbol(pdbname, pdbfile, guid, background_tasks, db, is_gzip_supported) 168 | 169 | 170 | @sym.get("/symbols") 171 | def get_symbol_entries(db: Session = Depends(get_db)) -> list: 172 | return jsonable_encoder(db.query(models.SymbolEntry).all()) 173 | 174 | 175 | @sym.on_event("startup") 176 | def fastsym_init(): 177 | db = session_local() 178 | downloads = crud.find_still_downloading(db) 179 | for download in downloads: 180 | failed_tmp_download = os.path.join( 181 | SYMBOL_PATH, download.pdbname, download.guid, "tmp_"+download.pdbfile+".gzip") 182 | if os.path.exists(failed_tmp_download): 183 | os.remove(failed_tmp_download) 184 | download.downloading = False 185 | crud.modify_pdb_entry(db, download) 186 | -------------------------------------------------------------------------------- /fastsymapi/__init__.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from fastsymapi.sql_db import models 3 | from fastsymapi.logging import logger 4 | from fastsymapi.sql_db.database import engine 5 | from fastsymapi.symbols import sym 6 | 7 | 8 | def create_app(): 9 | """ Create the application context """ 10 | 11 | # Create the database tables 12 | models.base.metadata.create_all(bind=engine) 13 | 14 | # instantiate FastAPI 15 | app = FastAPI() 16 | 17 | # Symbol API 18 | app.include_router(sym) 19 | 20 | logger.info("Starting FastSymApi server...") 21 | 22 | return app 23 | 24 | 25 | app = create_app() 26 | 27 | 28 | @app.get("/health") 29 | def health_check(): 30 | return {"status": "ok"} 31 | -------------------------------------------------------------------------------- /fastsymapi/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger("uvicorn") 4 | 5 | logger.setLevel("DEBUG") -------------------------------------------------------------------------------- /fastsymapi/sql_db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/P1tt1cus/FastSymApi/bb4242c0b081afa34766ba7be5d313b2349cd539/fastsymapi/sql_db/__init__.py -------------------------------------------------------------------------------- /fastsymapi/sql_db/crud.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from fastsymapi.sql_db import models 3 | 4 | def find_pdb_entry(db: Session, guid: str, pdbfile: str): 5 | """ Find a PDB entry """ 6 | return db.query(models.SymbolEntry).filter(models.SymbolEntry.guid == guid, 7 | models.SymbolEntry.pdbfile == pdbfile).first() 8 | 9 | def find_still_downloading(db: Session): 10 | """ Return all still downloading PDB entries """ 11 | return db.query(models.SymbolEntry).filter(models.SymbolEntry.downloading == True).all() 12 | 13 | def create_pdb_entry(db: Session, guid: str, pdbname: str, pdbfile: str, found: bool = False): 14 | """ Create a new PDB entry """ 15 | pdb_entry = models.SymbolEntry(pdbname=pdbname, guid=guid, pdbfile=pdbfile, found=found) 16 | db.add(pdb_entry) 17 | db.commit() 18 | db.refresh(pdb_entry) 19 | return pdb_entry 20 | 21 | def modify_pdb_entry(db: Session, pdbentry: models.SymbolEntry) -> None: 22 | """ Modify a PDB entry """ 23 | db.add(pdbentry) 24 | db.commit() 25 | db.refresh(pdbentry) -------------------------------------------------------------------------------- /fastsymapi/sql_db/database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.ext.declarative import declarative_base 3 | from sqlalchemy.orm import sessionmaker 4 | 5 | SQLALCHEMY_DATABASE_URL = "sqlite:///./fsymapi.db" 6 | 7 | engine = create_engine( 8 | SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread":False}, 9 | pool_size=20, 10 | max_overflow=30 11 | ) 12 | 13 | session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine) 14 | 15 | base = declarative_base() 16 | 17 | def get_db() -> sessionmaker: 18 | db = session_local() 19 | try: 20 | yield db 21 | finally: 22 | db.close() -------------------------------------------------------------------------------- /fastsymapi/sql_db/models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Integer, String, Boolean, UniqueConstraint 2 | from fastsymapi.sql_db.database import base 3 | 4 | class SymbolEntry(base): 5 | __tablename__ = "symbolentry" 6 | id = Column(Integer, primary_key=True, index=True, unique=True) 7 | guid = Column(String, index=True) 8 | pdbname = Column(String, index=True) 9 | pdbfile = Column(String, index=True) 10 | downloading = Column(Boolean, index=True, default=False) 11 | found = Column(Boolean, index=True, default=False) 12 | 13 | # Adds a unique constraint on the guid, pdbfile 14 | __table_args__ = (UniqueConstraint('guid', 'pdbfile'),) 15 | 16 | -------------------------------------------------------------------------------- /fastsymapi/symbols.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, Response, BackgroundTasks, Request 2 | from fastapi.responses import FileResponse, StreamingResponse 3 | from fastapi.encoders import jsonable_encoder 4 | from fastsymapi.sql_db.database import get_db, session_local 5 | from fastsymapi.sql_db import crud, models 6 | from fastsymapi.logging import logger 7 | import requests 8 | import click 9 | from sqlalchemy.orm import Session 10 | import os 11 | import shutil 12 | import gzip 13 | 14 | sym = APIRouter() 15 | 16 | CHUNK_SIZE = 1024*1024*2 17 | 18 | SYMBOL_PATH = os.path.join(os.path.dirname(__file__), "symbols") 19 | 20 | SYM_URLS = [ 21 | "http://msdl.microsoft.com/download/symbols", 22 | "http://chromium-browser-symsrv.commondatastorage.googleapis.com", 23 | "http://symbols.mozilla.org", 24 | "http://symbols.mozilla.org/try" 25 | ] 26 | 27 | 28 | def download_symbol(pdbentry: models.SymbolEntry, db: Session) -> None: 29 | """ Iterate over SYM_URLs looking for the requested PDB file """ 30 | 31 | # Iterate over the symbol server URLs 32 | for sym_url in SYM_URLS: 33 | 34 | # Check if symbol exists on the server 35 | symbol_url = sym_url + \ 36 | f"/{pdbentry.pdbname}/{pdbentry.guid}/{pdbentry.pdbfile}" 37 | resp = requests.get(symbol_url, stream=True) 38 | 39 | # If the symbol was found download it 40 | if resp.status_code == 200: 41 | pdbentry.found = True 42 | download_and_save_symbol(pdbentry, resp, db) 43 | break 44 | 45 | # Unable to find PDB at any of the Symbol Servers 46 | else: 47 | logger.debug(f"Could not find symbol: { 48 | symbol_url} {resp.status_code}") 49 | 50 | # Set the PDB entry to 'finished' downloading 51 | pdbentry.downloading = False 52 | crud.modify_pdb_entry(db, pdbentry) 53 | 54 | 55 | def download_and_save_symbol(pdbentry, resp, db): 56 | """ Download the symbol and save it to disk """ 57 | 58 | # Notify that the download is beginning 59 | logger.warning(f"Downloading... {pdbentry.guid} {pdbentry.pdbfile}") 60 | 61 | # Create the PDB directory with GUID if it does not exist 62 | pdb_file_path = os.path.join(SYMBOL_PATH, pdbentry.pdbname, pdbentry.guid) 63 | if not os.path.exists(pdb_file_path): 64 | os.makedirs(pdb_file_path) 65 | 66 | # Logic that identifies whether its a gzip or not 67 | content_encoding = resp.headers.get("Content-Encoding", "") 68 | is_gzip_supported = "gzip" in content_encoding.lower() 69 | 70 | # Create the PDB file and iterate over it writing the chunks 71 | pdb_tmp_file_path = os.path.join( 72 | pdb_file_path, "tmp_"+pdbentry.pdbfile+".gzip") 73 | 74 | # if the file is already compressed, just write the raw bytes 75 | if is_gzip_supported: 76 | pdbfile_handle = open(pdb_tmp_file_path, 'wb') 77 | # else, we must compress it ourselves 78 | else: 79 | pdbfile_handle = gzip.open(pdb_tmp_file_path, 'wb') 80 | 81 | # Get the size of the PDB buffer being downloaded 82 | pdb_size = get_pdb_size(resp) 83 | if pdb_size is None: 84 | pdbentry.downloading = False 85 | crud.modify_pdb_entry(db, pdbentry) 86 | return 87 | 88 | # Dpwnload percentage calculation 89 | downloaded = 0 90 | percent = 0 91 | last_logged_percent = -1 92 | while downloaded < pdb_size: 93 | remaining = pdb_size - downloaded 94 | chunk = resp.raw.read(min(CHUNK_SIZE, remaining)) 95 | pdbfile_handle.write(chunk) 96 | downloaded += len(chunk) 97 | percent = int((downloaded / pdb_size) * 100) 98 | if percent // 5 > last_logged_percent: # Log every 5% 99 | last_logged_percent = percent // 5 100 | logger.warning(f"Downloading... {pdbentry.guid} { 101 | pdbentry.pdbfile} {percent}%") 102 | 103 | # Close the file handle 104 | pdbfile_handle.close() 105 | 106 | # Finished downloading PDB 107 | logger.info(f"Successfully downloaded... { 108 | pdbentry.guid} {pdbentry.pdbfile}") 109 | 110 | # If the file is already a gzip, there is no need to compress it 111 | pdb_file_path = os.path.join(pdb_file_path, pdbentry.pdbfile+".gzip") 112 | shutil.move(pdb_tmp_file_path, pdb_file_path) 113 | 114 | 115 | def get_pdb_size(resp): 116 | """ Get the size of the PDB buffer being downloaded """ 117 | 118 | for header in ["Content-Length", "x-goog-stored-content-length"]: 119 | if resp.headers.get(header): 120 | return int(resp.headers[header]) 121 | 122 | # Output an error stating the content-length could not be found. 123 | logger.error(f"Could not get content length from server: { 124 | resp.status_code}") 125 | return None 126 | 127 | 128 | def get_symbol(pdbname: str, pdbfile: str, guid: str, background_tasks: BackgroundTasks, db: Session, is_gzip_supported: bool): 129 | pdb_file_path = os.path.join(SYMBOL_PATH, pdbname, guid, pdbfile+".gzip") 130 | 131 | if not os.path.isfile(pdb_file_path): 132 | pdbentry = crud.find_pdb_entry(db, guid, pdbfile) 133 | if not pdbentry: 134 | pdbentry = crud.create_pdb_entry(db, guid, pdbname, pdbfile) 135 | if pdbentry.downloading: 136 | return Response(status_code=404) 137 | pdbentry.downloading = True 138 | crud.modify_pdb_entry(db, pdbentry) 139 | background_tasks.add_task(download_symbol, pdbentry, db) 140 | return Response(status_code=404) 141 | 142 | pdbentry = crud.find_pdb_entry(db, guid, pdbfile) 143 | if not pdbentry: 144 | pdbentry = crud.create_pdb_entry(db, guid, pdbname, pdbfile, True) 145 | 146 | if is_gzip_supported: 147 | logger.debug("Returning gzip compressed stream...") 148 | return FileResponse(pdb_file_path, headers={"content-encoding": "gzip"}, media_type="application/octet-stream") 149 | 150 | def stream_decompressed_data(chunk_size=CHUNK_SIZE): 151 | with gzip.open(pdb_file_path, 'rb') as gzip_file: 152 | while True: 153 | chunk = gzip_file.read(chunk_size) 154 | if not chunk: 155 | break 156 | yield chunk 157 | 158 | logger.debug("Returning decompressed stream...") 159 | return StreamingResponse(stream_decompressed_data(), media_type="application/octet-stream") 160 | 161 | 162 | @sym.get("/{pdbname}/{guid}/{pdbfile}") 163 | @sym.get("/download/symbols/{pdbname}/{guid}/{pdbfile}") 164 | async def get_symbol_api(pdbname: str, guid: str, pdbfile: str, request: Request, background_tasks: BackgroundTasks, db: Session = Depends(get_db)): 165 | accept_encoding = request.headers.get("Accept-Encoding", "") 166 | is_gzip_supported = "gzip" in accept_encoding.lower() 167 | return get_symbol(pdbname, pdbfile, guid, background_tasks, db, is_gzip_supported) 168 | 169 | 170 | @sym.get("/symbols") 171 | def get_symbol_entries(db: Session = Depends(get_db)) -> list: 172 | return jsonable_encoder(db.query(models.SymbolEntry).all()) 173 | 174 | 175 | @sym.on_event("startup") 176 | def fastsym_init(): 177 | db = session_local() 178 | downloads = crud.find_still_downloading(db) 179 | for download in downloads: 180 | failed_tmp_download = os.path.join( 181 | SYMBOL_PATH, download.pdbname, download.guid, "tmp_"+download.pdbfile+".gzip") 182 | if os.path.exists(failed_tmp_download): 183 | os.remove(failed_tmp_download) 184 | download.downloading = False 185 | crud.modify_pdb_entry(db, download) 186 | -------------------------------------------------------------------------------- /fastsymapi_tests.py: -------------------------------------------------------------------------------- 1 | from fastsymapi.symbols import download_symbol 2 | from fastsymapi.sql_db import models 3 | from unittest.mock import patch, MagicMock 4 | from fastapi.testclient import TestClient 5 | from fastsymapi import app 6 | import pytest 7 | 8 | client = TestClient(app) 9 | 10 | 11 | @pytest.fixture 12 | def mock_gzip_open(): 13 | return MagicMock() 14 | 15 | 16 | def test_fail_get_symbol_api(): 17 | """Test a failed symbol retrieval""" 18 | 19 | response = client.get("/download/symbols/notreal/notreal/pdbfile") 20 | 21 | assert response.status_code == 404 # or whatever status code you expect 22 | 23 | 24 | @patch("gzip.open") 25 | @patch("fastsymapi.symbols.requests.get") 26 | @patch("fastsymapi.symbols.crud.modify_pdb_entry") 27 | @patch("fastsymapi.symbols.crud.create_pdb_entry") 28 | @patch("fastsymapi.symbols.os.path.exists") 29 | @patch("fastsymapi.symbols.os.makedirs") 30 | @patch("fastsymapi.symbols.open", new_callable=MagicMock) 31 | @patch("fastsymapi.symbols.shutil.move") 32 | def test_successful_pdb_download( 33 | self, 34 | mock_move, 35 | mock_open, 36 | mock_makedirs, 37 | mock_exists, 38 | mock_create_pdb_entry, 39 | mock_modify_pdb_entry, 40 | mock_get, 41 | mock_gzip_open, 42 | ): 43 | # Arrange 44 | mock_response = MagicMock() 45 | mock_response.status_code = 200 46 | mock_get.return_value = mock_response 47 | mock_exists.return_value = False 48 | mock_open.return_value.__enter__.return_value = MagicMock() 49 | mock_gzip_open.return_value.__enter__.return_value = MagicMock() 50 | pdbentry = models.SymbolEntry(pdbname="test", guid="test", pdbfile="test") 51 | db = MagicMock() 52 | 53 | # Act 54 | download_symbol(pdbentry, db) 55 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.7.0 2 | anyio==4.6.0 3 | certifi==2024.8.30 4 | charset-normalizer==3.4.0 5 | click==8.1.7 6 | colorama==0.4.6 7 | construct==2.10.70 8 | exceptiongroup==1.2.2 9 | fastapi==0.100.0 10 | greenlet==3.1.1 11 | h11==0.14.0 12 | httpcore==1.0.6 13 | httpx==0.27.2 14 | idna==3.4 15 | iniconfig==2.0.0 16 | packaging==24.1 17 | pip-review==1.3.0 18 | pipdeptree==2.23.4 19 | pluggy==1.5.0 20 | pydantic==2.1.1 21 | pydantic_core==2.4.0 22 | pytest==8.3.3 23 | requests==2.31.0 24 | setuptools==75.1.0 25 | sniffio==1.3.0 26 | SQLAlchemy==2.0.19 27 | starlette==0.27.0 28 | typing_extensions==4.7.1 29 | urllib3==2.2.3 30 | uvicorn==0.31.1 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='fastsymapi', 5 | version='1.2', 6 | packages=find_packages() 7 | ) 8 | --------------------------------------------------------------------------------