├── .github └── workflows │ └── python-symbols-publish.yaml ├── .gitignore ├── CNAME ├── README.rst ├── generateindex.py ├── pysymsrv.py └── requirements.txt /.github/workflows/python-symbols-publish.yaml: -------------------------------------------------------------------------------- 1 | name: PythonSymbols 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | schedule: 11 | - cron: "0 0 * * 0" 12 | 13 | permissions: 14 | contents: write 15 | pages: write 16 | id-token: write 17 | 18 | jobs: 19 | build: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Checkout gh-pages branch. 23 | uses: actions/checkout@v3 24 | with: 25 | ref: gh-pages 26 | fetch-depth: 0 # Fetch all branches so we can merge. 27 | - name: Merge any recent changes into the gh-pages branch. 28 | run: | 29 | git config user.name github-actions 30 | git config user.email github-actions@github.com 31 | git merge -X theirs origin/main 32 | - name: Setup Python. 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: 3.x 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 40 | - name: Update symbol server. 41 | run: | 42 | python pysymsrv.py 43 | - name: Generate index.html. 44 | run: | 45 | python generateindex.py 46 | - name: Deploy new symbols to GH Pages. 47 | run: | 48 | # date > lastrun.txt 49 | git add --all :/ 50 | git diff-index --quiet HEAD || git commit -m "Automatic update of gh-pages" 51 | git push 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /*temp/ 2 | deploy-key 3 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | pythonsymbols.sdcline.com -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ========================== 2 | WinDbg symbols for CPython 3 | ========================== 4 | 5 | 6 | .. image:: https://img.shields.io/github/last-commit/SeanCline/PythonSymbols/gh-pages.svg?label=Symbol%20Server%20Updated 7 | :target: https://github.com/SeanCline/PythonSymbols/actions 8 | :alt: Build Status 9 | 10 | This repository hosts the symbols for all recent Windows builds of the CPython interpreter. (Both x86 and x64.) 11 | 12 | It stays up to date automatically by looking for new Python releases weekly and adding their symbols to the symbols store. 13 | 14 | To use the symbols server, add the following to your symbol path: 15 | 16 | .. code-block:: 17 | 18 | srv*c:\symbols*http://pythonsymbols.sdcline.com/symbols 19 | -------------------------------------------------------------------------------- /generateindex.py: -------------------------------------------------------------------------------- 1 | """Generates an index.html file for the repository.""" 2 | 3 | import docutils.core 4 | 5 | if __name__ == "__main__": 6 | docutils.core.publish_file(source_path ="README.rst", destination_path ="index.html", writer_name ="html") 7 | -------------------------------------------------------------------------------- /pysymsrv.py: -------------------------------------------------------------------------------- 1 | """Downloads all Python symbols and checks them into a symbol server.""" 2 | 3 | import sys, os, datetime, subprocess, logging, shutil, re, urllib.request, urllib.parse, json 4 | import symstore 5 | 6 | _logger = logging.getLogger(__name__) 7 | 8 | def fetch_page(url): 9 | """Fetchs a the contents at a URL as a string.""" 10 | _logger.info("Fetching %r...", url) 11 | with urllib.request.urlopen(url) as resp: 12 | return resp.read().decode(resp.headers.get_content_charset("utf-8")) 13 | 14 | 15 | def download_file(source_url, destination_filename): 16 | """Downloads a file at the provided url, to the destination on disk.""" 17 | _logger.info("Downloading %r to %r...", source_url, destination_filename) 18 | with urllib.request.urlopen(source_url) as resp, open(destination_filename, 'wb') as destination_file: 19 | shutil.copyfileobj(resp, destination_file) 20 | 21 | 22 | def try_download_file(source_url, destination_filename, already_downloaded_files): 23 | """Downloads a file at the provided url and adds it to the set of downloaded archives if it hasn't already been downloaded.""" 24 | # If we've already downloaded the file, then it's already in the symbol store. 25 | if source_url in already_downloaded_files: 26 | _logger.info("Skipping already downloaded file: %r", source_url) 27 | return 28 | 29 | try: 30 | download_file(source_url, destination_filename) 31 | already_downloaded_files.add(source_url) 32 | except urllib.error.HTTPError as err: 33 | _logger.info("Failed to download: %r", source_url) 34 | 35 | if err.code == 404: 36 | already_downloaded_files.add(source_url) # Mark file-not-found as downloaded. 37 | pass # Just keep going. 38 | 39 | 40 | def get_available_python_versions(root): 41 | """Returns a list of python versions available from python.org.""" 42 | indexhtml = fetch_page(root) 43 | return re.findall(r"", indexhtml) 44 | 45 | 46 | def read_downloaded_files_list(filename="downloaded.json"): 47 | """Reads the list of archives that have already been added to the store.""" 48 | try: 49 | with open(filename, "r") as fp: 50 | return json.load(fp) 51 | except FileNotFoundError: 52 | return [] 53 | 54 | 55 | def save_downloaded_files_list(files, filename="downloaded.json"): 56 | """Saves the list of archives we've already download to disk.""" 57 | filelist = list(files) 58 | filelist.sort() 59 | with open(filename, "w") as fp: 60 | json.dump(list(filelist), fp, indent=0) 61 | 62 | 63 | def download_pdbs_for_version(root, version, target_dir, already_downloaded_files): 64 | """Downloads the debugging symbols for a given version of Python.""" 65 | base_url = urllib.parse.urljoin(root, version + "/") 66 | if not os.path.exists(target_dir): 67 | os.makedirs(target_dir) 68 | os.makedirs(os.path.join(target_dir, "win32")) 69 | os.makedirs(os.path.join(target_dir, "amd64")) 70 | 71 | archives = [ 72 | "python-" + version + "-pdb.zip", # Python 2.x 73 | "python-" + version + ".amd64-pdb.zip", # Python 2.x 74 | "win32/core_pdb.msi", # Python 3.x 75 | "win32/exe_pdb.msi", # Python 3.x 76 | "win32/lib_pdb.msi", # Python 3.x 77 | "win32/tkltk_pdb.msi", # Python 3.x 78 | "win32/test_pdb.msi", # Python 3.x 79 | "amd64/core_pdb.msi", # Python 3.x 80 | "amd64/exe_pdb.msi", # Python 3.x 81 | "amd64/lib_pdb.msi", # Python 3.x 82 | "amd64/tkltk_pdb.msi", # Python 3.x 83 | "amd64/test_pdb.msi", # Python 3.x 84 | ] 85 | 86 | for archive in archives: 87 | source_url = urllib.parse.urljoin(base_url, archive) 88 | destination_filename = os.path.join(target_dir, archive) 89 | try_download_file(source_url, destination_filename, already_downloaded_files) 90 | 91 | 92 | def extract_archive(archive_filename): 93 | """Unzips zip files and MSIs.""" 94 | _logger.info("Extracting: %r...", archive_filename) 95 | absolute_path = os.path.abspath(archive_filename) 96 | targetdir = os.path.splitext(absolute_path)[0] 97 | subprocess.call(["7z", "x", "-y", absolute_path, "-o" + targetdir]) 98 | 99 | 100 | def extract_archives_in_direcory(path): 101 | """Recursively extracts all of the zipped/MSI'd files in a provided directory.""" 102 | for root, subdirs, files in os.walk(path): 103 | for file in files: 104 | file_path = os.path.join(root, file) 105 | extract_archive(file_path) 106 | 107 | 108 | def store_pdbs_in_directory(pdb_path, store_path, product, version): 109 | """Recursively adds all of the pdb files in a provided directory to the symbols store.""" 110 | sym_store = symstore.Store(store_path) 111 | now = datetime.datetime.now().isoformat() 112 | comment = f"{product} - {version} - {now}" 113 | transaction = sym_store.new_transaction(product, version, comment) 114 | num_files_stored = 0 115 | for root, subdirs, files in os.walk(pdb_path): 116 | for file in files: 117 | file_path = os.path.join(root, file) 118 | 119 | # Rename the python.pdb to python##.pdb. 120 | if "core_pdb" in root and file == "python.pdb": 121 | major, minor, *_ = version.split(".") 122 | new_file_path = file_path.replace("python.pdb", "python" + major + minor + ".pdb") 123 | os.rename(file_path, new_file_path) 124 | file_path = new_file_path 125 | 126 | # Add PDB files 127 | is_compression_supported = symstore.cab.compress is not None 128 | if file_path.endswith(".pdb"): 129 | entry = transaction.new_entry(file_path, compress=is_compression_supported) 130 | transaction.add_entry(entry) 131 | num_files_stored += 1 132 | 133 | if num_files_stored > 0: 134 | sym_store.commit(transaction) 135 | 136 | 137 | def fetch_and_store_pdbs_for_version(root, product, version, already_downloaded_files): 138 | temp_folder = version + "-temp/" 139 | store_folder = "./symbols" 140 | download_pdbs_for_version(root, version, temp_folder, already_downloaded_files) 141 | extract_archives_in_direcory(temp_folder) 142 | store_pdbs_in_directory(temp_folder, store_folder, product, version) 143 | shutil.rmtree(temp_folder) 144 | 145 | 146 | def download_pdbs_at_root(root, product, already_downloaded_files): 147 | available_versions = get_available_python_versions(root) 148 | _logger.info("Versions available from %s: %r", root, available_versions) 149 | 150 | for version in available_versions: 151 | fetch_and_store_pdbs_for_version(root, product, version, already_downloaded_files) 152 | 153 | 154 | def get_old_stackless_pdb_file_list(root): 155 | indexhtml = fetch_page(root) 156 | return re.findall(r"", indexhtml) 157 | 158 | 159 | def fetch_and_store_old_stackless_pdbs(root, already_downloaded_files): 160 | archives = get_old_stackless_pdb_file_list(root) 161 | 162 | store_folder = "./symbols" 163 | for archive in archives: 164 | temp_folder = archive + "-temp/" 165 | if not os.path.exists(temp_folder): 166 | os.makedirs(temp_folder) 167 | 168 | source_url = urllib.parse.urljoin(root, archive) 169 | destination_filename = os.path.join(temp_folder, archive) 170 | try_download_file(source_url, destination_filename, already_downloaded_files) 171 | extract_archives_in_direcory(temp_folder) 172 | store_pdbs_in_directory(temp_folder, store_folder, "StacklessPython", archive) 173 | shutil.rmtree(temp_folder) 174 | 175 | 176 | if __name__ == "__main__": 177 | logging.basicConfig(level=logging.INFO, stream=sys.stdout) 178 | 179 | already_downloaded_files = set(read_downloaded_files_list()); 180 | _logger.info("Files already downloaded: %r", already_downloaded_files) 181 | 182 | # Download official Python symbols. 183 | download_pdbs_at_root("https://www.python.org/ftp/python/", "CPython", already_downloaded_files) 184 | 185 | #Download Stackless Python symbols. (3.5+) 186 | download_pdbs_at_root("http://www.stackless.com/binaries/MSI/", "StacklessPython", already_downloaded_files) 187 | 188 | # Download Stackless Python symbols (Older versions.) 189 | fetch_and_store_old_stackless_pdbs("http://www.stackless.com/binaries/", already_downloaded_files) 190 | 191 | save_downloaded_files_list(already_downloaded_files) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | symstore~=0.3.2 2 | docutils~=0.17.0 3 | --------------------------------------------------------------------------------