├── .gitignore ├── LICENSE ├── README.md ├── gpt_scraper ├── __init__.py ├── cli.py ├── gpt_scraper.py ├── html_utils.py ├── prompts.py ├── sandbox.py ├── selenium_utils.py └── utils.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GPT-Scraper 2 | 3 | GPT-Scraper is an autonomous, LLM-based agent that generates code to extract structured information from web pages. 4 | It is specifically designed to facilitate the process of web scraping using advanced language models such as GPT-4. 5 | This project aims to simplify the extraction of data from web pages by converting user-defined requirements into Python code that executes the desired web scraping tasks. 6 | 7 | ## Features 8 | 9 | - **Dynamic Code Generation**: Generates Python parsing code based on user requirements and webpage content. 10 | - **Flexible Data Structures**: Supports the use of Pydantic models to define the structure of the scraped data. 11 | - **Webpage Source Handling**: Capable of extracting HTML content from both static and dynamic web pages using Selenium. 12 | 13 | ## Installation 14 | 15 | ### Prerequisites 16 | 17 | - **Python 3.6 or higher**: Ensure you have Python installed. You can download it from the [official website](https://www.python.org/downloads/). 18 | - **ChromeDriver**: Selenium requires ChromeDriver to interact with the Chrome browser. Download it from [here](https://sites.google.com/a/chromium.org/chromedriver/downloads) and ensure it's in your system's PATH. 19 | 20 | ### Install from git 21 | 22 | ```bash 23 | $ pip install git+https://github.com/bes-dev/gpt-scraper.git 24 | ``` 25 | 26 | ### Install from pip 27 | 28 | ```bash 29 | $ pip install gpt-scraper 30 | ``` 31 | 32 | ## CLI Tool Usage 33 | 34 | ### Commands 35 | 36 | ```bash 37 | $ gpt-scraper --help 38 | usage: gpt-scraper [-h] (--requirements REQUIREMENTS | --scraper-file SCRAPER_FILE) --url URL [--output OUTPUT] [--wait-by {id,xpath,css_selector}] [--wait-value WAIT_VALUE] 39 | [--save-file SAVE_FILE] [--model-name MODEL_NAME] [--simplify-html] [--use-sandbox] 40 | 41 | GPT-Scraper CLI 42 | 43 | options: 44 | -h, --help show this help message and exit 45 | --requirements REQUIREMENTS 46 | Scraping requirements 47 | --scraper-file SCRAPER_FILE 48 | Path to the scraper file to load 49 | --url URL URL of the webpage to scrape 50 | --output OUTPUT Output file path to save scraped data as JSON 51 | --wait-by {id,xpath,css_selector} 52 | Type of locator to wait for 53 | --wait-value WAIT_VALUE 54 | Value of the locator to wait for 55 | --save-file SAVE_FILE 56 | Path to save the created GPTScraper to file 57 | --model-name MODEL_NAME 58 | Name of the model to use for scraping 59 | --simplify-html Simplify the HTML content before parsing 60 | --use-sandbox Use the sandboxed environment for parsing 61 | 62 | ``` 63 | 64 | ### Sample session 65 | 66 | ```bash 67 | $ gpt-scraper --url https://news.ycombinator.com/ --requirements 'extract threads list from the web page (extract link and title)' --save-file hn.py --model-name o1-mini 68 | 2024-10-29 05:23:25,989 [INFO] Fetching page content from URL: https://news.ycombinator.com/ 69 | 2024-10-29 05:23:25,989 [INFO] Attempt 1 to fetch URL: https://news.ycombinator.com/ 70 | 2024-10-29 05:23:27,915 [INFO] Successfully fetched page source for URL: https://news.ycombinator.com/ 71 | 2024-10-29 05:23:27,977 [INFO] Generating parser using GPTScraper. 72 | 2024-10-29 05:23:34,517 [INFO] HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" 73 | 2024-10-29 05:23:34,605 [INFO] Saving scraper to file: hn.json 74 | 2024-10-29 05:23:34,605 [INFO] Scraper saved successfully. 75 | 2024-10-29 05:23:34,605 [INFO] Parsing HTML content. 76 | 2024-10-29 05:23:34,638 [INFO] Printing scraped data: 77 | [ 78 | { 79 | "title": "Excel Turing Machine (2013)", 80 | "link": "https://www.felienne.com/archives/2974" 81 | }, 82 | { 83 | "title": "High-resolution postmortem human brain MRI at 7 tesla", 84 | "link": "https://pulkit-khandelwal.github.io/exvivo-brain-upenn/" 85 | }, 86 | { 87 | "title": "How Gothic architecture became spooky", 88 | "link": "https://www.architecturaldigest.com/story/how-gothic-architecture-became-spooky" 89 | }, 90 | { 91 | "title": "Using reinforcement learning and $4.80 of GPU time to find the best HN post", 92 | "link": "https://openpipe.ai/blog/hacker-news-rlhf-part-1" 93 | } 94 | ] 95 | ``` 96 | 97 | ## Example 98 | 99 | ```python 100 | from gpt_scraper import GPTScraper 101 | from gpt_scraper.selenium_utils import fetch_dynamic_page 102 | from pydantic import BaseModel 103 | 104 | class Data(BaseModel): 105 | title: str 106 | url: str 107 | 108 | page_source = fetch_dynamic_page("https://news.ycombinator.com/") 109 | scraper = GPTScraper.from_html( 110 | page_source, 111 | "extract threads list from the web page (extract link and title)", 112 | data_structure=Data, 113 | model_name="o1-mini" 114 | ) 115 | data = scraper.parse_html(page_source, use_sandbox=True) 116 | print(data) 117 | ``` 118 | 119 | 120 | # Disclaimer 121 | This application assists users in generating code with AI. 122 | While a sandbox environment with limited system access is provided for added security, we cannot guarantee complete protection. 123 | We strongly recommend executing all generated code within the provided sandbox environment to help minimize potential risks. 124 | However, users should not rely on the sandbox as an absolute security measure. 125 | 126 | The development team is not liable for any consequences resulting from the generated code, including system damage, data loss, or any incurred losses. 127 | By using this application, you acknowledge and accept all risks associated with the generated code and assume full responsibility for any potential impact on your system. 128 | -------------------------------------------------------------------------------- /gpt_scraper/__init__.py: -------------------------------------------------------------------------------- 1 | from .gpt_scraper import GPTScraper 2 | -------------------------------------------------------------------------------- /gpt_scraper/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import argparse 18 | import json 19 | import logging 20 | from .gpt_scraper import GPTScraper 21 | from .selenium_utils import fetch_dynamic_page 22 | from pydantic import BaseModel 23 | 24 | def setup_logging(): 25 | logging.basicConfig( 26 | level=logging.INFO, 27 | format="%(asctime)s [%(levelname)s] %(message)s", 28 | handlers=[ 29 | logging.StreamHandler() 30 | ] 31 | ) 32 | return logging.getLogger(__name__) 33 | 34 | def main(): 35 | logger = setup_logging() 36 | 37 | parser = argparse.ArgumentParser(description='GPT-Scraper CLI') 38 | group = parser.add_mutually_exclusive_group(required=True) 39 | group.add_argument('--requirements', help='Scraping requirements') 40 | group.add_argument('--scraper-file', help='Path to the scraper file to load') 41 | 42 | parser.add_argument('--url', required=True, help='URL of the webpage to scrape') 43 | parser.add_argument('--output', help='Output file path to save scraped data as JSON') 44 | parser.add_argument('--wait-by', choices=['id', 'xpath', 'css_selector'], help='Type of locator to wait for') 45 | parser.add_argument('--wait-value', help='Value of the locator to wait for') 46 | parser.add_argument('--save-file', help='Path to save the created GPTScraper to file') 47 | parser.add_argument('--model-name', default='gpt-4o', help='Name of the model to use for scraping') 48 | parser.add_argument('--simplify-html', action='store_true', help='Simplify the HTML content before parsing') 49 | parser.add_argument('--use-sandbox', action='store_true', help='Use the sandboxed environment for parsing') 50 | 51 | args = parser.parse_args() 52 | 53 | wait_condition = None 54 | if args.wait_by and args.wait_value: 55 | from selenium.webdriver.common.by import By 56 | from selenium.webdriver.support import expected_conditions as EC 57 | 58 | condition_mapping = { 59 | 'id': By.ID, 60 | 'xpath': By.XPATH, 61 | 'css_selector': By.CSS_SELECTOR 62 | } 63 | wait_condition = { 64 | 'by': condition_mapping[args.wait_by], 65 | 'value': args.wait_value, 66 | 'condition': EC.presence_of_element_located 67 | } 68 | 69 | logger.info(f"Fetching page content from URL: {args.url}") 70 | page_source = fetch_dynamic_page(args.url, wait_condition=wait_condition) 71 | 72 | if not page_source: 73 | logger.error("Failed to fetch the webpage.") 74 | return 75 | 76 | if args.scraper_file: 77 | logger.info(f"Loading scraper from file: {args.scraper_file}") 78 | try: 79 | scraper = GPTScraper.load(args.scraper_file) 80 | logger.info("Scraper loaded successfully.") 81 | except Exception as e: 82 | logger.error(f"Failed to load scraper from file: {e}") 83 | return 84 | else: 85 | logger.info("Generating parser using GPTScraper.") 86 | scraper = GPTScraper.from_html( 87 | page_source, 88 | args.requirements, 89 | simplify_html=args.simplify_html, 90 | model_name=args.model_name 91 | ) 92 | if args.save_file: 93 | try: 94 | logger.info(f"Saving scraper to file: {args.save_file}") 95 | scraper.save(args.save_file) 96 | logger.info("Scraper saved successfully.") 97 | except Exception as e: 98 | logger.error(f"Failed to save scraper to file: {e}") 99 | 100 | logger.info("Parsing HTML content.") 101 | try: 102 | data = scraper.parse_html(page_source, use_sandbox=args.use_sandbox) 103 | except Exception as e: 104 | logger.error(f"Error parsing HTML: {e}") 105 | return 106 | 107 | if args.output: 108 | logger.info(f"Saving scraped data to {args.output}") 109 | try: 110 | with open(args.output, 'w', encoding='utf-8') as f: 111 | json.dump( 112 | [item if isinstance(item, dict) else item.dict() for item in data], 113 | f, 114 | indent=4, 115 | ensure_ascii=False 116 | ) 117 | logger.info("Data saved successfully.") 118 | except Exception as e: 119 | logger.error(f"Failed to save data: {e}") 120 | else: 121 | logger.info("Printing scraped data:") 122 | print( 123 | json.dumps( 124 | [item if isinstance(item, dict) else item.dict() for item in data], 125 | indent=4, 126 | ensure_ascii=False 127 | ) 128 | ) 129 | 130 | if __name__ == '__main__': 131 | main() 132 | -------------------------------------------------------------------------------- /gpt_scraper/gpt_scraper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import json 18 | from pydantic import BaseModel 19 | from .utils import * 20 | from .html_utils import preprocess_html_with_summary 21 | from .prompts import SCRAPER_GENERATOR_PROMPT, DATA_STRUCTURE_PROMPT 22 | from .sandbox import execute_sandboxed 23 | 24 | class GPTScraper: 25 | def __init__(self, source_code: str, data_structure: BaseModel = None): 26 | """ 27 | The GPTScraper class is designed to parse HTML content using a source code parser 28 | and an optional data structure model. 29 | 30 | Args: 31 | source_code (str): The source code for parsing HTML content. 32 | data_structure (BaseModel, optional): A Pydantic model representing the data structure of the parsed output. 33 | """ 34 | self.source_code = source_code 35 | self.parse = parser_from_text(source_code) 36 | self.data_structure = data_structure 37 | 38 | @classmethod 39 | def from_html( 40 | cls, 41 | page_source: str, 42 | requirements: str, 43 | data_structure: BaseModel = None, 44 | simplify_html: bool = False, 45 | model_name: str = "gpt-4o" 46 | ) -> "GPTScraper": 47 | """ 48 | Create a GPTScraper instance from HTML content and user requirements. 49 | 50 | Args: 51 | page_source (str): The HTML content of the webpage. 52 | requirements (str): The user-defined requirements for scraping. 53 | data_structure (BaseModel, optional): An optional data structure for the output. 54 | simplify_html (bool): Simplify HTML before processing. 55 | model_name (str): The model name to be used by the LLMProvider. 56 | 57 | Returns: 58 | GPTScraper: An instance of the GPTScraper class. 59 | """ 60 | llm_provider = ProviderOpenAI(model_name) 61 | if simplify_html: 62 | page_source, _ = preprocess_html_with_summary( 63 | page_source, 64 | remove_attributes=True, 65 | minify=True, 66 | summarize=True, 67 | max_sentences=1 68 | ) 69 | # prepare prompt 70 | system_prompt = SCRAPER_GENERATOR_PROMPT.replace( 71 | "{{USER_REQUIREMENTS}}", requirements 72 | ) 73 | system_prompt = system_prompt.replace( 74 | "{{HTML_CODE}}", page_source 75 | ) 76 | if data_structure is not None: 77 | data_structure_prompt = DATA_STRUCTURE_PROMPT.replace( 78 | "{{DATA_STRUCTURE}}", data_structure_to_str(data_structure) 79 | ) 80 | system_prompt = system_prompt.replace( 81 | "{{EXTRA_REQUIREMENTS}}", data_structure_prompt 82 | ) 83 | else: 84 | system_prompt = system_prompt.replace("{{EXTRA_REQUIREMENTS}}", "") 85 | # get response 86 | parser_code = llm_provider(system_prompt) 87 | parser_code = extract_code_block(parser_code) 88 | return cls(parser_code, data_structure) 89 | 90 | @classmethod 91 | def from_source_code( 92 | cls, 93 | source_code: str, 94 | data_structure: BaseModel = None 95 | ) -> "GPTScraper": 96 | """ 97 | Create a GPTScraper instance from the given source code and data structure. 98 | 99 | Args: 100 | source_code (str): The source code for parsing HTML content. 101 | data_structure (BaseModel, optional): An optional data structure for the output. 102 | 103 | Returns: 104 | GPTScraper: An instance of the GPTScraper class. 105 | """ 106 | return cls(source_code, data_structure) 107 | 108 | def save(self, path: str) -> None: 109 | """ 110 | Save the source code of the parser to a file. 111 | 112 | Args: 113 | path (str): The path to save the source code. 114 | """ 115 | with open(path, "w") as f: 116 | f.write(self.source_code) 117 | 118 | @classmethod 119 | def load(cls, path: str) -> "GPTScraper": 120 | """ 121 | Load the source code of the parser from a file. 122 | 123 | Args: 124 | path (str): The path to load the source code from. 125 | 126 | Returns: 127 | GPTScraper: An instance of the GPTScraper class. 128 | """ 129 | with open(path, "r") as f: 130 | source_code = f.read() 131 | return cls(source_code) 132 | 133 | def parse_html(self, html: str, use_sandbox: bool = True) -> list: 134 | """ 135 | Parse the provided HTML string into a structured format. 136 | 137 | Args: 138 | html (str): The HTML content to be parsed. 139 | use_sandbox (bool): Whether to use the sandboxed environment for parsing. 140 | 141 | Returns: 142 | list: A list of parsed data according to the data structure provided, if any. 143 | """ 144 | if use_sandbox: 145 | output = execute_sandboxed(self.source_code, html) 146 | else: 147 | output = self.parse(html) 148 | if self.data_structure is not None: 149 | return [self.data_structure(**item) for item in output] 150 | return output 151 | 152 | def check_parser(self, html: str) -> bool: 153 | """ 154 | Check if the parser can successfully parse the HTML content without errors. 155 | 156 | Args: 157 | html (str): The HTML content to be parsed. 158 | 159 | Returns: 160 | bool: True if parsing is successful, otherwise False. 161 | """ 162 | try: 163 | self.parse_html(html) 164 | return True 165 | except Exception as e: 166 | return False 167 | -------------------------------------------------------------------------------- /gpt_scraper/html_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | import re 17 | from typing import Tuple, Optional 18 | from bs4 import BeautifulSoup, Comment 19 | import htmlmin 20 | import nltk 21 | from nltk.tokenize import sent_tokenize 22 | 23 | 24 | # Ensure the NLTK 'punkt' tokenizer is downloaded 25 | nltk.download('punkt', quiet=True) 26 | 27 | 28 | def simplify_html( 29 | html_content: str, 30 | remove_attributes: bool = True, 31 | minify: bool = True 32 | ) -> str: 33 | """ 34 | Simplifies HTML content by removing scripts, styles, comments, and optionally unnecessary attributes. 35 | 36 | Args: 37 | html_content (str): The raw HTML content. 38 | remove_attributes (bool): Whether to remove attributes from tags. 39 | minify (bool): Whether to minify the resulting HTML. 40 | 41 | Returns: 42 | str: Simplified HTML. 43 | """ 44 | soup = BeautifulSoup(html_content, 'lxml') 45 | 46 | # Remove 163 | 164 | 165 |

Welcome to the Sample Page

166 |

This is a sample paragraph with a link.

167 | 168 | 169 |

Another paragraph to demonstrate preprocessing.

170 | Sample Image 171 | 172 | 173 | """ 174 | 175 | simplified_html, summary = preprocess_html_with_summary( 176 | sample_html, 177 | remove_attributes=True, 178 | minify=True, 179 | summarize=True, 180 | max_sentences=1 181 | ) 182 | 183 | print("Simplified and Minified HTML with Attributes Removed:\n") 184 | print(simplified_html) 185 | 186 | if summary: 187 | print("\nSummarized Text:\n") 188 | print(summary) 189 | -------------------------------------------------------------------------------- /gpt_scraper/prompts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | DATA_STRUCTURE_PROMPT = """ 18 | ** Output data structure: ** 19 | {{DATA_STRUCTURE}} 20 | """ 21 | 22 | SCRAPER_GENERATOR_PROMPT = """ 23 | You are an expert website analyzer specializing in web scraping processes. 24 | Your task is to convert the user's requirements into clean, efficient Python code that scrapes the website and extracts the desired data. 25 | 26 | ** User requirements: ** 27 | {{USER_REQUIREMENTS}} 28 | 29 | {{EXTRA_REQUIREMENTS}} 30 | 31 | ** Please generate a Python function with the following signature: ** 32 | ```python 33 | from typing import List, Dict 34 | 35 | def parse(html: str) -> List[Dict]: 36 | ... 37 | ``` 38 | 39 | ** Implementation requirements: ** 40 | - Utilize appropriate libraries for HTML parsing, such as BeautifulSoup, lxml, or etc.. 41 | - Implement robust error handling to manage potential parsing issues. 42 | - Ensure the function returns the extracted data as a list of dictionaries with a well-defined structure. 43 | - Include all necessary imports in the code. 44 | - Adhere to PEP8 coding standards for readability and maintainability. 45 | - Optimize the code for performance where possible. 46 | - Do not include any explanations, comments, or additional text—return only the code block. 47 | - Return implementation as block of code formated using: 48 | ```python 49 | 50 | ``` 51 | 52 | ** HTML Code to parse: ** 53 | {{HTML_CODE}} 54 | """ 55 | -------------------------------------------------------------------------------- /gpt_scraper/sandbox.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import builtins 3 | import logging 4 | from typing import Any, Dict, List, Optional, Set, Callable 5 | from types import FunctionType 6 | 7 | # Configure logging 8 | logging.basicConfig(level=logging.INFO) 9 | logger = logging.getLogger("sandbox") 10 | 11 | 12 | # Define a set of safe modules that are allowed to be imported 13 | SAFE_MODULES: Set[str] = { 14 | 'math', 15 | 'json', 16 | 're', 17 | 'datetime', 18 | 'logging', 19 | 'typing', 20 | 'bs4', # BeautifulSoup 21 | # Add other modules that you deem safe 22 | } 23 | 24 | 25 | def extract_imports(code: str) -> Set[str]: 26 | """ 27 | Parses the code and extracts all imported module names. 28 | 29 | Args: 30 | code (str): The Python code to analyze. 31 | 32 | Returns: 33 | Set[str]: A set of module names that are imported in the code. 34 | """ 35 | tree = ast.parse(code) 36 | imports = set() 37 | 38 | for node in ast.walk(tree): 39 | if isinstance(node, ast.Import): 40 | for alias in node.names: 41 | imports.add(alias.name.split('.')[0]) 42 | elif isinstance(node, ast.ImportFrom): 43 | if node.module: 44 | imports.add(node.module.split('.')[0]) 45 | 46 | logger.debug(f"Extracted imports: {imports}") 47 | return imports 48 | 49 | 50 | def safe_import( 51 | name: str, 52 | globals: Optional[Dict[str, Any]] = None, 53 | locals: Optional[Dict[str, Any]] = None, 54 | fromlist: Optional[List[str]] = None, 55 | level: int = 0 56 | ) -> Any: 57 | """ 58 | A custom __import__ function that restricts imports to SAFE_MODULES only. 59 | 60 | Args: 61 | name (str): The name of the module to import. 62 | globals (dict, optional): Not used. 63 | locals (dict, optional): Not used. 64 | fromlist (list, optional): Not used. 65 | level (int, optional): Not used. 66 | 67 | Returns: 68 | module: The imported module if allowed. 69 | 70 | Raises: 71 | ImportError: If the module is not in the SAFE_MODULES list. 72 | """ 73 | if name in SAFE_MODULES: 74 | logger.debug(f"Importing allowed module: {name}") 75 | return builtins.__import__(name, globals, locals, fromlist, level) 76 | else: 77 | logger.error(f"Attempted to import disallowed module: {name}") 78 | raise ImportError(f"Importing module '{name}' is not allowed.") 79 | 80 | 81 | def execute_sandboxed( 82 | code: str, 83 | html_content: str 84 | ) -> List[Dict[str, Optional[str]]]: 85 | """ 86 | Executes the provided code in a restricted environment and calls the parse function. 87 | 88 | Args: 89 | code (str): The code containing the parse function. 90 | html_content (str): The HTML content to parse. 91 | 92 | Returns: 93 | List[Dict[str, Optional[str]]]: The result from the parse function. 94 | 95 | Raises: 96 | ImportError: If the code contains disallowed imports. 97 | ValueError: If the 'parse' function is not defined properly. 98 | Exception: If any other error occurs during execution. 99 | """ 100 | # Step 1: Extract and validate imports 101 | imported_modules = extract_imports(code) 102 | disallowed_modules = imported_modules - SAFE_MODULES 103 | if disallowed_modules: 104 | logger.error(f"Disallowed modules detected: {disallowed_modules}") 105 | raise ImportError(f"The following modules are not allowed: {disallowed_modules}") 106 | 107 | # Step 2: Prepare restricted built-ins, including the custom __import__ 108 | restricted_builtins: Dict[str, Any] = { 109 | 'abs': abs, 110 | 'bool': bool, 111 | 'dict': dict, 112 | 'float': float, 113 | 'int': int, 114 | 'len': len, 115 | 'list': list, 116 | 'max': max, 117 | 'min': min, 118 | 'range': range, 119 | 'str': str, 120 | 'print': print, # Optionally allow print 121 | '__import__': safe_import, # Include the custom __import__ here 122 | 'Exception': Exception, # Add Exception class 123 | 'AttributeError': AttributeError, # Add AttributeError class 124 | # Add other safe built-ins as needed 125 | } 126 | 127 | # Step 3: Define the global execution environment 128 | restricted_globals: Dict[str, Any] = { 129 | '__builtins__': restricted_builtins, 130 | 'logging': logging, # Allow logging if needed by the code 131 | } 132 | 133 | try: 134 | # Step 4: Execute the code with restricted_globals as both globals and locals 135 | exec(code, restricted_globals) 136 | logger.info("Code executed successfully in sandbox.") 137 | 138 | # Step 5: Retrieve the parse function from restricted_globals 139 | parse_func: Optional[FunctionType] = restricted_globals.get('parse') 140 | if not isinstance(parse_func, FunctionType): 141 | logger.error("The 'parse' function is not defined properly in the provided code.") 142 | raise ValueError("The 'parse' function is not defined properly in the provided code.") 143 | 144 | # Step 6: Call the parse function with the provided HTML content 145 | result = parse_func(html_content) 146 | logger.info("Parse function executed successfully.") 147 | return result 148 | 149 | except Exception as e: 150 | logger.exception(f"An error occurred during sandboxed execution: {e}") 151 | raise 152 | -------------------------------------------------------------------------------- /gpt_scraper/selenium_utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import logging 18 | from selenium import webdriver 19 | from selenium.webdriver.chrome.options import Options 20 | from selenium.webdriver.support.ui import WebDriverWait 21 | from selenium.webdriver.support import expected_conditions as EC 22 | from selenium.common.exceptions import ( 23 | WebDriverException, 24 | TimeoutException, 25 | NoSuchElementException, 26 | ) 27 | from selenium.webdriver.common.by import By 28 | from typing import Optional, Dict, Any 29 | import time 30 | 31 | # Configure logging 32 | logging.basicConfig( 33 | level=logging.INFO, 34 | format="%(asctime)s [%(levelname)s] %(message)s", 35 | handlers=[ 36 | logging.StreamHandler() 37 | ] 38 | ) 39 | logger = logging.getLogger(__name__) 40 | 41 | 42 | def fetch_page(url: str) -> Optional[str]: 43 | """ 44 | Fetches the HTML content of a web page using Selenium. 45 | 46 | Args: 47 | url (str): The URL of the web page to fetch. 48 | 49 | Returns: 50 | Optional[str]: The HTML content of the page if successful, else None. 51 | """ 52 | chrome_options = Options() 53 | chrome_options.add_argument("--headless") # Run in headless mode 54 | chrome_options.add_argument("--disable-gpu") 55 | chrome_options.add_argument("--no-sandbox") 56 | chrome_options.add_argument( 57 | "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) " 58 | "AppleWebKit/537.36 (KHTML, like Gecko) " 59 | "Chrome/117.0.0.0 Safari/537.36" 60 | ) 61 | chrome_options.add_argument("--window-size=1920,1080") 62 | 63 | try: 64 | driver = webdriver.Chrome(options=chrome_options) 65 | driver.get(url) 66 | 67 | page_source = driver.page_source 68 | driver.quit() 69 | return page_source 70 | except WebDriverException as e: 71 | print(f"Selenium WebDriver Error: {e}") 72 | except Exception as e: 73 | print(f"An error occurred: {e}") 74 | 75 | return None 76 | 77 | 78 | def fetch_dynamic_page( 79 | url: str, 80 | timeout: int = 30, 81 | wait_condition: Optional[Dict[str, Any]] = None, 82 | retries: int = 3, 83 | delay: int = 5, 84 | driver_path: Optional[str] = None 85 | ) -> Optional[str]: 86 | """ 87 | Fetches the dynamic web page source using Selenium. 88 | 89 | Args: 90 | url (str): The URL of the web page to fetch. 91 | timeout (int, optional): Maximum time to wait for the page to load. Defaults to 30 seconds. 92 | wait_condition (Dict[str, Any], optional): Conditions to wait for before fetching the page source. 93 | Example: 94 | { 95 | "by": By.ID, 96 | "value": "main-content", 97 | "condition": EC.presence_of_element_located 98 | } 99 | retries (int, optional): Number of retry attempts in case of failure. Defaults to 3. 100 | delay (int, optional): Delay between retries in seconds. Defaults to 5. 101 | driver_path (str, optional): Path to the ChromeDriver executable. If None, assumes it's in PATH. 102 | 103 | Returns: 104 | Optional[str]: The page source if successful, else None. 105 | """ 106 | chrome_options = Options() 107 | chrome_options.add_argument("--headless=new") # Use new headless mode 108 | chrome_options.add_argument("--disable-gpu") 109 | chrome_options.add_argument("--no-sandbox") 110 | chrome_options.add_argument( 111 | "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) " 112 | "AppleWebKit/537.36 (KHTML, like Gecko) " 113 | "Chrome/117.0.0.0 Safari/537.36" 114 | ) 115 | chrome_options.add_argument("--window-size=1920,1080") 116 | chrome_options.add_argument("--disable-extensions") 117 | chrome_options.add_argument("--disable-dev-shm-usage") # Overcome limited resource problems 118 | 119 | for attempt in range(1, retries + 1): 120 | try: 121 | logger.info(f"Attempt {attempt} to fetch URL: {url}") 122 | # Initialize the WebDriver 123 | if driver_path: 124 | driver = webdriver.Chrome(executable_path=driver_path, options=chrome_options) 125 | else: 126 | driver = webdriver.Chrome(options=chrome_options) 127 | 128 | try: 129 | driver.set_page_load_timeout(timeout) 130 | driver.get(url) 131 | 132 | if wait_condition: 133 | condition = wait_condition.get("condition", EC.presence_of_element_located) 134 | locator = (wait_condition.get("by", By.ID), wait_condition.get("value", "")) 135 | logger.info(f"Waiting for condition: {wait_condition}") 136 | WebDriverWait(driver, timeout).until(condition(locator)) 137 | 138 | # Optionally, you can add more sophisticated waits here 139 | 140 | page_source = driver.page_source 141 | logger.info(f"Successfully fetched page source for URL: {url}") 142 | return page_source 143 | finally: 144 | driver.quit() 145 | 146 | except TimeoutException: 147 | logger.warning(f"Timeout while loading the page: {url}") 148 | except NoSuchElementException: 149 | logger.warning(f"Desired element not found on the page: {url}") 150 | except WebDriverException as e: 151 | logger.error(f"Selenium WebDriver error: {e}") 152 | except Exception as e: 153 | logger.error(f"An unexpected error occurred: {e}") 154 | 155 | if attempt < retries: 156 | logger.info(f"Retrying in {delay} seconds...") 157 | time.sleep(delay) 158 | else: 159 | logger.error(f"All {retries} attempts failed for URL: {url}") 160 | 161 | return None 162 | -------------------------------------------------------------------------------- /gpt_scraper/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | import json 18 | import re 19 | import pydantic 20 | from pydantic import BaseModel 21 | import types 22 | from openai import OpenAI 23 | from typing import Optional, Dict, Any, get_origin, get_args 24 | 25 | 26 | def parser_from_text(source_code: str) -> types.FunctionType: 27 | """ 28 | Compile source code into a Python module and extract the 'parse' function. 29 | 30 | Args: 31 | source_code (str): The source code containing the 'parse' function. 32 | 33 | Returns: 34 | types.FunctionType: The function object of 'parse'. 35 | 36 | Raises: 37 | AssertionError: If 'parse' function is not found in the source code. 38 | """ 39 | module = types.ModuleType("dynamic_module") 40 | exec(source_code, module.__dict__) 41 | assert hasattr(module, "parse"), "Model class not found in source code" 42 | return module.parse 43 | 44 | 45 | def extract_code_block(text: str, language: str = "python") -> Optional[str]: 46 | """ 47 | Extracts the first code block of a specified language from the given text. 48 | 49 | Args: 50 | text (str): The input text containing code blocks. 51 | language (str, optional): The programming language of the desired code block. Defaults to "python". 52 | 53 | Returns: 54 | Optional[str]: The extracted code block, or None if not found. 55 | """ 56 | code_blocks = re.findall(r'```(.*?)```', text, re.DOTALL) 57 | for block in code_blocks: 58 | block = block.strip() 59 | if block.startswith(language): 60 | code = block[len(language):].strip() 61 | return code 62 | return None 63 | 64 | 65 | def type_to_str(tp): 66 | """ 67 | Converts a type annotation to a readable string. 68 | Handles generic types like List[int], Optional[str], etc. 69 | """ 70 | origin = get_origin(tp) 71 | args = get_args(tp) 72 | 73 | if origin is None: 74 | return tp.__name__ if isinstance(tp, type) else str(tp) 75 | elif origin is list: 76 | return f"List[{type_to_str(args[0])}]" 77 | elif origin is dict: 78 | return f"Dict[{type_to_str(args[0])}, {type_to_str(args[1])}]" 79 | elif origin is Union: 80 | # Handle Optional and Union types 81 | return " | ".join(type_to_str(arg) for arg in args) 82 | else: 83 | return str(tp) 84 | 85 | 86 | def data_structure_to_str(model: BaseModel) -> str: 87 | """ 88 | Converts a Pydantic model into a string representation of its fields. 89 | 90 | Args: 91 | model (BaseModel): The Pydantic model to convert. 92 | 93 | Returns: 94 | str: A JSON-formatted string representing the fields of the model with their types. 95 | """ 96 | fields_dict = {} 97 | for field_name, field_info in model.model_fields.items(): 98 | field_type = field_info.annotation 99 | type_str = type_to_str(field_type) 100 | fields_dict[field_name] = f"<{type_str}>" 101 | return json.dumps(fields_dict, indent=4) 102 | 103 | 104 | class ProviderOpenAI: 105 | def __init__(self, model_name: str = "gpt-4o"): 106 | """ 107 | Initializes the ProviderOpenAI class to interact with the OpenAI GPT model. 108 | 109 | Args: 110 | model_name (str, optional): The name of the model to be used. Defaults to "gpt-4o". 111 | """ 112 | self.model_name = model_name 113 | self.client = OpenAI() 114 | 115 | def __call__(self, prompt: str) -> Optional[str]: 116 | """ 117 | Calls the GPT model with a given prompt and returns the response. 118 | 119 | Args: 120 | prompt (str): The input prompt to send to the model. 121 | 122 | Returns: 123 | Optional[str]: The generated text by the model, or None if an error occurs. 124 | """ 125 | try: 126 | completion = self.client.chat.completions.create( 127 | model=self.model_name, 128 | messages=[ 129 | {"role": "user", "content": prompt} 130 | ] 131 | ) 132 | return completion.choices[0].message.content 133 | except Exception as e: 134 | print(f"An error occurred: {e}") 135 | return None 136 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2024 by Sergei Belousov 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | """ 16 | 17 | from setuptools import setup, find_packages 18 | 19 | setup( 20 | name='gpt_scraper', 21 | version='0.3.2', 22 | description='GPT-Scraper is an autonomous, LLM-based agent that generates code to extract structured information from web pages.', 23 | author='Sergei Belousov aka BeS', 24 | author_email='sergei.o.belousov@gmail.com', 25 | url='https://github.com/bes-dev/gpt-scraper.git', 26 | packages=find_packages(exclude=[".git", ".gitignore"]), 27 | install_requires=[ 28 | 'openai', 29 | 'pydantic', 30 | 'selenium', 31 | 'nltk', 32 | 'htmlmin', 33 | 'beautifulsoup4' 34 | ], 35 | entry_points={ 36 | 'console_scripts': [ 37 | 'gpt-scraper = gpt_scraper.cli:main', 38 | ], 39 | }, 40 | classifiers=[ 41 | 'Programming Language :: Python :: 3', 42 | 'License :: OSI Approved :: Apache Software License', 43 | 'Operating System :: OS Independent', 44 | ], 45 | python_requires='>=3.6', 46 | ) 47 | --------------------------------------------------------------------------------