├── .gitignore ├── crawlbase ├── __init__.py ├── leads_api.py ├── scraper_api.py ├── crawling_api.py ├── screenshots_api.py ├── storage_api.py └── base_api.py ├── setup.py ├── test.py ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | crawlbase.egg-info 4 | dist 5 | -------------------------------------------------------------------------------- /crawlbase/__init__.py: -------------------------------------------------------------------------------- 1 | from crawlbase.crawling_api import CrawlingAPI 2 | from crawlbase.scraper_api import ScraperAPI 3 | from crawlbase.leads_api import LeadsAPI 4 | from crawlbase.screenshots_api import ScreenshotsAPI 5 | from crawlbase.storage_api import StorageAPI 6 | -------------------------------------------------------------------------------- /crawlbase/leads_api.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from crawlbase.base_api import BaseAPI 3 | 4 | # 5 | # A Python class that acts as wrapper for Crawlbase Leads API. 6 | # 7 | # Read Crawlbase API documentation https://crawlbase.com/docs/leads-api/ 8 | # 9 | # Copyright Crawlbase 10 | # Licensed under the Apache License 2.0 11 | # 12 | class LeadsAPI(BaseAPI): 13 | base_path = 'leads' 14 | 15 | def get_from_domain(self, domain, options = {}): 16 | options['domain'] = domain 17 | return self.request(options) 18 | -------------------------------------------------------------------------------- /crawlbase/scraper_api.py: -------------------------------------------------------------------------------- 1 | from crawlbase.base_api import BaseAPI 2 | 3 | # 4 | # A Python class that acts as wrapper for Crawlbase Scraper API. 5 | # 6 | # Read Crawlbase API documentation https://crawlbase.com/docs/scraper-api/ 7 | # 8 | # Copyright Crawlbase 9 | # Licensed under the Apache License 2.0 10 | # 11 | class ScraperAPI(BaseAPI): 12 | base_path = 'scraper' 13 | 14 | def get(self, url, options = {}): 15 | options['url'] = url 16 | return self.request(options) 17 | 18 | def post(self, url, data, options = {}): 19 | raise Exception('Only GET is allowed on the Scraper API') 20 | -------------------------------------------------------------------------------- /crawlbase/crawling_api.py: -------------------------------------------------------------------------------- 1 | try: 2 | # For Python 3.0 and later 3 | from urllib.parse import urlencode, quote_plus 4 | except ImportError: 5 | # Fall back to Python 2's 6 | from urllib import urlencode, quote_plus 7 | 8 | from crawlbase.base_api import BaseAPI 9 | 10 | # 11 | # A Python class that acts as wrapper for Crawlbase Crawling API. 12 | # 13 | # Read Crawlbase API documentation https://crawlbase.com/docs/crawling-api/ 14 | # 15 | # Copyright Crawlbase 16 | # Licensed under the Apache License 2.0 17 | # 18 | class CrawlingAPI(BaseAPI): 19 | def get(self, url, options = {}): 20 | options['url'] = url 21 | return self.request(options) 22 | 23 | def post(self, url, data, options = {}): 24 | if isinstance(data, dict): 25 | data = urlencode(data) 26 | data = data.encode('utf-8') 27 | options['url'] = url 28 | return self.request(options, data) 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Python API wrapper for the Crawlbase API.""" 2 | 3 | import os 4 | 5 | try: 6 | from setuptools import setup, find_packages 7 | except ImportError: 8 | from distutils.core import setup, find_packages 9 | 10 | readme = open('README.md').read() 11 | 12 | setup( 13 | name = 'crawlbase', 14 | license = 'Apache-2.0', 15 | version = '1.0.0', 16 | description = 'A Python class that acts as wrapper for Crawlbase scraping and crawling API', 17 | long_description = readme, 18 | long_description_content_type = 'text/markdown', 19 | author = 'Crawlbase', 20 | author_email = 'info@crawlbase.com', 21 | url = 'https://github.com/crawlbase-source/crawlbase-python', 22 | keywords = 'scraping scraper crawler crawling crawlbase api', 23 | include_package_data = True, 24 | packages = find_packages(), 25 | classifiers = ( 26 | 'Programming Language :: Python :: 2', 27 | 'Programming Language :: Python :: 2.7', 28 | 'Programming Language :: Python :: 3', 29 | 'Programming Language :: Python :: 3.5', 30 | 'Programming Language :: Python :: 3.9', 31 | 'Development Status :: 5 - Production/Stable', 32 | 'Intended Audience :: Developers', 33 | 'License :: OSI Approved :: Apache Software License', 34 | 'Operating System :: OS Independent', 35 | 'Topic :: Utilities', 36 | ), 37 | ) 38 | -------------------------------------------------------------------------------- /crawlbase/screenshots_api.py: -------------------------------------------------------------------------------- 1 | import uuid, re, os, tempfile 2 | from crawlbase.base_api import BaseAPI 3 | 4 | # 5 | # A Python class that acts as wrapper for Crawlbase Screenshots API. 6 | # 7 | # Read Crawlbase API documentation https://crawlbase.com/docs/screenshots-api/ 8 | # 9 | # Copyright Crawlbase 10 | # Licensed under the Apache License 2.0 11 | # 12 | class ScreenshotsAPI(BaseAPI): 13 | base_path = 'screenshots' 14 | 15 | def get(self, url, options = {}): 16 | screenshotPath = options.pop('save_to_path') if 'save_to_path' in options else self.__generateFilepath() 17 | if not re.match(r".+\.(jpg|JPG|jpeg|JPEG)$", screenshotPath): 18 | raise Exception('save_to_path must end with .jpg or .jpeg') 19 | options['url'] = url 20 | response = self.request(options) 21 | with open(screenshotPath,'wb') as f: 22 | f.write(response['body']) 23 | response['file'] = screenshotPath 24 | return response 25 | 26 | def post(self, url, data, options = {}): 27 | raise Exception('Only GET is allowed on the Screenshots API') 28 | 29 | def parseRegularResponse(self, handler): 30 | headers = handler.headers 31 | BaseAPI.parseRegularResponse(self, handler) 32 | self.response['headers']['success'] = str(headers.get('success')) 33 | self.response['headers']['remaining_requests'] = str(headers.get('remaining_requests')) 34 | self.response['headers']['screenshot_url'] = str(headers.get('screenshot_url')) 35 | 36 | def __generateFilename(self): 37 | return str(uuid.uuid4()) + '.jpg' 38 | 39 | def __generateFilepath(self): 40 | return os.path.join(tempfile.gettempdir(), self.__generateFilename()) -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | 4 | from crawlbase import CrawlingAPI, ScraperAPI, LeadsAPI, ScreenshotsAPI, StorageAPI 5 | 6 | normal_token = '' 7 | javascript_token = '' 8 | 9 | def process_response(response): 10 | if response['status_code'] == 200: 11 | print('Test passed') 12 | else: 13 | print('Test failed, expected status_code 200 but got: ' + str(response['status_code'])) 14 | sys.exit(0) 15 | 16 | normal_api = CrawlingAPI({ 'token': normal_token }) 17 | 18 | process_response(normal_api.get('http://httpbin.org/anything?hello=world')) 19 | 20 | process_response(normal_api.get('http://httpbin.org/anything?useragent=test', { 'user_agent': 'Mozilla/5.0 (Windows NT 6.2 rv:20.0) Gecko/20121202 Firefox/20.0' })) 21 | 22 | process_response(normal_api.get('http://httpbin.org/anything', { 'format': 'json' })) 23 | 24 | process_response(normal_api.post('http://httpbin.org/post', { 'hello': 'post' })) 25 | 26 | process_response(normal_api.post('http://httpbin.org/post', json.dumps({ 'hello': 'json' }), { 'post_content_type': 'application/json' })) 27 | 28 | javascript_api = CrawlingAPI({ 'token': javascript_token }) 29 | 30 | process_response(javascript_api.get('http://httpbin.org/anything?hello=world')) 31 | 32 | scraper_api = ScraperAPI({ 'token': normal_token }) 33 | 34 | process_response(scraper_api.get('https://www.amazon.com/DualSense-Wireless-Controller-PlayStation-5/dp/B08FC6C75Y/')) 35 | 36 | leads_api = LeadsAPI({ 'token': normal_token }) 37 | 38 | process_response(leads_api.get_from_domain('microsoft.com')) 39 | 40 | screenshots_api = ScreenshotsAPI({ 'token': normal_token }) 41 | 42 | process_response(screenshots_api.get('https://www.apple.com')) 43 | 44 | storage_api = StorageAPI({ 'token': normal_token }) 45 | 46 | rids = storage_api.rids() 47 | print('Test passed') 48 | response = storage_api.bulk(rids) 49 | if response['status_code'] == 200: 50 | print('Test passed') 51 | for item in response['json']: 52 | process_response(storage_api.get(item['url'])) 53 | process_response(storage_api.get(item['rid'])) 54 | 55 | if (len(response['json']) == storage_api.totalCount()): 56 | print('Test passed') 57 | -------------------------------------------------------------------------------- /crawlbase/storage_api.py: -------------------------------------------------------------------------------- 1 | import re, json 2 | from crawlbase.base_api import BaseAPI 3 | 4 | # 5 | # A Python class that acts as wrapper for Crawlbase Storage API. 6 | # 7 | # Read Crawlbase API documentation https://crawlbase.com/docs/storage-api/ 8 | # 9 | # Copyright Crawlbase 10 | # Licensed under the Apache License 2.0 11 | # 12 | 13 | INVALID_TOKEN = 'Token is required' 14 | INVALID_RID = 'RID is required' 15 | INVALID_RID_ARRAY = 'One or more RIDs are required' 16 | INVALID_URL_OR_RID = 'Either URL or RID is required' 17 | 18 | class StorageAPI(BaseAPI): 19 | 20 | def get(self, url_or_rid, options = {}): 21 | if url_or_rid is None or url_or_rid == '': 22 | raise Exception(INVALID_URL_OR_RID) 23 | if 'format' not in options: 24 | options['format'] = 'html' 25 | options.update(self.__decideUrlOrRID(url_or_rid)) 26 | self.base_path = 'storage' 27 | response = self.request(options) 28 | if options['format'] == 'json': 29 | response['json_body'] = response.pop('json') 30 | return response 31 | 32 | def delete(self, rid): 33 | if rid is None or rid == '': 34 | raise Exception(INVALID_RID) 35 | options = { 'rid': rid, 'HTTP_METHOD': 'DELETE' } 36 | self.base_path = 'storage' 37 | response = self.request(options) 38 | return response['status_code'] == 200 39 | 40 | def bulk(self, ridsArray = []): 41 | if not ridsArray: 42 | raise Exception(INVALID_RID_ARRAY) 43 | self.base_path = 'storage/bulk' 44 | self.headers = { 'Accept-Encoding': 'gzip', 'Content-Type': 'application/json' } 45 | data = { 'rids': ridsArray } 46 | response = self.request({}, json.dumps(data)) 47 | return response 48 | 49 | def rids(self, limit = -1): 50 | self.base_path = 'storage/rids' 51 | options = {} 52 | if limit >= 0: 53 | options['limit'] = limit 54 | response = self.request(options) 55 | return response['json'] 56 | 57 | def totalCount(self): 58 | self.base_path = 'storage/total_count' 59 | response = self.request({}) 60 | return int(response['json']['totalCount']) 61 | 62 | def __decideUrlOrRID(self, url_or_rid): 63 | if re.match(r"^https?://", url_or_rid): 64 | return { 'url': url_or_rid } 65 | else: 66 | return { 'rid': url_or_rid } 67 | 68 | def parseJsonResponse(self): 69 | BaseAPI.parseJsonResponse(self) 70 | parsed_json = json.loads(self.response['body']) 71 | if 'original_status' in parsed_json: 72 | self.response['headers']['rid'] = str(parsed_json['rid']) 73 | self.response['headers']['stored_at'] = str(parsed_json['stored_at']) 74 | 75 | def parseRegularResponse(self, handler): 76 | BaseAPI.parseRegularResponse(self, handler) 77 | headers = handler.headers 78 | self.response['headers']['rid'] = str(headers.get('rid')) 79 | self.response['headers']['stored_at'] = str(headers.get('stored_at')) 80 | -------------------------------------------------------------------------------- /crawlbase/base_api.py: -------------------------------------------------------------------------------- 1 | import json 2 | import gzip 3 | import ssl 4 | import sys 5 | try: 6 | # For Python 3.0 and later 7 | from urllib.request import urlopen, HTTPError, Request 8 | except ImportError: 9 | # Fall back to Python 2's 10 | from urllib2 import urlopen, HTTPError, Request 11 | try: 12 | # For Python 3.0 and later 13 | from urllib.parse import urlencode, quote_plus 14 | except ImportError: 15 | # Fall back to Python 2's 16 | from urllib import urlencode, quote_plus 17 | try: 18 | # For Python 3.0 and later 19 | from io import BytesIO 20 | except ImportError: 21 | # Fall back to Python 2's 22 | from BytesIO import BytesIO 23 | 24 | # 25 | # A Python class that acts as base for Crawlbase APIs. 26 | # 27 | # This is not meant to be use directly, please use the other classes. 28 | # 29 | # Copyright Crawlbase 30 | # Licensed under the Apache License 2.0 31 | # 32 | CRAWLBASE_API_URL = 'https://api.crawlbase.com/' 33 | 34 | class BaseAPI(object): 35 | timeout = 120 36 | headers = { 'Accept-Encoding': 'gzip' } 37 | base_path = '' 38 | 39 | def __init__(self, options): 40 | if options['token'] is None or options['token'] == '': 41 | raise Exception('You need to specify the token') 42 | if 'timeout' in options: 43 | self.timeout = options['timeout'] 44 | self.options = options 45 | 46 | def request(self, options = {}, data = None): 47 | self.response = {} 48 | self.response['headers'] = {} 49 | http_method = options.pop('HTTP_METHOD') if 'HTTP_METHOD' in options else None 50 | url = self.buildURL(options) 51 | req = Request(url, headers=self.headers) 52 | if not http_method is None: 53 | req.get_method = lambda: http_method 54 | ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS) 55 | 56 | try: 57 | if (type(data) == str): 58 | data = data.encode('utf-8') 59 | handler = urlopen(req, data, self.timeout, context=ssl_context) 60 | except HTTPError as error: 61 | self.response['body'] = '' 62 | self.response['status_code'] = error.code 63 | return self.response 64 | 65 | self.response['status_code'] = handler.getcode() 66 | response_headers = dict(handler.info()) 67 | if ('Content-Encoding' in response_headers and response_headers['Content-Encoding'] == 'gzip') or ('content-encoding' in response_headers and response_headers['content-encoding'] == 'gzip'): 68 | self.response['body'] = self.decompressBody(handler) 69 | else: 70 | self.response['body'] = handler.read() 71 | 72 | if (handler.headers.get('Content-Type') == 'application/json; charset=utf-8' or 73 | (options and not options.get('callback') and options.get('format') == 'json')): 74 | self.parseJsonResponse() 75 | else: 76 | self.parseRegularResponse(handler) 77 | 78 | return self.response 79 | 80 | def buildURL(self, options): 81 | options = urlencode(options or {}) 82 | url = CRAWLBASE_API_URL + self.base_path + '?token=' + self.options['token'] + '&' + options 83 | 84 | return url 85 | 86 | def decompressBody(self, handler): 87 | body_stream = BytesIO(handler.read()) 88 | body_gzip = gzip.GzipFile(fileobj=body_stream) 89 | 90 | return body_gzip.read() 91 | 92 | def parseJsonResponse(self): 93 | parsed_json = json.loads(self.response['body']) 94 | if 'original_status' in parsed_json: 95 | self.response['headers']['original_status'] = str(parsed_json['original_status']) 96 | self.response['headers']['pc_status'] = str(parsed_json['pc_status']) 97 | self.response['headers']['url'] = str(parsed_json['url']) 98 | 99 | if 'body' in parsed_json: 100 | compare_str = str if sys.version_info[0] > 2 else basestring 101 | if isinstance(parsed_json['body'], compare_str): 102 | try: 103 | self.response['json'] = json.loads(parsed_json['body']) 104 | except ValueError: 105 | self.response['json'] = parsed_json['body'] 106 | else: 107 | self.response['json'] = parsed_json['body'] 108 | else: 109 | self.response['json'] = parsed_json 110 | 111 | def parseRegularResponse(self, handler): 112 | headers = handler.headers 113 | self.response['headers']['original_status'] = str(headers.get('original_status')) 114 | self.response['headers']['pc_status'] = str(headers.get('pc_status')) 115 | self.response['headers']['url'] = str(headers.get('url')) 116 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Crawlbase API Python class 2 | 3 | A lightweight, dependency free Python class that acts as wrapper for Crawlbase API. 4 | 5 | ## Installing 6 | 7 | Choose a way of installing: 8 | 9 | - Download the python class from Github. 10 | - Or use [PyPi](https://pypi.org/project/crawlbase/) Python package manager. `pip install crawlbase` 11 | 12 | Then import the CrawlingAPI, ScraperAPI, etc as needed. 13 | 14 | ```python 15 | from crawlbase import CrawlingAPI, ScraperAPI, LeadsAPI, ScreenshotsAPI, StorageAPI 16 | ``` 17 | 18 | ## Crawling API 19 | 20 | First initialize the CrawlingAPI class. 21 | 22 | ```python 23 | api = CrawlingAPI({ 'token': 'YOUR_CRAWLBASE_TOKEN' }) 24 | ``` 25 | 26 | ### GET requests 27 | 28 | Pass the url that you want to scrape plus any options from the ones available in the [API documentation](https://crawlbase.com/docs). 29 | 30 | ```python 31 | api.get(url, options = {}) 32 | ``` 33 | 34 | Example: 35 | 36 | ```python 37 | response = api.get('https://www.facebook.com/britneyspears') 38 | if response['status_code'] == 200: 39 | print(response['body']) 40 | ``` 41 | 42 | You can pass any options from Crawlbase API. 43 | 44 | Example: 45 | 46 | ```python 47 | response = api.get('https://www.reddit.com/r/pics/comments/5bx4bx/thanks_obama/', { 48 | 'user_agent': 'Mozilla/5.0 (Windows NT 6.2; rv:20.0) Gecko/20121202 Firefox/30.0', 49 | 'format': 'json' 50 | }) 51 | if response['status_code'] == 200: 52 | print(response['body']) 53 | ``` 54 | 55 | ### POST requests 56 | 57 | Pass the url that you want to scrape, the data that you want to send which can be either a json or a string, plus any options from the ones available in the [API documentation](https://crawlbase.com/docs). 58 | 59 | ```python 60 | api.post(url, dictionary or string data, options = {}) 61 | ``` 62 | 63 | Example: 64 | 65 | ```python 66 | response = api.post('https://producthunt.com/search', { 'text': 'example search' }) 67 | if response['status_code'] == 200: 68 | print(response['body']) 69 | ``` 70 | 71 | You can send the data as `application/json` instead of `x-www-form-urlencoded` by setting option `post_content_type` as json. 72 | 73 | ```python 74 | import json 75 | response = api.post('https://httpbin.org/post', json.dumps({ 'some_json': 'with some value' }), { 'post_content_type': 'json' }) 76 | if response['status_code'] == 200: 77 | print(response['body']) 78 | ``` 79 | 80 | ### Javascript requests 81 | 82 | If you need to scrape any website built with Javascript like React, Angular, Vue, etc. You just need to pass your javascript token and use the same calls. Note that only `.get` is available for javascript and not `.post`. 83 | 84 | ```python 85 | api = CrawlingAPI({ 'token': 'YOUR_JAVASCRIPT_TOKEN' }) 86 | ``` 87 | 88 | ```python 89 | response = api.get('https://www.nfl.com') 90 | if response['status_code'] == 200: 91 | print(response['body']) 92 | ``` 93 | 94 | Same way you can pass javascript additional options. 95 | 96 | ```python 97 | response = api.get('https://www.freelancer.com', { 'page_wait': 5000 }) 98 | if response['status_code'] == 200: 99 | print(response['body']) 100 | ``` 101 | 102 | ## Original status 103 | 104 | You can always get the original status and crawlbase status from the response. Read the [Crawlbase documentation](https://crawlbase.com/docs) to learn more about those status. 105 | 106 | ```python 107 | response = api.get('https://craiglist.com') 108 | print(response['headers']['original_status']) 109 | print(response['headers']['pc_status']) 110 | ``` 111 | 112 | If you have questions or need help using the library, please open an issue or [contact us](https://crawlbase.com/contact). 113 | 114 | ## Scraper API 115 | 116 | The usage of the Scraper API is very similar, just change the class name to initialize. 117 | 118 | ```python 119 | scraper_api = ScraperAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 120 | 121 | response = scraper_api.get('https://www.amazon.com/DualSense-Wireless-Controller-PlayStation-5/dp/B08FC6C75Y/') 122 | if response['status_code'] == 200: 123 | print(response['json']['name']) # Will print the name of the Amazon product 124 | ``` 125 | 126 | ## Leads API 127 | 128 | To find email leads you can use the leads API, you can check the full [API documentation](https://crawlbase.com/docs/leads-api/) if needed. 129 | 130 | ```python 131 | leads_api = LeadsAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 132 | 133 | response = leads_api.get_from_domain('microsoft.com') 134 | 135 | if response['status_code'] == 200: 136 | print(response['json']['leads']) 137 | ``` 138 | 139 | ## Screenshots API 140 | 141 | Initialize with your Screenshots API token and call the `get` method. 142 | 143 | ```python 144 | screenshots_api = ScreenshotsAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 145 | response = screenshots_api.get('https://www.apple.com') 146 | if response['status_code'] == 200: 147 | print(response['headers']['success']) 148 | print(response['headers']['url']) 149 | print(response['headers']['remaining_requests']) 150 | print(response['file']) 151 | ``` 152 | 153 | or specifying a file path 154 | 155 | ```python 156 | screenshots_api = ScreenshotsAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 157 | response = screenshots_api.get('https://www.apple.com', { 'save_to_path': 'apple.jpg' }) 158 | if response['status_code'] == 200: 159 | print(response['headers']['success']) 160 | print(response['headers']['url']) 161 | print(response['headers']['remaining_requests']) 162 | print(response['file']) 163 | ``` 164 | 165 | or if you set `store=true` then `screenshot_url` is set in the returned headers 166 | 167 | ```python 168 | screenshots_api = ScreenshotsAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 169 | response = screenshots_api.get('https://www.apple.com', { 'store': 'true' }) 170 | if response['status_code'] == 200: 171 | print(response['headers']['success']) 172 | print(response['headers']['url']) 173 | print(response['headers']['remaining_requests']) 174 | print(response['file']) 175 | print(response['headers']['screenshot_url']) 176 | ``` 177 | 178 | Note that `screenshots_api.get(url, options)` method accepts an [options](https://crawlbase.com/docs/screenshots-api/parameters) 179 | 180 | ## Storage API 181 | 182 | Initialize the Storage API using your private token. 183 | 184 | ```python 185 | storage_api = StorageAPI({ 'token': 'YOUR_NORMAL_TOKEN' }) 186 | ``` 187 | 188 | Pass the [url](https://crawlbase.com/docs/storage-api/parameters/#url) that you want to get from [Crawlbase Storage](https://crawlbase.com/dashboard/storage). 189 | 190 | ```python 191 | response = storage_api.get('https://www.apple.com') 192 | if response['status_code'] == 200: 193 | print(response['headers']['original_status']) 194 | print(response['headers']['pc_status']) 195 | print(response['headers']['url']) 196 | print(response['headers']['rid']) 197 | print(response['headers']['stored_at']) 198 | print(response['body']) 199 | ``` 200 | 201 | or you can use the [RID](https://crawlbase.com/docs/storage-api/parameters/#rid) 202 | 203 | ```python 204 | response = storage_api.get('RID_REPLACE') 205 | if response['status_code'] == 200: 206 | print(response['headers']['original_status']) 207 | print(response['headers']['pc_status']) 208 | print(response['headers']['url']) 209 | print(response['headers']['rid']) 210 | print(response['headers']['stored_at']) 211 | print(response['body']) 212 | ``` 213 | 214 | Note: One of the two RID or URL must be sent. So both are optional but it's mandatory to send one of the two. 215 | 216 | ### [Delete](https://crawlbase.com/docs/storage-api/delete/) request 217 | 218 | To delete a storage item from your storage area, use the correct RID 219 | 220 | ```python 221 | if storage_api.delete('RID_REPLACE'): 222 | print('delete success') 223 | else: 224 | print('Unable to delete') 225 | ``` 226 | 227 | ### [Bulk](https://crawlbase.com/docs/storage-api/bulk/) request 228 | 229 | To do a bulk request with a list of RIDs, please send the list of rids as an array 230 | 231 | ```python 232 | response = storage_api.bulk(['RID1', 'RID2', 'RID3', ...]) 233 | if response['status_code'] == 200: 234 | for item in response['json']: 235 | print(item['original_status']) 236 | print(item['pc_status']) 237 | print(item['url']) 238 | print(item['rid']) 239 | print(item['stored_at']) 240 | print(item['body']) 241 | ``` 242 | 243 | ### [RIDs](https://crawlbase.com/docs/storage-api/rids) request 244 | 245 | To request a bulk list of RIDs from your storage area 246 | 247 | ```python 248 | rids = storage_api.rids() 249 | print(rids) 250 | ``` 251 | 252 | You can also specify a limit as a parameter 253 | 254 | ```python 255 | storage_api.rids(100) 256 | ``` 257 | 258 | ### [Total Count](https://crawlbase.com/docs/storage-api/total_count) 259 | 260 | To get the total number of documents in your storage area 261 | 262 | ```python 263 | total_count = storage_api.totalCount() 264 | print(total_count) 265 | ``` 266 | 267 | ## Custom timeout 268 | 269 | If you need to use a custom timeout, you can pass it to the class instance creation like the following: 270 | 271 | ```python 272 | api = CrawlingAPI({ 'token': 'TOKEN', 'timeout': 120 }) 273 | ``` 274 | 275 | Timeout is in seconds. 276 | 277 | --- 278 | 279 | Copyright 2025 Crawlbase 280 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------