├── python ├── ipdecodo.py ├── facebookpage.py ├── facebookgroup.py └── facebookpost.py ├── php ├── ipdecodo.php ├── facebookpage.php ├── facebookgroup.php └── facebookpost.php ├── nodejs ├── ipdecodo.js ├── facebookpage.js ├── facebookgroup.js └── facebookpost.js ├── LICENSE.md └── README.md /python/ipdecodo.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = "https://scraper-api.decodo.com/v2/scrape" 4 | 5 | payload = { 6 | "target": "universal", 7 | "parse": False, 8 | "url": "https://ip.decodo.com/" 9 | } 10 | headers = { 11 | "Accept": "application/json", 12 | "Content-Type": "application/json", 13 | "Authorization": "Basic AUTH" 14 | } 15 | 16 | response = requests.post(url, json=payload, headers=headers) 17 | 18 | print(response.text) 19 | -------------------------------------------------------------------------------- /python/facebookpage.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = "https://scraper-api.decodo.com/v2/scrape" 4 | 5 | payload = { 6 | "target": "universal", 7 | "parse": False, 8 | "url": "https://www.facebook.com/ladygaga" 9 | } 10 | headers = { 11 | "Accept": "application/json", 12 | "Content-Type": "application/json", 13 | "Authorization": "Basic AUTH" 14 | } 15 | 16 | response = requests.post(url, json=payload, headers=headers) 17 | 18 | print(response.text) 19 | -------------------------------------------------------------------------------- /php/ipdecodo.php: -------------------------------------------------------------------------------- 1 | request('POST', 'https://scraper-api.decodo.com/v2/scrape', [ 7 | 'body' => '{"target":"universal","parse":false,"url":"https://ip.decodo.com/"}', 8 | 'headers' => [ 9 | 'Accept' => 'application/json', 10 | 'Authorization' => 'Basic AUTH', 11 | 'Content-Type' => 'application/json', 12 | ], 13 | ]); 14 | 15 | echo $response->getBody(); 16 | -------------------------------------------------------------------------------- /python/facebookgroup.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = "https://scraper-api.decodo.com/v2/scrape" 4 | 5 | payload = { 6 | "target": "universal", 7 | "parse": False, 8 | "url": "https://www.facebook.com/groups/1394454774138066" 9 | } 10 | headers = { 11 | "Accept": "application/json", 12 | "Content-Type": "application/json", 13 | "Authorization": "Basic AUTH" 14 | } 15 | 16 | response = requests.post(url, json=payload, headers=headers) 17 | 18 | print(response.text) 19 | -------------------------------------------------------------------------------- /php/facebookpage.php: -------------------------------------------------------------------------------- 1 | request('POST', 'https://scraper-api.decodo.com/v2/scrape', [ 7 | 'body' => '{"target":"universal","parse":false,"url":"https://www.facebook.com/ladygaga"}', 8 | 'headers' => [ 9 | 'Accept' => 'application/json', 10 | 'Authorization' => 'Basic AUTH', 11 | 'Content-Type' => 'application/json', 12 | ], 13 | ]); 14 | 15 | echo $response->getBody(); 16 | -------------------------------------------------------------------------------- /php/facebookgroup.php: -------------------------------------------------------------------------------- 1 | request('POST', 'https://scraper-api.decodo.com/v2/scrape', [ 7 | 'body' => '{"target":"universal","parse":false,"url":"https://www.facebook.com/groups/1394454774138066"}', 8 | 'headers' => [ 9 | 'Accept' => 'application/json', 10 | 'Authorization' => 'Basic AUTH', 11 | 'Content-Type' => 'application/json', 12 | ], 13 | ]); 14 | 15 | echo $response->getBody(); 16 | -------------------------------------------------------------------------------- /python/facebookpost.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | url = "https://scraper-api.decodo.com/v2/scrape" 4 | 5 | payload = { 6 | "target": "universal", 7 | "parse": False, 8 | "headless": "html", 9 | "url": "https://www.facebook.com/zuck/posts/pfbid0HeY54v4LMcv2EMxDz5RvnWaR6swsGFWikzUbrsEFtvxu9n4GCx7zA2YTM69XdiYnl" 10 | } 11 | headers = { 12 | "Accept": "application/json", 13 | "Content-Type": "application/json", 14 | "Authorization": "Basic AUTH" 15 | } 16 | 17 | response = requests.post(url, json=payload, headers=headers) 18 | 19 | print(response.text) 20 | -------------------------------------------------------------------------------- /php/facebookpost.php: -------------------------------------------------------------------------------- 1 | request('POST', 'https://scraper-api.decodo.com/v2/scrape', [ 7 | 'body' => '{"target":"universal","parse":false,"headless":"html","url":"https://www.facebook.com/zuck/posts/pfbid0HeY54v4LMcv2EMxDz5RvnWaR6swsGFWikzUbrsEFtvxu9n4GCx7zA2YTM69XdiYnl"}', 8 | 'headers' => [ 9 | 'Accept' => 'application/json', 10 | 'Authorization' => 'Basic AUTH', 11 | 'Content-Type' => 'application/json', 12 | ], 13 | ]); 14 | 15 | echo $response->getBody(); 16 | -------------------------------------------------------------------------------- /nodejs/ipdecodo.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | 3 | const username = 'YOUR_USERNAME'; 4 | const password = 'YOUR_PASSWORD'; 5 | 6 | const options = { 7 | method: 'POST', 8 | url: 'https://scraper-api.decodo.com/v2/scrape', 9 | headers: { 10 | 'Content-Type': 'application/json', 11 | 'Authorization': 'Basic ' + Buffer.from(username + ':' + password).toString('base64') 12 | }, 13 | body: JSON.stringify({ 14 | target: 'universal', 15 | parse: false, 16 | url: 'https://ip.decodo.com/' 17 | }) 18 | }; 19 | 20 | request(options, function (error, response, body) { 21 | if (error) throw new Error(error); 22 | 23 | console.log(body); 24 | }); 25 | -------------------------------------------------------------------------------- /nodejs/facebookpage.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | 3 | const username = 'YOUR_USERNAME'; 4 | const password = 'YOUR_PASSWORD'; 5 | 6 | const options = { 7 | method: 'POST', 8 | url: 'https://scraper-api.decodo.com/v2/scrape', 9 | headers: { 10 | 'Content-Type': 'application/json', 11 | 'Authorization': 'Basic ' + Buffer.from(username + ':' + password).toString('base64') 12 | }, 13 | body: JSON.stringify({ 14 | target: 'universal', 15 | parse: false, 16 | url: 'https://www.facebook.com/ladygaga' 17 | }) 18 | }; 19 | 20 | request(options, function (error, response, body) { 21 | if (error) throw new Error(error); 22 | 23 | console.log(body); 24 | }); 25 | -------------------------------------------------------------------------------- /nodejs/facebookgroup.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | 3 | const username = 'YOUR_USERNAME'; 4 | const password = 'YOUR_PASSWORD'; 5 | 6 | const options = { 7 | method: 'POST', 8 | url: 'https://scraper-api.decodo.com/v2/scrape', 9 | headers: { 10 | 'Content-Type': 'application/json', 11 | 'Authorization': 'Basic ' + Buffer.from(username + ':' + password).toString('base64') 12 | }, 13 | body: JSON.stringify({ 14 | target: 'universal', 15 | parse: false, 16 | url: 'https://www.facebook.com/groups/1394454774138066' 17 | }) 18 | }; 19 | 20 | request(options, function (error, response, body) { 21 | if (error) throw new Error(error); 22 | 23 | console.log(body); 24 | }); 25 | -------------------------------------------------------------------------------- /nodejs/facebookpost.js: -------------------------------------------------------------------------------- 1 | const request = require('request'); 2 | 3 | const username = 'YOUR_USERNAME'; 4 | const password = 'YOUR_PASSWORD'; 5 | 6 | const options = { 7 | method: 'POST', 8 | url: 'https://scraper-api.decodo.com/v2/scrape', 9 | headers: { 10 | 'Content-Type': 'application/json', 11 | 'Authorization': 'Basic ' + Buffer.from(username + ':' + password).toString('base64') 12 | }, 13 | body: JSON.stringify({ 14 | target: 'universal', 15 | parse: false, 16 | url: 'https://www.facebook.com/zuck/posts/pfbid0HeY54v4LMcv2EMxDz5RvnWaR6swsGFWikzUbrsEFtvxu9n4GCx7zA2YTM69XdiYnl' 17 | }) 18 | }; 19 | 20 | request(options, function (error, response, body) { 21 | if (error) throw new Error(error); 22 | 23 | console.log(body); 24 | }); 25 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Decodo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web Scraping API 2 | 3 |

4 | 5 |

6 | 7 | 8 | [![](https://dcbadge.vercel.app/api/server/Ja8dqKgvbZ)](https://discord.gg/Ja8dqKgvbZ) 9 | 10 | ## List of contents 11 | - [Introduction](#introduction) 12 | - [Authentication](#authentication) 13 | - [Scraping](#scraping) 14 | - [Headless](#headless) 15 | - [Facebook](#facebook) 16 | - [Parameters](#parameters) 17 | - [Response Codes](#response-codes) 18 | - [License](#license) 19 | 20 | ## Introduction 21 | 22 | With [our Web Scraping API](https://decodo.com/scraping/web), you can scrape various websites en masse. 23 | 24 | ## Authentication 25 | 26 | Once you have an active Web Scraping API subscription, you can try sending a request right from the dashboard Web Scraping API > Authentication method tab simply by entering your username, password, and clicking on Generate. You will also see an example of curl request generated right below your entered ``` user:pass. ``` 27 | 28 | Note that this is only an example with preset values to get you on the right track for forming your own request, meaning you will not be able to change the request values in the dashboard itself – that will have to be done in your code. 29 | 30 | 31 | ## Scraping 32 | 33 | ### You can use universal parameter as your target and supply any URL you want, which will return the HTML of the targeted URL. 34 | 35 | API Link: https://scraper-api.decodo.com/v2/scrape 36 | 37 | ```http 38 | POST /scrape 39 | ``` 40 | 41 | ### Target: ```universal``` (not parseable) 42 | Required parameters: ```url``` (ip.decodo.com in this example) 43 | 44 | | Parameter | Type | Description | 45 | | :-------- | :------- | :------------------------- | 46 | | `url` | `url` | Target URL | 47 | | `target` | `string` | Scraping target - ```universal``` | 48 | 49 | ### Examples 50 | 51 | | Programming Language | Example location | Download | 52 | | -------------------- | ------------------------ | -------- | 53 | | Python | [python/ipdecodo.py](https://github.com/Decodo/Web-Scraping-API/blob/main/python/ipDecodo.py) |``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/python/ipdecodo.py > ipdecodo.py ``` | 54 | | PHP | [php/ipdecodo.php](https://github.com/Decodo/Web-Scraping-API/blob/main/php/ipDecodo.php) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/php/ipdecodo.php > ipdecodo.php ``` | 55 | | Node.js | [nodejs/ipdecodo.js](https://github.com/Decodo/Web-Scraping-API/blob/main/nodejs/ipDecodo.js) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/nodejs/ipdecodo.js > ipdecodo.js ``` | 56 | 57 | 58 | ### Response 59 | 60 | ```http 61 | { 62 | "results": [ 63 | { 64 | "content": "Your Ip is: 213.87.163.6", 65 | "status_code": 200, 66 | "url": "https://ip.decodo.com/", 67 | "task_id": "6971034977135771649", 68 | "created_at": "2022-09-01 09:24:14", 69 | "updated_at": "2022-09-01 09:24:17" 70 | } 71 | ] 72 | } 73 | ``` 74 | 75 | ## Headless 76 | 77 | **Not seeing the results you wanted?** 78 | 79 | Try enabling JavaScript rendering using the ```headless``` parameter. - [Parameters](#parameters) 80 | 81 | This parameter renders JavaScript on the target website making more data available for scraping. 82 | 83 | 84 | ## Facebook 85 | 86 | ### Facebook Page 87 | 88 | ### Target: ```universal``` (not parseable) 89 | Required parameters: ```url``` 90 | 91 | | Parameter | Type | Description | 92 | | :-------- | :------- | :------------------------- | 93 | | `url` | `url` | Target URL | 94 | | `target` | `string` | Scraping target - ```universal``` | 95 | 96 | ### Response 97 | 98 | ```http 99 | 100 | { 101 | "results": [ 102 | { 103 | "content": " Facebook page content" 104 | "status_code": 200, 105 | "url": "https://www.facebook.com/ladygaga", 106 | "task_id": "6972452679540839425", 107 | "created_at": "2022-09-05 07:17:40", 108 | "updated_at": "2022-09-05 07:17:45" 109 | } 110 | ] 111 | } 112 | ``` 113 | ### Examples 114 | 115 | | Programming Language | Example location | Download | 116 | | -------------------- | ------------------------ | -------- | 117 | | Python | [python/facebookpage.py](https://github.com/Decodo/Web-Scraping-API/blob/main/python/facebookpage.py) |``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/python/facebookpage.py > facebookpage.py ``` | 118 | | PHP | [php/facebookpage.php](https://github.com/Decodo/Web-Scraping-API/blob/main/php/facebookpage.php) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/php/facebookpage.php > facebookpage.php ``` | 119 | | Node.js | [nodejs/facebookpage.js](https://github.com/Decodo/Web-Scraping-API/blob/main/nodejs/facebookpage.js) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/nodejs/facebookpage.js > facebookpage.js ``` | 120 | 121 | 122 | ### Facebook Post 123 | 124 | ### Target: ```universal``` (not parseable) 125 | Required parameters: ```url``` 126 | 127 | | Parameter | Type | Description | 128 | | :-------- | :------- | :------------------------- | 129 | | `url` | `url` | Target URL | 130 | | `target` | `string` | Scraping target - ```universal``` | 131 | | `headless` | `string` | Javascript rendering - ```html``` | 132 | 133 | ### Response 134 | 135 | 136 | ```http 137 | { 138 | "results": [ 139 | { 140 | "content": " Facebook page content" 141 | "status_code": 200, 142 | "url": "https://www.facebook.com/zuck/posts/pfbid0HeY54v4LMcv2EMxDz5RvnWaR6swsGFWikzUbrsEFtvxu9n4GCx7zA2YTM69XdiYnl", 143 | "task_id": "6972484278999372801", 144 | "created_at": "2022-09-05 09:23:14", 145 | "updated_at": "2022-09-05 09:23:32" 146 | } 147 | ] 148 | } 149 | ``` 150 | 151 | ### Examples 152 | 153 | | Programming Language | Example location | Download | 154 | | -------------------- | ------------------------ | -------- | 155 | | Python | [python/facebookpost.py](https://github.com/Decodo/Web-Scraping-API/blob/main/python/facebookpost.py) |``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/python/facebookpost.py > facebookpost.py ``` | 156 | | PHP | [php/facebookpost.php](https://github.com/Decodo/Web-Scraping-API/blob/main/php/facebookpost.php) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/php/facebookpost.php > facebookpost.php ``` | 157 | | Node.js | [nodejs/facebookpost.js](https://github.com/Decodo/Web-Scraping-API/blob/main/nodejs/facebookpost.js) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/nodejs/facebookpost.js > facebookpost.js ``` | 158 | 159 | 160 | ### Facebook Group 161 | 162 | ### Target: ```universal``` (not parseable) 163 | Required parameters: ```url``` 164 | 165 | | Parameter | Type | Description | 166 | | :-------- | :------- | :------------------------- | 167 | | `url` | `url` | Target URL | 168 | | `target` | `string` | Scraping target - ```universal``` | 169 | 170 | ### Response 171 | 172 | ```http 173 | { 174 | "results": [ 175 | { 176 | "content": " Facebook page content" 177 | "status_code": 200, 178 | "url": "https://www.facebook.com/groups/1394454774138066", 179 | "task_id": "6972486765374350337", 180 | "created_at": "2022-09-05 09:33:07", 181 | "updated_at": "2022-09-05 09:33:33" 182 | } 183 | ] 184 | } 185 | ``` 186 | ### Examples 187 | 188 | | Programming Language | Example location | Download | 189 | | -------------------- | ------------------------ | -------- | 190 | | Python | [python/facebookgroup.py](https://github.com/Decodo/Web-Scraping-API/blob/main/python/facebookgroup.py) |``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/python/facebookgroup.py > facebookgroup.py ``` | 191 | | PHP | [php/facebookgroup.php](https://github.com/Decodo/Web-Scraping-API/blob/main/php/facebookgroup.php) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/php/facebookgroup.php > facebookgroup.php ``` | 192 | | Node.js | [nodejs/facebookgroup.js](https://github.com/Decodo/Web-Scraping-API/blob/main/nodejs/facebookgroup.js) | ``` curl https://raw.githubusercontent.com/Decodo/Web-Scraping-API/main/nodejs/facebookgroup.js > facebookgroup.js ``` | 193 | 194 | ## Parameters 195 | 196 | | Parameter | Type | Description | 197 | | :-------- | :------- | :------------------------- | 198 | | `target` | `string` | Data source. (universal) | 199 | | `url` | `url` | Direct URL (link) | 200 | | `locale` | `string` | This will change the web interface language. Example: – en-US – en-GB | 201 | | `geo` | `string` | The geographical location that the result depends on. Full Country names required | 202 | | `device_type` | `string` | Device type and browser. Supported: ```desktop```, ```desktop_chrome```, ```desktop_firefox```, ```mobile```, ```mobile_android```, ```mobile_ios```. | 203 | | `headless` | `string` | Enable JavaScript rendering. Supported: ```html```, ```png``` | 204 | 205 | ## Response Codes 206 | 207 | ### HTTP Response Codes 208 | 209 | | Response | Description | Solution | 210 | | :-------- | :------- | :------------------------- | 211 | | **200** - Success | Server has replied and given requested response. | Celebrate! | 212 | | **204** - No content | Job not completed yet. | Wait a few seconds before trying again. | 213 | | **400** - Multiple error messages | Bad structure of the request. | Re-check your request to make sure it is in the correct format. | 214 | | **401** - Invalid / not provided authorization header (client not found) | Incorrect login credentials or missing authorization. | Re-check your provided credentials for authorization. | 215 | | **403** - Forbidden | Your account does not have access to this resource. | Make sure the target is supported by us | 216 | | **404** - Not found | Your target was not found. | Re-check your targeted URL. | 217 | | **429** - Too many requests | Exceeded rate limit for your subscription. | Make sure you still have at least one request left. Wait a couple minutes and try again. If you are encountering the error often – chat with us to see if your rate limit can be increased. | 218 | | **500** - Internal error | Service unavailable, possibly due to some issues we are encountering. | Wait a couple minutes and send another request. Contact us for more information. | 219 | | **524** - Timeout | Service unavailable, possibly due to some issues we are encountering. | Wait a couple minutes and send another request. Contact us for more information. | 220 | 221 | ## License 222 | 223 | 224 | All code is released under [MIT License](https://github.com/Decodo/Decodo/blob/master/LICENSE) 225 | --------------------------------------------------------------------------------