23 |
24 |
61 |
62 |
67 |
68 |
69 |
70 |
71 |
--------------------------------------------------------------------------------
/test/fixtures/captcha.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
25 |
60 |
26 |
33 |
34 |
35 |
36 | 27 | Error 28 | 1006 29 | Ray ID: 19400a3d29e30f8d 30 |
31 |Access denied
32 |
37 |
46 |
47 |
57 |
58 |
59 |
38 |
45 |
39 |
42 |
43 |
44 | What happened?
40 |The owner of this website (site.com.com) has banned your IP address (91.91.111.11).
41 |
25 |
26 |
118 |
119 |
120 |
121 |
122 |
127 |
128 |
129 |
130 |
--------------------------------------------------------------------------------
/test/fixtures/cf_recaptcha_01_12_2019.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
27 |
117 |
28 |
31 |
32 | One more step
29 |Please complete the security check to access site.com
30 |
33 |
83 |
84 |
34 |
82 |
35 |
81 |
36 |
72 |
73 |
37 |
69 |
70 |
71 |
74 |
80 |
75 |
76 |
77 |
78 |
79 |
85 |
101 |
102 |
114 |
115 |
116 |
86 |
100 |
87 |
91 |
92 | Why do I have to complete a CAPTCHA?
88 | 89 |Completing the CAPTCHA proves you are a human and gives you temporary access to the web property.
90 |
93 |
99 | What can I do to prevent this in the future?
94 | 95 |If you are on a personal connection, like at home, you can run an anti-virus scan on your device to make sure it is not infected with malware.
96 | 97 |If you are at an office or shared network, you can ask the network administrator to run a scan across the network looking for misconfigured or infected devices.
98 |
28 |
29 |
112 |
113 |
118 |
119 |
120 |
121 |
146 |
147 |
148 |
--------------------------------------------------------------------------------
/test/fixtures/cf_recaptcha_15_04_2019.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
30 |
111 |
31 |
34 |
35 | One more step
32 |Please complete the security check to access www.cloudflare.com
33 |
36 |
72 |
73 |
37 |
71 |
38 |
70 |
39 |
40 |
61 |
62 |
41 |
57 |
58 |
59 |
60 |
63 |
69 |
64 |
65 |
66 |
67 |
68 |
74 |
96 |
97 |
98 |
108 |
109 |
110 |
75 |
95 |
76 |
80 |
81 | Why do I have to complete a CAPTCHA?
77 | 78 |Completing the CAPTCHA proves you are a human and gives you temporary access to the web property.
79 |
82 |
94 | What can I do to prevent this in the future?
83 | 84 | 85 |If you are on a personal connection, like at home, you can run an anti-virus scan on your device to make sure it is not infected with malware.
86 | 87 |If you are at an office or shared network, you can ask the network administrator to run a scan across the network looking for misconfigured or infected devices.
88 | 89 | 90 |Another way to prevent getting this page in the future is to use Privacy Pass. You may need to download version 2.0 now from the Chrome Web Store.
91 | 92 | 93 |
28 |
29 |
154 |
155 |
160 |
161 |
162 |
163 |
--------------------------------------------------------------------------------
/test/fixtures/invalid_js_challenge.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
30 |
153 |
31 |
34 |
35 | One more step
32 |Please complete the security check to access example-site.dev
33 |
36 |
118 |
119 |
37 |
117 |
38 |
116 |
39 |
40 |
107 |
108 |
41 |
57 |
58 |
104 |
105 |
106 |
109 |
115 |
110 |
111 |
112 |
113 |
114 |
120 |
138 |
139 |
140 |
150 |
151 |
152 |
121 |
137 |
122 |
126 |
127 | Why do I have to complete a CAPTCHA?
123 | 124 |Completing the CAPTCHA proves you are a human and gives you temporary access to the web property.
125 |
128 |
136 | What can I do to prevent this in the future?
129 | 130 | 131 |If you are on a personal connection, like at home, you can run an anti-virus scan on your device to make sure it is not infected with malware.
132 | 133 |If you are at an office or shared network, you can ask the network administrator to run a scan across the network looking for misconfigured or infected devices.
134 | 135 ||
46 |
47 |
48 |
54 |
58 |
59 |
60 |
61 |
62 | |
63 |
50 | |
52 |
53 |
54 |
65 |
66 |
71 |
72 |
73 |
74 |
75 | DDoS protection by Cloudflare
76 |
79 | 77 | Ray ID: 4834ce407815974a 78 | |
80 |
81 |
|
52 |
53 |
54 |
65 |
66 |
71 |
72 |
73 |
74 |
75 | DDoS protection by Cloudflare
76 |
79 | 77 | Ray ID: 4834ce66ab7b9706 78 | |
80 |
81 |
|
51 |
52 |
53 |
63 |
68 |
69 |
70 |
71 |
72 | DDoS protection by CloudFlare
73 |
76 | 74 | Ray ID: 2b05d3393e872d77 75 | |
77 |
|
66 |
67 |
68 |
79 |
80 |
86 |
87 |
88 |
89 |
90 |
91 | |
92 |
93 |
|
52 |
53 |
54 |
66 |
71 |
72 |
73 |
74 |
75 | DDoS protection by Cloudflare
76 |
78 | 77 | |
79 |
80 |
|
52 |
53 |
54 |
65 |
66 |
72 |
73 |
74 |
75 | |
76 |
77 |
|
85 |
86 |
89 |
101 |
102 |
108 |
109 |
110 |
111 |
113 |
114 |
115 |
120 | |
121 |
122 |
|
66 |
67 |
68 |
80 |
81 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 | DDoS protection by Cloudflare
95 |
98 | 96 | Ray ID: 53cb1af29bc6c2d6 97 | |
99 |
100 |
The email is [email protected] 14 |
15 | 16 | 17 | -------------------------------------------------------------------------------- /test/fixtures/requested_page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |The email is ' + EMAIL + '
'); 43 | const enc = genHTML('The email is ' + protection + '
'); 44 | 45 | expect(decode(enc)).to.equal(raw); 46 | }); 47 | 48 | it('should replace spans that have a data-cfemail attribute', function () { 49 | const protection = '[email protected]'; 51 | 52 | const raw = genHTML('The email is ' + EMAIL + '
'); 53 | const enc = genHTML('The email is ' + protection + '
'); 54 | 55 | expect(decode(enc)).to.equal(raw); 56 | }); 57 | 58 | it('should be space agnostic', function () { 59 | const protection = '\n The email
is ' + EMAIL + '\r\n
\n The email
is ' + protection + '\r\n
The email is ' + protection + '
'); 72 | 73 | expect(decode(enc)).to.equal(enc); 74 | }); 75 | 76 | it('should not replace malformed html', function () { 77 | const protection = '\n<\n'; 78 | const enc = genHTML('The email is ' + protection + '
'); 79 | 80 | expect(decode(enc)).to.equal(enc); 81 | }); 82 | 83 | it('should account for self-closing nodes', function () { 84 | const protection = 'test'; 85 | 86 | expect(decode(protection)).to.equal(EMAIL + 'test'); 87 | }); 88 | 89 | it('should update href attribute values', function () { 90 | const protection = ''; 91 | 92 | const raw = genHTML(''); 93 | const enc = genHTML(protection); 94 | 95 | expect(decode(enc)).to.equal(raw); 96 | }); 97 | }); 98 | -------------------------------------------------------------------------------- /test/test-errors.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | /* eslint-env node, mocha */ 3 | 'use strict'; 4 | 5 | const cloudscraper = require('../index'); 6 | const request = require('request-promise'); 7 | const helper = require('./helper'); 8 | const brotli = require('../lib/brotli'); 9 | const errors = require('../errors'); 10 | 11 | const sinon = require('sinon'); 12 | const expect = require('chai').expect; 13 | const assert = require('chai').assert; 14 | 15 | describe('Cloudscraper', function () { 16 | let sandbox; 17 | let Request; 18 | let uri; 19 | 20 | before(function (done) { 21 | helper.listen(function () { 22 | uri = helper.resolve('/test'); 23 | 24 | // Speed up tests 25 | cloudscraper.defaultParams.cloudflareTimeout = 1; 26 | done(); 27 | }); 28 | }); 29 | 30 | after(function () { 31 | helper.server.close(); 32 | }); 33 | 34 | beforeEach(function () { 35 | // Prepare stubbed Request 36 | sandbox = sinon.createSandbox(); 37 | Request = sandbox.spy(request, 'Request'); 38 | }); 39 | 40 | afterEach(function () { 41 | helper.reset(); 42 | sandbox.restore(); 43 | }); 44 | 45 | it('should return error if it was thrown by request', function (done) { 46 | helper.router.get('/test', function (req, res) { 47 | res.endAbruptly(); 48 | }); 49 | 50 | const promise = cloudscraper.get(uri, function (error) { 51 | expect(error).to.be.instanceOf(errors.RequestError); 52 | expect(error.error).to.be.an('error'); 53 | expect(error).to.have.property('errorType', 0); 54 | 55 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 56 | expect(promise).to.be.rejectedWith(errors.RequestError).and.notify(done); 57 | }); 58 | }); 59 | 60 | it('should return error if cloudflare response is empty', function (done) { 61 | helper.router.get('/test', function (req, res) { 62 | res.cloudflare().status(504).end(); 63 | }); 64 | 65 | const promise = cloudscraper.get(uri, function (error) { 66 | // errorType 1, means captcha is served 67 | expect(error).to.be.instanceOf(errors.CloudflareError); 68 | expect(error).to.have.property('error', 504); 69 | expect(error).to.have.property('errorType', 2); 70 | expect(error.message).to.be.equal('504, Gateway Timeout'); 71 | 72 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 73 | 74 | expect(error.response.body).to.be.eql(Buffer.alloc(0)); 75 | expect(promise).to.be.rejectedWith(errors.CloudflareError).and.notify(done); 76 | }); 77 | }); 78 | 79 | it('should return error if captcha is served by cloudflare', function (done) { 80 | helper.router.get('/test', function (req, res) { 81 | res.sendChallenge('captcha.html'); 82 | }); 83 | 84 | const promise = cloudscraper.get(uri, function (error) { 85 | // errorType 1, means captcha is served 86 | expect(error).to.be.instanceOf(errors.CaptchaError); 87 | expect(error).to.have.property('error', 'captcha'); 88 | expect(error).to.have.property('errorType', 1); 89 | 90 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 91 | expect(promise).to.be.rejectedWith(errors.CaptchaError).and.notify(done); 92 | }); 93 | }); 94 | 95 | it('should return error if cloudflare returned some inner error', function (done) { 96 | // https://support.cloudflare.com/hc/en-us/sections/200820298-Error-Pages 97 | // Error codes: 1012, 1011, 1002, 1000, 1004, 1010, 1006, 1007, 1008 98 | // Error codes can also be the same as the HTTP status code in the 5xx range. 99 | 100 | helper.router.get('/test', function (req, res) { 101 | res.cloudflare().status(500).sendFixture('access_denied.html'); 102 | }); 103 | 104 | const promise = cloudscraper.get(uri, function (error) { 105 | // errorType 2, means inner cloudflare error 106 | expect(error).to.be.instanceOf(errors.CloudflareError); 107 | expect(error).to.have.property('error', 1006); 108 | expect(error.message).to.equal('1006, Access Denied: Your IP address has been banned'); 109 | expect(error).to.have.property('errorType', 2); 110 | 111 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 112 | expect(promise).to.be.rejectedWith(errors.CloudflareError).and.notify(done); 113 | }); 114 | }); 115 | 116 | it('should add a description to 5xx range cloudflare errors', function (done) { 117 | const html = helper.getFixture('access_denied.html').toString('utf8'); 118 | 119 | helper.router.get('/test', function (req, res) { 120 | res.cloudflare().status(504).send(html.replace('1006', '504')); 121 | }); 122 | 123 | const promise = cloudscraper.get(uri, function (error) { 124 | // errorType 2, means inner cloudflare error 125 | expect(error).to.be.instanceOf(errors.CloudflareError); 126 | expect(error).to.have.property('error', 504); 127 | expect(error.message).to.equal('504, Gateway Timeout'); 128 | expect(error).to.have.property('errorType', 2); 129 | 130 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 131 | expect(promise).to.be.rejectedWith(errors.CloudflareError).and.notify(done); 132 | }); 133 | }); 134 | 135 | it('should not error if error description is unavailable', function (done) { 136 | const html = helper.getFixture('access_denied.html').toString('utf8'); 137 | 138 | helper.router.get('/test', function (req, res) { 139 | res.cloudflare().status(500).send(html.replace('1006', '5111')); 140 | }); 141 | 142 | const promise = cloudscraper.get(uri, function (error) { 143 | // errorType 2, means inner cloudflare error 144 | expect(error).to.be.instanceOf(errors.CloudflareError); 145 | expect(error).to.have.property('error', 5111); 146 | expect(error.message).to.equal('5111'); 147 | expect(error).to.have.property('errorType', 2); 148 | 149 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 150 | expect(promise).to.be.rejectedWith(errors.CloudflareError).and.notify(done); 151 | }); 152 | }); 153 | 154 | it('should return error if cf presented more than 3 challenges in a row', function (done) { 155 | helper.router.get('*', function (req, res) { 156 | res.sendChallenge('js_challenge_09_06_2016.html'); 157 | }); 158 | 159 | // The expected params for all subsequent calls to Request 160 | const expectedParams = helper.extendParams({ 161 | uri: helper.resolve('/cdn-cgi/l/chk_jschl') 162 | }); 163 | 164 | // Perform less strict matching on headers and qs to simplify this test 165 | Object.assign(expectedParams, { 166 | headers: sinon.match.object, 167 | qs: sinon.match.object 168 | }); 169 | 170 | const promise = cloudscraper.get(uri, function (error) { 171 | expect(error).to.be.instanceOf(errors.CloudflareError); 172 | expect(error).to.have.property('error', 'Cloudflare challenge loop'); 173 | expect(error).to.have.property('errorType', 4); 174 | 175 | assert.equal(Request.callCount, 4, 'Request call count'); 176 | expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); 177 | 178 | const total = helper.defaultParams.challengesToSolve + 1; 179 | // noinspection ES6ConvertVarToLetConst 180 | for (var i = 1; i < total; i++) { 181 | // Decrement the number of challengesToSolve to match actual params 182 | expectedParams.challengesToSolve -= 1; 183 | expect(Request.getCall(i)).to.be.calledWithExactly(expectedParams); 184 | } 185 | 186 | expect(promise).to.be.rejectedWith(errors.CloudflareError).and.notify(done); 187 | }); 188 | }); 189 | 190 | it('should return error if body is undefined', function (done) { 191 | helper.router.get('/test', function (req, res) { 192 | res.status(503).end(); 193 | }); 194 | 195 | const expectedParams = helper.extendParams({ json: true }); 196 | const options = { uri: uri, json: true }; 197 | 198 | const promise = cloudscraper.get(options, function (error) { 199 | expect(error).to.be.instanceOf(errors.RequestError); 200 | expect(error).to.have.property('error', null); 201 | expect(error).to.have.property('errorType', 0); 202 | 203 | assert.equal(error.response.statusCode, 503, 'status code'); 204 | 205 | expect(error.response.body).to.be.equal(undefined); 206 | expect(Request).to.be.calledOnceWithExactly(expectedParams); 207 | expect(promise).to.be.rejectedWith(errors.RequestError).and.notify(done); 208 | }); 209 | }); 210 | 211 | (brotli.isAvailable ? it.skip : it)('should return error if content-type is brotli and missing dep', function (done) { 212 | // Brotli compressed JSON: {"a":"test"} 213 | const compressed = Buffer.from([ 214 | 0x8b, 0x05, 0x80, 0x7b, 0x22, 0x61, 0x22, 0x3a, 215 | 0x22, 0x74, 0x65, 0x73, 0x74, 0x22, 0x7d, 0x03 216 | ]); 217 | 218 | helper.router.get('/test', function (req, res) { 219 | res.set('content-encoding', 'br'); 220 | res.status(503).end(compressed, 'binary'); 221 | }); 222 | 223 | const expectedParams = helper.extendParams({ json: true }); 224 | const options = { uri: uri, json: true }; 225 | 226 | const promise = cloudscraper.get(options, function (error) { 227 | expect(error).to.be.instanceOf(errors.RequestError); 228 | expect(error).to.have.property('error').that.is.ok; 229 | expect(error).to.have.property('errorType', 0); 230 | 231 | assert.equal(error.response.statusCode, 503, 'status code'); 232 | 233 | assert(Buffer.isBuffer(error.response.body), 'response type'); 234 | expect(error.response.body).to.be.eql(compressed); 235 | expect(Request).to.be.calledOnceWithExactly(expectedParams); 236 | expect(promise).to.be.rejectedWith(errors.RequestError).and.notify(done); 237 | }); 238 | }); 239 | 240 | it('should return error if challenge page failed to be parsed', function (done) { 241 | helper.router.get('/test', function (req, res) { 242 | res.sendChallenge('invalid_js_challenge.html'); 243 | }); 244 | 245 | const promise = cloudscraper.get(uri, function (error) { 246 | expect(error).to.be.instanceOf(errors.ParserError); 247 | expect(error).to.have.property('error').that.is.ok; 248 | expect(error).to.have.property('errorType', 3); 249 | 250 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 251 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 252 | }); 253 | }); 254 | 255 | it('should return error if js challenge has error during evaluation', function (done) { 256 | const html = helper.getFixture('js_challenge_03_12_2018_1.html'); 257 | 258 | helper.router.get('/test', function (req, res) { 259 | // Adds a syntax error near the end of line 37 260 | res.cloudflare().status(503).send(html.replace(/\.toFixed/gm, '..toFixed')); 261 | }); 262 | 263 | const promise = cloudscraper.get(uri, function (error) { 264 | expect(error).to.be.instanceOf(errors.ParserError); 265 | expect(error).to.have.property('error').that.is.an('error'); 266 | expect(error).to.have.property('errorType', 3); 267 | expect(error.message).to.include('Challenge evaluation failed'); 268 | 269 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 270 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 271 | }); 272 | }); 273 | 274 | it('should return error if pass extraction fails', function (done) { 275 | const html = helper.getFixture('js_challenge_03_12_2018_1.html'); 276 | 277 | helper.router.get('/test', function (req, res) { 278 | res.cloudflare().status(503).send(html.replace(/name="pass"/gm, '')); 279 | }); 280 | 281 | const promise = cloudscraper.get(uri, function (error) { 282 | expect(error).to.be.instanceOf(errors.ParserError); 283 | expect(error).to.have.property('error', 'Attribute (pass) value extraction failed'); 284 | expect(error).to.have.property('errorType', 3); 285 | 286 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 287 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 288 | }); 289 | }); 290 | 291 | it('should return error if challengeId extraction fails', function (done) { 292 | const html = helper.getFixture('js_challenge_03_12_2018_1.html'); 293 | 294 | helper.router.get('/test', function (req, res) { 295 | res.cloudflare().status(503).send(html.replace(/name="jschl_vc"/gm, '')); 296 | }); 297 | 298 | const promise = cloudscraper.get(uri, function (error) { 299 | expect(error).to.be.instanceOf(errors.ParserError); 300 | expect(error).to.have.property('error', 'challengeId (jschl_vc) extraction failed'); 301 | expect(error).to.have.property('errorType', 3); 302 | 303 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 304 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 305 | }); 306 | }); 307 | 308 | it('should return error if challenge answer is not a number', function (done) { 309 | const html = helper.getFixture('js_challenge_03_12_2018_1.html'); 310 | 311 | helper.router.get('/test', function (req, res) { 312 | res.cloudflare().status(503) 313 | .send(html.replace(/a.value.*/, 'a.value="abc" + t.length')); 314 | }); 315 | 316 | const promise = cloudscraper.get(uri, function (error) { 317 | expect(error).to.be.instanceOf(errors.ParserError); 318 | expect(error).to.have.property('error', 'Challenge answer is not a number'); 319 | expect(error).to.have.property('errorType', 3); 320 | 321 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 322 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 323 | }); 324 | }); 325 | 326 | it('should return error if it was thrown by request when solving challenge', function (done) { 327 | helper.router 328 | .get('/test', function (req, res) { 329 | res.sendChallenge('js_challenge_21_05_2015.html'); 330 | }) 331 | .get('/cdn-cgi/l/chk_jschl', function (req, res) { 332 | res.endAbruptly(); 333 | }); 334 | 335 | const promise = cloudscraper.get(uri, function (error) { 336 | // errorType 0, a connection error for example 337 | expect(error).to.be.instanceOf(errors.RequestError); 338 | expect(error.error).to.be.an('error'); 339 | expect(error).to.have.property('errorType', 0); 340 | 341 | expect(Request).to.be.calledTwice; 342 | expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); 343 | expect(promise).to.be.rejectedWith(errors.RequestError).and.notify(done); 344 | }); 345 | }); 346 | 347 | it('should properly handle a case when after a challenge another one is returned', function (done) { 348 | helper.router 349 | .get('/test', function (req, res) { 350 | res.sendChallenge('js_challenge_09_06_2016.html'); 351 | }) 352 | .get('/cdn-cgi/l/chk_jschl', function (req, res) { 353 | res.sendChallenge('captcha.html'); 354 | }); 355 | 356 | // Second call to request.get returns recaptcha 357 | const expectedParams = helper.extendParams({ 358 | uri: helper.resolve('/cdn-cgi/l/chk_jschl'), 359 | challengesToSolve: 2 360 | }); 361 | 362 | // Perform less strict matching on headers and qs to simplify this test 363 | Object.assign(expectedParams, { 364 | headers: sinon.match.object, 365 | qs: sinon.match.object 366 | }); 367 | 368 | const promise = cloudscraper.get(uri, function (error) { 369 | // errorType 1, means captcha is served 370 | expect(error).to.be.instanceOf(errors.CaptchaError); 371 | expect(error).to.have.property('error', 'captcha'); 372 | expect(error).to.have.property('errorType', 1); 373 | 374 | expect(Request).to.be.calledTwice; 375 | expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams); 376 | expect(Request.secondCall).to.be.calledWithExactly(expectedParams); 377 | expect(promise).to.be.rejectedWith(errors.CaptchaError).and.notify(done); 378 | }); 379 | }); 380 | 381 | it('should return error if challenge page cookie extraction fails', function (done) { 382 | const html = helper.getFixture('sucuri_waf_18_08_2016.html').toString('utf8'); 383 | 384 | helper.router.get('/test', function (req, res) { 385 | // The cookie extraction codes looks for the `S` variable assignment 386 | res.cloudflare().status(503).send(html.replace(/S=/gm, 'Z=')); 387 | }); 388 | 389 | const promise = cloudscraper.get(uri, function (error) { 390 | expect(error).to.be.instanceOf(errors.ParserError); 391 | expect(error).to.have.property('error', 'Cookie code extraction failed'); 392 | expect(error).to.have.property('errorType', 3); 393 | 394 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 395 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 396 | }); 397 | }); 398 | 399 | it('should throw a TypeError if callback is not a function', function (done) { 400 | const spy = sinon.spy(function () { 401 | // request-promise always provides a callback so change requester 402 | const options = { uri: uri, requester: require('request') }; 403 | cloudscraper.get(options); 404 | }); 405 | 406 | expect(spy).to.throw(TypeError, /Expected a callback function/); 407 | done(); 408 | }); 409 | 410 | it('should throw a TypeError if requester is not a function', function (done) { 411 | const spy = sinon.spy(function () { 412 | cloudscraper.get({ requester: null }); 413 | }); 414 | 415 | expect(spy).to.throw(TypeError, /`requester` option .*function/); 416 | done(); 417 | }); 418 | 419 | it('should throw a TypeError if challengesToSolve is not a number', function (done) { 420 | const spy = sinon.spy(function () { 421 | const options = { uri: uri, challengesToSolve: 'abc' }; 422 | 423 | cloudscraper.get(options); 424 | }); 425 | 426 | expect(spy).to.throw(TypeError, /`challengesToSolve` option .*number/); 427 | done(); 428 | }); 429 | 430 | it('should throw a TypeError if cloudflareMaxTimeout is not a number', function (done) { 431 | const spy = sinon.spy(function () { 432 | const options = { uri: uri, cloudflareMaxTimeout: 'abc' }; 433 | 434 | cloudscraper.get(options, function () {}); 435 | }); 436 | 437 | expect(spy).to.throw(TypeError, /`cloudflareMaxTimeout` option .*number/); 438 | done(); 439 | }); 440 | 441 | it('should return error if cookie setting code evaluation fails', function (done) { 442 | // Change the cookie setting code so the vm will throw an error 443 | const html = helper.getFixture('sucuri_waf_18_08_2016.html').toString('utf8'); 444 | const b64 = Buffer.from('throw new Error(\'vm eval failed\');').toString('base64'); 445 | 446 | helper.router.get('/test', function (req, res) { 447 | res.cloudflare().status(503).send(html.replace(/S='([^']+)'/, 'S=\'' + b64 + '\'')); 448 | }); 449 | 450 | const promise = cloudscraper.get(uri, function (error) { 451 | expect(error).to.be.instanceOf(errors.ParserError); 452 | expect(error).to.have.property('error').that.is.an('error'); 453 | expect(error).to.have.property('errorType', 3); 454 | expect(error.message).to.include('vm eval failed'); 455 | 456 | expect(Request).to.be.calledOnceWithExactly(helper.defaultParams); 457 | expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done); 458 | }); 459 | }); 460 | 461 | it('should not error if Error.captureStackTrace is undefined', function () { 462 | const desc = Object.getOwnPropertyDescriptor(Error, 'captureStackTrace'); 463 | 464 | Object.defineProperty(Error, 'captureStackTrace', { 465 | configurable: true, 466 | value: undefined 467 | }); 468 | 469 | const spy = sinon.spy(function () { 470 | throw new errors.RequestError(); 471 | }); 472 | 473 | try { 474 | expect(spy).to.throw(errors.RequestError); 475 | } finally { 476 | Object.defineProperty(Error, 'captureStackTrace', desc); 477 | } 478 | }); 479 | }); 480 | -------------------------------------------------------------------------------- /test/test-headers.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | /* eslint-env node, mocha */ 3 | 'use strict'; 4 | 5 | const sinon = require('sinon'); 6 | const expect = require('chai').expect; 7 | 8 | describe('Headers (lib)', function () { 9 | const { getDefaultHeaders, caseless } = require('../lib/headers'); 10 | const browsers = require('../lib/browsers'); 11 | 12 | it('should export getDefaultHeaders function', function () { 13 | expect(getDefaultHeaders).to.be.a('function'); 14 | }); 15 | 16 | it('should export caseless function', function () { 17 | expect(caseless).to.be.a('function'); 18 | }); 19 | 20 | it('caseless should return an object with lowercase keys', function () { 21 | sinon.assert.match(caseless({ AbC: 'foobar' }), { abc: 'foobar' }); 22 | }); 23 | 24 | it('getDefaultHeaders should always return an object with user agent', function () { 25 | for (let i = 0; i < 100; i++) { 26 | sinon.assert.match(getDefaultHeaders(), { 'User-Agent': sinon.match.string }); 27 | } 28 | 29 | browsers.chrome.forEach(function (options) { 30 | try { 31 | expect(options['User-Agent']).to.be.an('array'); 32 | expect(options['User-Agent'].length).to.be.above(0); 33 | } catch (error) { 34 | error.message += '\n\n' + JSON.stringify(options, null, 2); 35 | throw error; 36 | } 37 | }); 38 | }); 39 | 40 | it('getDefaultHeaders should always retain insertion order', function () { 41 | for (let keys, i = 0; i < 100; i++) { 42 | keys = Object.keys(getDefaultHeaders({ Host: 'foobar' })); 43 | expect(keys[0]).to.equal('Host'); 44 | expect(keys[1]).to.equal('Connection'); 45 | } 46 | 47 | for (let keys, i = 0; i < 100; i++) { 48 | keys = Object.keys(getDefaultHeaders({ Host: 'foobar', 'N/A': null })); 49 | expect(keys[0]).to.equal('Host'); 50 | expect(keys[1]).to.equal('N/A'); 51 | expect(keys[2]).to.equal('Connection'); 52 | } 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /test/test-rp.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable promise/always-return,promise/catch-or-return,promise/no-callback-in-promise */ 2 | /* eslint-env node, mocha */ 3 | 'use strict'; 4 | 5 | const cloudscraper = require('../index'); 6 | const request = require('request-promise'); 7 | const helper = require('./helper'); 8 | 9 | const sinon = require('sinon'); 10 | const expect = require('chai').expect; 11 | 12 | describe('Cloudscraper', function () { 13 | let sandbox; 14 | let Request; 15 | let uri; 16 | 17 | const requestedPage = helper.getFixture('requested_page.html'); 18 | 19 | before(function (done) { 20 | helper.listen(function () { 21 | uri = helper.resolve('/test'); 22 | 23 | // Speed up tests 24 | cloudscraper.defaultParams.cloudflareTimeout = 1; 25 | done(); 26 | }); 27 | }); 28 | 29 | after(function () { 30 | helper.server.close(); 31 | }); 32 | 33 | beforeEach(function () { 34 | // Prepare stubbed Request 35 | sandbox = sinon.createSandbox(); 36 | Request = sandbox.spy(request, 'Request'); 37 | }); 38 | 39 | afterEach(function () { 40 | helper.reset(); 41 | sandbox.restore(); 42 | }); 43 | 44 | it('should resolve with response body', function () { 45 | helper.router.get('/test', function (req, res) { 46 | res.send(requestedPage); 47 | }); 48 | 49 | const expectedParams = helper.extendParams({ callback: undefined }); 50 | 51 | return cloudscraper.get(uri).then(function (body) { 52 | expect(Request).to.be.calledOnceWithExactly(expectedParams); 53 | expect(body).to.be.equal(requestedPage); 54 | }); 55 | }); 56 | 57 | it('should resolve with full response', function () { 58 | helper.router.get('/test', function (req, res) { 59 | res.send(requestedPage); 60 | }); 61 | 62 | const expectedParams = helper.extendParams({ 63 | callback: undefined, 64 | resolveWithFullResponse: true 65 | }); 66 | 67 | // The method is implicitly GET 68 | delete expectedParams.method; 69 | 70 | const options = { 71 | uri: uri, 72 | resolveWithFullResponse: true 73 | }; 74 | 75 | return cloudscraper(options).then(function (response) { 76 | expect(Request).to.be.calledOnceWithExactly(expectedParams); 77 | expect(response.body).to.be.equal(requestedPage); 78 | }); 79 | }); 80 | 81 | // The helper calls the fake request callback synchronously. This results 82 | // in the promise being rejected before we catch it in the test. 83 | // This can be noticeable if we return the promise instead of calling done. 84 | it('should define catch', function (done) { 85 | helper.router.get('/test', function (req, res) { 86 | res.endAbruptly(); 87 | }); 88 | 89 | let caught = false; 90 | 91 | cloudscraper(uri) 92 | .catch(function () { 93 | caught = true; 94 | }) 95 | .then(function () { 96 | if (caught) done(); 97 | }); 98 | }); 99 | 100 | it('should define finally', function (done) { 101 | helper.router.get('/test', function (req, res) { 102 | res.endAbruptly(); 103 | }); 104 | 105 | let caught = false; 106 | 107 | cloudscraper(uri) 108 | .then(function () { 109 | caught = true; 110 | }) 111 | .finally(function () { 112 | if (!caught) done(); 113 | }) 114 | .catch(function () {}); 115 | }); 116 | }); 117 | -------------------------------------------------------------------------------- /test/test-sandbox.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | /* eslint-env node, mocha */ 3 | 'use strict'; 4 | 5 | const sandbox = require('../lib/sandbox'); 6 | const expect = require('chai').expect; 7 | 8 | describe('Sandbox (lib)', function () { 9 | it('should export Context', function () { 10 | expect(sandbox.Context).to.be.a('function'); 11 | }); 12 | 13 | it('should export eval', function () { 14 | expect(sandbox.eval).to.be.a('function'); 15 | expect(sandbox.eval('0')).to.equal(0); 16 | expect(sandbox.eval('true')).to.be.true; 17 | expect(sandbox.eval('undefined')).to.equal(undefined); 18 | expect(sandbox.eval('NaN')).to.be.a('number'); 19 | expect(String(sandbox.eval('NaN'))).to.equal('NaN'); 20 | }); 21 | 22 | it('new Context() should return an object', function () { 23 | expect(new sandbox.Context()).to.be.an('object'); 24 | }); 25 | 26 | it('Context() should define atob', function () { 27 | const ctx = new sandbox.Context(); 28 | 29 | expect(ctx.atob).to.be.a('function'); 30 | expect(ctx.atob('YWJj')).to.equal('abc'); 31 | expect(sandbox.eval('atob("YWJj")', ctx)).to.equal('abc'); 32 | }); 33 | 34 | it('Context() should define location.reload', function () { 35 | const ctx = new sandbox.Context(); 36 | 37 | expect(sandbox.eval('location.reload()', ctx)).to.equal(undefined); 38 | }); 39 | 40 | it('Context() should define document.createElement', function () { 41 | let ctx = new sandbox.Context(); 42 | let pseudoElement = { firstChild: { href: 'http:///' } }; 43 | 44 | expect(sandbox.eval('document.createElement("a")', ctx)).to.eql(pseudoElement); 45 | 46 | ctx = new sandbox.Context({ hostname: 'test.com' }); 47 | pseudoElement = { firstChild: { href: 'http://test.com/' } }; 48 | 49 | expect(sandbox.eval('document.createElement("a")', ctx)).to.eql(pseudoElement); 50 | }); 51 | 52 | it('Context() should define document.geElementById', function () { 53 | let ctx = new sandbox.Context(); 54 | expect(sandbox.eval('document.getElementById()', ctx)).to.be.null; 55 | 56 | // Missing element 57 | ctx = new sandbox.Context(); 58 | expect(sandbox.eval('document.getElementById("foobar")', ctx)).to.be.null; 59 | 60 | // Double quotes 61 | ctx = new sandbox.Context({ body: 'foobar
' });
62 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: 'foobar' });
63 |
64 | // Single quotes
65 | ctx = new sandbox.Context({ body: 'foobar
' });
66 | expect(sandbox.eval('document.getElementById(\'test\')', ctx)).eql({ innerHTML: 'foobar' });
67 |
68 | // Empty
69 | ctx = new sandbox.Context({ body: '' });
70 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: '' });
71 |
72 | // Space agnostic tests
73 | ctx = new sandbox.Context({ body: '\nabc\n\n
' });
74 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: '\nabc\n\n' });
75 |
76 | ctx = new sandbox.Context({ body: ' abc
' });
77 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: ' abc ' });
78 |
79 | ctx = new sandbox.Context({ body: 'foo="bar" id=\'test\' a=b > abc <' });
80 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: ' abc ' });
81 |
82 | // Cache test
83 | ctx = new sandbox.Context({ body: 'foobar
' });
84 | expect(sandbox.eval('document.getElementById("test")', ctx)).eql({ innerHTML: 'foobar' });
85 | });
86 | });
87 |
--------------------------------------------------------------------------------
/test/test-timeout.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lenivene/cloudflare-scraper-nodejs/bad27a5a553ba55814b7955e559f656a0615de00/test/test-timeout.js
--------------------------------------------------------------------------------