├── .idea
├── .gitignore
├── misc.xml
├── modules.xml
├── serverless-search-engine.iml
└── vcs.xml
├── Readme.md
├── frontend
├── .gitignore
├── client
│ └── dist
│ │ └── index.html
├── handler.js
├── package-lock.json
├── package.json
└── serverless.yml
├── query-service
├── .gitignore
├── package-lock.json
├── package.json
├── serverless.yml
└── src
│ └── functions
│ ├── addKeywords.js
│ └── query.js
└── spider-service
├── .gitignore
├── package-lock.json
├── package.json
├── serverless.yml
└── src
└── functions
├── scraper.js
└── startScraping.js
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 | # Datasource local storage ignored files
5 | /dataSources/
6 | /dataSources.local.xml
7 | # Editor-based HTTP Client requests
8 | /httpRequests/
9 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/serverless-search-engine.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/Readme.md:
--------------------------------------------------------------------------------
1 | **📦 Archived - This repository is archived and preserved for reference only. No updates, issues, or pull requests will be accepted. If you have questions, please reach out to our support team.**
2 |
3 | ---
4 |
5 | # Serverless Search Engine
6 |
7 | This project is a part of the Serverless Live series of streams hosted on YouTube. The playlist of content can be found here: https://www.youtube.com/playlist?list=PLIIjEI2fYC-B_bnRmGFX-gPNujchODE-X
8 |
--------------------------------------------------------------------------------
/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | # package directories
2 | node_modules
3 | jspm_packages
4 |
5 | # Serverless directories
6 | .serverless
--------------------------------------------------------------------------------
/frontend/client/dist/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Query Page
6 |
7 |
8 | Serverless Search Engine
9 |
10 |
11 |
12 |
17 |
35 |
36 |
37 |
--------------------------------------------------------------------------------
/frontend/handler.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports.hello = async event => {
4 | return {
5 | statusCode: 200,
6 | body: JSON.stringify(
7 | {
8 | message: 'Go Serverless v1.0! Your function executed successfully!',
9 | input: event,
10 | },
11 | null,
12 | 2
13 | ),
14 | };
15 |
16 | // Use this code if you don't use the http event with the LAMBDA-PROXY integration
17 | // return { message: 'Go Serverless v1.0! Your function executed successfully!', event };
18 | };
19 |
--------------------------------------------------------------------------------
/frontend/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "frontend",
3 | "version": "0.1.0",
4 | "lockfileVersion": 1,
5 | "requires": true,
6 | "dependencies": {
7 | "ansi-bgblack": {
8 | "version": "0.1.1",
9 | "resolved": "https://registry.npmjs.org/ansi-bgblack/-/ansi-bgblack-0.1.1.tgz",
10 | "integrity": "sha1-poulAHiHcBtqr74/oNrf36juPKI=",
11 | "dev": true,
12 | "requires": {
13 | "ansi-wrap": "0.1.0"
14 | }
15 | },
16 | "ansi-bgblue": {
17 | "version": "0.1.1",
18 | "resolved": "https://registry.npmjs.org/ansi-bgblue/-/ansi-bgblue-0.1.1.tgz",
19 | "integrity": "sha1-Z73ATtybm1J4lp2hlt6j11yMNhM=",
20 | "dev": true,
21 | "requires": {
22 | "ansi-wrap": "0.1.0"
23 | }
24 | },
25 | "ansi-bgcyan": {
26 | "version": "0.1.1",
27 | "resolved": "https://registry.npmjs.org/ansi-bgcyan/-/ansi-bgcyan-0.1.1.tgz",
28 | "integrity": "sha1-WEiUJWAL3p9VBwaN2Wnr/bUP52g=",
29 | "dev": true,
30 | "requires": {
31 | "ansi-wrap": "0.1.0"
32 | }
33 | },
34 | "ansi-bggreen": {
35 | "version": "0.1.1",
36 | "resolved": "https://registry.npmjs.org/ansi-bggreen/-/ansi-bggreen-0.1.1.tgz",
37 | "integrity": "sha1-TjGRJIUplD9DIelr8THRwTgWr0k=",
38 | "dev": true,
39 | "requires": {
40 | "ansi-wrap": "0.1.0"
41 | }
42 | },
43 | "ansi-bgmagenta": {
44 | "version": "0.1.1",
45 | "resolved": "https://registry.npmjs.org/ansi-bgmagenta/-/ansi-bgmagenta-0.1.1.tgz",
46 | "integrity": "sha1-myhDLAduqpmUGGcqPvvhk5HCx6E=",
47 | "dev": true,
48 | "requires": {
49 | "ansi-wrap": "0.1.0"
50 | }
51 | },
52 | "ansi-bgred": {
53 | "version": "0.1.1",
54 | "resolved": "https://registry.npmjs.org/ansi-bgred/-/ansi-bgred-0.1.1.tgz",
55 | "integrity": "sha1-p2+Sg4OCukMpCmwXeEJPmE1vEEE=",
56 | "dev": true,
57 | "requires": {
58 | "ansi-wrap": "0.1.0"
59 | }
60 | },
61 | "ansi-bgwhite": {
62 | "version": "0.1.1",
63 | "resolved": "https://registry.npmjs.org/ansi-bgwhite/-/ansi-bgwhite-0.1.1.tgz",
64 | "integrity": "sha1-ZQRlE3elim7OzQMxmU5IAljhG6g=",
65 | "dev": true,
66 | "requires": {
67 | "ansi-wrap": "0.1.0"
68 | }
69 | },
70 | "ansi-bgyellow": {
71 | "version": "0.1.1",
72 | "resolved": "https://registry.npmjs.org/ansi-bgyellow/-/ansi-bgyellow-0.1.1.tgz",
73 | "integrity": "sha1-w/4usIzUdmSAKeaHTRWgs49h1E8=",
74 | "dev": true,
75 | "requires": {
76 | "ansi-wrap": "0.1.0"
77 | }
78 | },
79 | "ansi-black": {
80 | "version": "0.1.1",
81 | "resolved": "https://registry.npmjs.org/ansi-black/-/ansi-black-0.1.1.tgz",
82 | "integrity": "sha1-9hheiJNgslRaHsUMC/Bj/EMDJFM=",
83 | "dev": true,
84 | "requires": {
85 | "ansi-wrap": "0.1.0"
86 | }
87 | },
88 | "ansi-blue": {
89 | "version": "0.1.1",
90 | "resolved": "https://registry.npmjs.org/ansi-blue/-/ansi-blue-0.1.1.tgz",
91 | "integrity": "sha1-FbgEmQ6S/JyoxUds6PaZd3wh7b8=",
92 | "dev": true,
93 | "requires": {
94 | "ansi-wrap": "0.1.0"
95 | }
96 | },
97 | "ansi-bold": {
98 | "version": "0.1.1",
99 | "resolved": "https://registry.npmjs.org/ansi-bold/-/ansi-bold-0.1.1.tgz",
100 | "integrity": "sha1-PmOVCvWswq4uZw5vZ96xFdGl9QU=",
101 | "dev": true,
102 | "requires": {
103 | "ansi-wrap": "0.1.0"
104 | }
105 | },
106 | "ansi-colors": {
107 | "version": "0.2.0",
108 | "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-0.2.0.tgz",
109 | "integrity": "sha1-csMd4qDZoszQysMMyYI+6y9kNLU=",
110 | "dev": true,
111 | "requires": {
112 | "ansi-bgblack": "^0.1.1",
113 | "ansi-bgblue": "^0.1.1",
114 | "ansi-bgcyan": "^0.1.1",
115 | "ansi-bggreen": "^0.1.1",
116 | "ansi-bgmagenta": "^0.1.1",
117 | "ansi-bgred": "^0.1.1",
118 | "ansi-bgwhite": "^0.1.1",
119 | "ansi-bgyellow": "^0.1.1",
120 | "ansi-black": "^0.1.1",
121 | "ansi-blue": "^0.1.1",
122 | "ansi-bold": "^0.1.1",
123 | "ansi-cyan": "^0.1.1",
124 | "ansi-dim": "^0.1.1",
125 | "ansi-gray": "^0.1.1",
126 | "ansi-green": "^0.1.1",
127 | "ansi-grey": "^0.1.1",
128 | "ansi-hidden": "^0.1.1",
129 | "ansi-inverse": "^0.1.1",
130 | "ansi-italic": "^0.1.1",
131 | "ansi-magenta": "^0.1.1",
132 | "ansi-red": "^0.1.1",
133 | "ansi-reset": "^0.1.1",
134 | "ansi-strikethrough": "^0.1.1",
135 | "ansi-underline": "^0.1.1",
136 | "ansi-white": "^0.1.1",
137 | "ansi-yellow": "^0.1.1",
138 | "lazy-cache": "^2.0.1"
139 | }
140 | },
141 | "ansi-cyan": {
142 | "version": "0.1.1",
143 | "resolved": "https://registry.npmjs.org/ansi-cyan/-/ansi-cyan-0.1.1.tgz",
144 | "integrity": "sha1-U4rlKK+JgvKK4w2G8vF0VtJgmHM=",
145 | "dev": true,
146 | "requires": {
147 | "ansi-wrap": "0.1.0"
148 | }
149 | },
150 | "ansi-dim": {
151 | "version": "0.1.1",
152 | "resolved": "https://registry.npmjs.org/ansi-dim/-/ansi-dim-0.1.1.tgz",
153 | "integrity": "sha1-QN5MYDqoCG2Oeoa4/5mNXDbu/Ww=",
154 | "dev": true,
155 | "requires": {
156 | "ansi-wrap": "0.1.0"
157 | }
158 | },
159 | "ansi-gray": {
160 | "version": "0.1.1",
161 | "resolved": "https://registry.npmjs.org/ansi-gray/-/ansi-gray-0.1.1.tgz",
162 | "integrity": "sha1-KWLPVOyXksSFEKPetSRDaGHvclE=",
163 | "dev": true,
164 | "requires": {
165 | "ansi-wrap": "0.1.0"
166 | }
167 | },
168 | "ansi-green": {
169 | "version": "0.1.1",
170 | "resolved": "https://registry.npmjs.org/ansi-green/-/ansi-green-0.1.1.tgz",
171 | "integrity": "sha1-il2al55FjVfEDjNYCzc5C44Q0Pc=",
172 | "dev": true,
173 | "requires": {
174 | "ansi-wrap": "0.1.0"
175 | }
176 | },
177 | "ansi-grey": {
178 | "version": "0.1.1",
179 | "resolved": "https://registry.npmjs.org/ansi-grey/-/ansi-grey-0.1.1.tgz",
180 | "integrity": "sha1-WdmLasK6GfilF5jphT+6eDOaM8E=",
181 | "dev": true,
182 | "requires": {
183 | "ansi-wrap": "0.1.0"
184 | }
185 | },
186 | "ansi-hidden": {
187 | "version": "0.1.1",
188 | "resolved": "https://registry.npmjs.org/ansi-hidden/-/ansi-hidden-0.1.1.tgz",
189 | "integrity": "sha1-7WpMSY0rt8uyidvyqNHcyFZ/rg8=",
190 | "dev": true,
191 | "requires": {
192 | "ansi-wrap": "0.1.0"
193 | }
194 | },
195 | "ansi-inverse": {
196 | "version": "0.1.1",
197 | "resolved": "https://registry.npmjs.org/ansi-inverse/-/ansi-inverse-0.1.1.tgz",
198 | "integrity": "sha1-tq9Fgm/oJr+1KKbHmIV5Q1XM0mk=",
199 | "dev": true,
200 | "requires": {
201 | "ansi-wrap": "0.1.0"
202 | }
203 | },
204 | "ansi-italic": {
205 | "version": "0.1.1",
206 | "resolved": "https://registry.npmjs.org/ansi-italic/-/ansi-italic-0.1.1.tgz",
207 | "integrity": "sha1-EEdDRj9iXBQqA2c5z4XtpoiYbyM=",
208 | "dev": true,
209 | "requires": {
210 | "ansi-wrap": "0.1.0"
211 | }
212 | },
213 | "ansi-magenta": {
214 | "version": "0.1.1",
215 | "resolved": "https://registry.npmjs.org/ansi-magenta/-/ansi-magenta-0.1.1.tgz",
216 | "integrity": "sha1-BjtboW+z8j4c/aKwfAqJ3hHkMK4=",
217 | "dev": true,
218 | "requires": {
219 | "ansi-wrap": "0.1.0"
220 | }
221 | },
222 | "ansi-red": {
223 | "version": "0.1.1",
224 | "resolved": "https://registry.npmjs.org/ansi-red/-/ansi-red-0.1.1.tgz",
225 | "integrity": "sha1-jGOPnRCAgAo1PJwoyKgcpHBdlGw=",
226 | "dev": true,
227 | "requires": {
228 | "ansi-wrap": "0.1.0"
229 | }
230 | },
231 | "ansi-regex": {
232 | "version": "3.0.0",
233 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
234 | "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
235 | "dev": true
236 | },
237 | "ansi-reset": {
238 | "version": "0.1.1",
239 | "resolved": "https://registry.npmjs.org/ansi-reset/-/ansi-reset-0.1.1.tgz",
240 | "integrity": "sha1-5+cSksPH3c1NYu9KbHwFmAkRw7c=",
241 | "dev": true,
242 | "requires": {
243 | "ansi-wrap": "0.1.0"
244 | }
245 | },
246 | "ansi-strikethrough": {
247 | "version": "0.1.1",
248 | "resolved": "https://registry.npmjs.org/ansi-strikethrough/-/ansi-strikethrough-0.1.1.tgz",
249 | "integrity": "sha1-2Eh3FAss/wfRyT685pkE9oiF5Wg=",
250 | "dev": true,
251 | "requires": {
252 | "ansi-wrap": "0.1.0"
253 | }
254 | },
255 | "ansi-underline": {
256 | "version": "0.1.1",
257 | "resolved": "https://registry.npmjs.org/ansi-underline/-/ansi-underline-0.1.1.tgz",
258 | "integrity": "sha1-38kg9Ml7WXfqFi34/7mIMIqqcaQ=",
259 | "dev": true,
260 | "requires": {
261 | "ansi-wrap": "0.1.0"
262 | }
263 | },
264 | "ansi-white": {
265 | "version": "0.1.1",
266 | "resolved": "https://registry.npmjs.org/ansi-white/-/ansi-white-0.1.1.tgz",
267 | "integrity": "sha1-nHe3wZPF7pkuYBHTbsTJIbRXiUQ=",
268 | "dev": true,
269 | "requires": {
270 | "ansi-wrap": "0.1.0"
271 | }
272 | },
273 | "ansi-wrap": {
274 | "version": "0.1.0",
275 | "resolved": "https://registry.npmjs.org/ansi-wrap/-/ansi-wrap-0.1.0.tgz",
276 | "integrity": "sha1-qCJQ3bABXponyoLoLqYDu/pF768=",
277 | "dev": true
278 | },
279 | "ansi-yellow": {
280 | "version": "0.1.1",
281 | "resolved": "https://registry.npmjs.org/ansi-yellow/-/ansi-yellow-0.1.1.tgz",
282 | "integrity": "sha1-y5NW8vRscy8OMZnmEClVp32oPB0=",
283 | "dev": true,
284 | "requires": {
285 | "ansi-wrap": "0.1.0"
286 | }
287 | },
288 | "arr-flatten": {
289 | "version": "1.1.0",
290 | "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
291 | "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==",
292 | "dev": true
293 | },
294 | "arr-swap": {
295 | "version": "1.0.1",
296 | "resolved": "https://registry.npmjs.org/arr-swap/-/arr-swap-1.0.1.tgz",
297 | "integrity": "sha1-FHWQ7WX8gVvAf+8Jl8Llgj1kNTQ=",
298 | "dev": true,
299 | "requires": {
300 | "is-number": "^3.0.0"
301 | },
302 | "dependencies": {
303 | "is-number": {
304 | "version": "3.0.0",
305 | "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
306 | "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
307 | "dev": true,
308 | "requires": {
309 | "kind-of": "^3.0.2"
310 | }
311 | }
312 | }
313 | },
314 | "balanced-match": {
315 | "version": "1.0.0",
316 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
317 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
318 | "dev": true
319 | },
320 | "brace-expansion": {
321 | "version": "1.1.11",
322 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
323 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
324 | "dev": true,
325 | "requires": {
326 | "balanced-match": "^1.0.0",
327 | "concat-map": "0.0.1"
328 | }
329 | },
330 | "choices-separator": {
331 | "version": "2.0.0",
332 | "resolved": "https://registry.npmjs.org/choices-separator/-/choices-separator-2.0.0.tgz",
333 | "integrity": "sha1-kv0XYxgteQM/XFxR0Lo1LlVnxpY=",
334 | "dev": true,
335 | "requires": {
336 | "ansi-dim": "^0.1.1",
337 | "debug": "^2.6.6",
338 | "strip-color": "^0.1.0"
339 | }
340 | },
341 | "clone-deep": {
342 | "version": "1.0.0",
343 | "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-1.0.0.tgz",
344 | "integrity": "sha512-hmJRX8x1QOJVV+GUjOBzi6iauhPqc9hIF6xitWRBbiPZOBb6vGo/mDRIK9P74RTKSQK7AE8B0DDWY/vpRrPmQw==",
345 | "dev": true,
346 | "requires": {
347 | "for-own": "^1.0.0",
348 | "is-plain-object": "^2.0.4",
349 | "kind-of": "^5.0.0",
350 | "shallow-clone": "^1.0.0"
351 | },
352 | "dependencies": {
353 | "kind-of": {
354 | "version": "5.1.0",
355 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
356 | "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
357 | "dev": true
358 | }
359 | }
360 | },
361 | "collection-visit": {
362 | "version": "1.0.0",
363 | "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
364 | "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
365 | "dev": true,
366 | "requires": {
367 | "map-visit": "^1.0.0",
368 | "object-visit": "^1.0.0"
369 | }
370 | },
371 | "component-emitter": {
372 | "version": "1.3.0",
373 | "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
374 | "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==",
375 | "dev": true
376 | },
377 | "concat-map": {
378 | "version": "0.0.1",
379 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
380 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
381 | "dev": true
382 | },
383 | "copy-descriptor": {
384 | "version": "0.1.1",
385 | "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
386 | "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=",
387 | "dev": true
388 | },
389 | "debug": {
390 | "version": "2.6.9",
391 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
392 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
393 | "dev": true,
394 | "requires": {
395 | "ms": "2.0.0"
396 | }
397 | },
398 | "define-property": {
399 | "version": "1.0.0",
400 | "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
401 | "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
402 | "dev": true,
403 | "requires": {
404 | "is-descriptor": "^1.0.0"
405 | }
406 | },
407 | "error-symbol": {
408 | "version": "0.1.0",
409 | "resolved": "https://registry.npmjs.org/error-symbol/-/error-symbol-0.1.0.tgz",
410 | "integrity": "sha1-Ck2uN9YA0VopukU9jvkg8YRDM/Y=",
411 | "dev": true
412 | },
413 | "extend-shallow": {
414 | "version": "2.0.1",
415 | "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
416 | "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
417 | "dev": true,
418 | "requires": {
419 | "is-extendable": "^0.1.0"
420 | }
421 | },
422 | "for-in": {
423 | "version": "1.0.2",
424 | "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
425 | "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=",
426 | "dev": true
427 | },
428 | "for-own": {
429 | "version": "1.0.0",
430 | "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
431 | "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=",
432 | "dev": true,
433 | "requires": {
434 | "for-in": "^1.0.1"
435 | }
436 | },
437 | "info-symbol": {
438 | "version": "0.1.0",
439 | "resolved": "https://registry.npmjs.org/info-symbol/-/info-symbol-0.1.0.tgz",
440 | "integrity": "sha1-J4QdcoZ920JCzWEtecEGM4gcang=",
441 | "dev": true
442 | },
443 | "is-accessor-descriptor": {
444 | "version": "1.0.0",
445 | "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
446 | "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
447 | "dev": true,
448 | "requires": {
449 | "kind-of": "^6.0.0"
450 | },
451 | "dependencies": {
452 | "kind-of": {
453 | "version": "6.0.3",
454 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
455 | "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
456 | "dev": true
457 | }
458 | }
459 | },
460 | "is-buffer": {
461 | "version": "1.1.6",
462 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
463 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
464 | "dev": true
465 | },
466 | "is-data-descriptor": {
467 | "version": "1.0.0",
468 | "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
469 | "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
470 | "dev": true,
471 | "requires": {
472 | "kind-of": "^6.0.0"
473 | },
474 | "dependencies": {
475 | "kind-of": {
476 | "version": "6.0.3",
477 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
478 | "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
479 | "dev": true
480 | }
481 | }
482 | },
483 | "is-descriptor": {
484 | "version": "1.0.2",
485 | "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
486 | "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
487 | "dev": true,
488 | "requires": {
489 | "is-accessor-descriptor": "^1.0.0",
490 | "is-data-descriptor": "^1.0.0",
491 | "kind-of": "^6.0.2"
492 | },
493 | "dependencies": {
494 | "kind-of": {
495 | "version": "6.0.3",
496 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
497 | "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
498 | "dev": true
499 | }
500 | }
501 | },
502 | "is-extendable": {
503 | "version": "0.1.1",
504 | "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
505 | "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=",
506 | "dev": true
507 | },
508 | "is-fullwidth-code-point": {
509 | "version": "2.0.0",
510 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
511 | "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
512 | "dev": true
513 | },
514 | "is-number": {
515 | "version": "6.0.0",
516 | "resolved": "https://registry.npmjs.org/is-number/-/is-number-6.0.0.tgz",
517 | "integrity": "sha512-Wu1VHeILBK8KAWJUAiSZQX94GmOE45Rg6/538fKwiloUu21KncEkYGPqob2oSZ5mUT73vLGrHQjKw3KMPwfDzg==",
518 | "dev": true
519 | },
520 | "is-plain-object": {
521 | "version": "2.0.4",
522 | "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
523 | "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
524 | "dev": true,
525 | "requires": {
526 | "isobject": "^3.0.1"
527 | }
528 | },
529 | "is-windows": {
530 | "version": "1.0.2",
531 | "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
532 | "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==",
533 | "dev": true
534 | },
535 | "is_js": {
536 | "version": "0.9.0",
537 | "resolved": "https://registry.npmjs.org/is_js/-/is_js-0.9.0.tgz",
538 | "integrity": "sha1-CrlFQFArp6+iTIVqqYVWFmnpxS0=",
539 | "dev": true
540 | },
541 | "isobject": {
542 | "version": "3.0.1",
543 | "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
544 | "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
545 | "dev": true
546 | },
547 | "kind-of": {
548 | "version": "3.2.2",
549 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
550 | "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
551 | "dev": true,
552 | "requires": {
553 | "is-buffer": "^1.1.5"
554 | }
555 | },
556 | "koalas": {
557 | "version": "1.0.2",
558 | "resolved": "https://registry.npmjs.org/koalas/-/koalas-1.0.2.tgz",
559 | "integrity": "sha1-MYQz8HQjXbePrlZhoCqMpT7ilc0=",
560 | "dev": true
561 | },
562 | "lazy-cache": {
563 | "version": "2.0.2",
564 | "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-2.0.2.tgz",
565 | "integrity": "sha1-uRkKT5EzVGlIQIWfio9whNiCImQ=",
566 | "dev": true,
567 | "requires": {
568 | "set-getter": "^0.1.0"
569 | }
570 | },
571 | "log-ok": {
572 | "version": "0.1.1",
573 | "resolved": "https://registry.npmjs.org/log-ok/-/log-ok-0.1.1.tgz",
574 | "integrity": "sha1-vqPdNqzQuKckDXhza1uXxlREozQ=",
575 | "dev": true,
576 | "requires": {
577 | "ansi-green": "^0.1.1",
578 | "success-symbol": "^0.1.0"
579 | }
580 | },
581 | "log-utils": {
582 | "version": "0.2.1",
583 | "resolved": "https://registry.npmjs.org/log-utils/-/log-utils-0.2.1.tgz",
584 | "integrity": "sha1-pMIXoN2aUFFdm5ICBgkas9TgMc8=",
585 | "dev": true,
586 | "requires": {
587 | "ansi-colors": "^0.2.0",
588 | "error-symbol": "^0.1.0",
589 | "info-symbol": "^0.1.0",
590 | "log-ok": "^0.1.1",
591 | "success-symbol": "^0.1.0",
592 | "time-stamp": "^1.0.1",
593 | "warning-symbol": "^0.1.0"
594 | }
595 | },
596 | "map-visit": {
597 | "version": "1.0.0",
598 | "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
599 | "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=",
600 | "dev": true,
601 | "requires": {
602 | "object-visit": "^1.0.0"
603 | }
604 | },
605 | "mime": {
606 | "version": "1.6.0",
607 | "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
608 | "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
609 | "dev": true
610 | },
611 | "minimatch": {
612 | "version": "3.0.4",
613 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
614 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
615 | "dev": true,
616 | "requires": {
617 | "brace-expansion": "^1.1.7"
618 | }
619 | },
620 | "mixin-object": {
621 | "version": "2.0.1",
622 | "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz",
623 | "integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=",
624 | "dev": true,
625 | "requires": {
626 | "for-in": "^0.1.3",
627 | "is-extendable": "^0.1.1"
628 | },
629 | "dependencies": {
630 | "for-in": {
631 | "version": "0.1.8",
632 | "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz",
633 | "integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE=",
634 | "dev": true
635 | }
636 | }
637 | },
638 | "ms": {
639 | "version": "2.0.0",
640 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
641 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
642 | "dev": true
643 | },
644 | "mute-stream": {
645 | "version": "0.0.7",
646 | "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
647 | "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
648 | "dev": true
649 | },
650 | "object-copy": {
651 | "version": "0.1.0",
652 | "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
653 | "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=",
654 | "dev": true,
655 | "requires": {
656 | "copy-descriptor": "^0.1.0",
657 | "define-property": "^0.2.5",
658 | "kind-of": "^3.0.3"
659 | },
660 | "dependencies": {
661 | "define-property": {
662 | "version": "0.2.5",
663 | "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
664 | "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
665 | "dev": true,
666 | "requires": {
667 | "is-descriptor": "^0.1.0"
668 | }
669 | },
670 | "is-accessor-descriptor": {
671 | "version": "0.1.6",
672 | "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
673 | "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
674 | "dev": true,
675 | "requires": {
676 | "kind-of": "^3.0.2"
677 | }
678 | },
679 | "is-data-descriptor": {
680 | "version": "0.1.4",
681 | "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
682 | "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
683 | "dev": true,
684 | "requires": {
685 | "kind-of": "^3.0.2"
686 | }
687 | },
688 | "is-descriptor": {
689 | "version": "0.1.6",
690 | "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
691 | "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
692 | "dev": true,
693 | "requires": {
694 | "is-accessor-descriptor": "^0.1.6",
695 | "is-data-descriptor": "^0.1.4",
696 | "kind-of": "^5.0.0"
697 | },
698 | "dependencies": {
699 | "kind-of": {
700 | "version": "5.1.0",
701 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
702 | "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
703 | "dev": true
704 | }
705 | }
706 | }
707 | }
708 | },
709 | "object-visit": {
710 | "version": "1.0.1",
711 | "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
712 | "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=",
713 | "dev": true,
714 | "requires": {
715 | "isobject": "^3.0.0"
716 | }
717 | },
718 | "pointer-symbol": {
719 | "version": "1.0.0",
720 | "resolved": "https://registry.npmjs.org/pointer-symbol/-/pointer-symbol-1.0.0.tgz",
721 | "integrity": "sha1-YPkRAgTqepKbYmRKITFVQ8uz1Ec=",
722 | "dev": true
723 | },
724 | "prompt-actions": {
725 | "version": "3.0.2",
726 | "resolved": "https://registry.npmjs.org/prompt-actions/-/prompt-actions-3.0.2.tgz",
727 | "integrity": "sha512-dhz2Fl7vK+LPpmnQ/S/eSut4BnH4NZDLyddHKi5uTU/2PDn3grEMGkgsll16V5RpVUh/yxdiam0xsM0RD4xvtg==",
728 | "dev": true,
729 | "requires": {
730 | "debug": "^2.6.8"
731 | }
732 | },
733 | "prompt-base": {
734 | "version": "4.1.0",
735 | "resolved": "https://registry.npmjs.org/prompt-base/-/prompt-base-4.1.0.tgz",
736 | "integrity": "sha512-svGzgLUKZoqomz9SGMkf1hBG8Wl3K7JGuRCXc/Pv7xw8239hhaTBXrmjt7EXA9P/QZzdyT8uNWt9F/iJTXq75g==",
737 | "dev": true,
738 | "requires": {
739 | "component-emitter": "^1.2.1",
740 | "debug": "^3.0.1",
741 | "koalas": "^1.0.2",
742 | "log-utils": "^0.2.1",
743 | "prompt-actions": "^3.0.2",
744 | "prompt-question": "^5.0.1",
745 | "readline-ui": "^2.2.3",
746 | "readline-utils": "^2.2.3",
747 | "static-extend": "^0.1.2"
748 | },
749 | "dependencies": {
750 | "debug": {
751 | "version": "3.2.6",
752 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
753 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
754 | "dev": true,
755 | "requires": {
756 | "ms": "^2.1.1"
757 | }
758 | },
759 | "ms": {
760 | "version": "2.1.2",
761 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
762 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
763 | "dev": true
764 | }
765 | }
766 | },
767 | "prompt-choices": {
768 | "version": "4.1.0",
769 | "resolved": "https://registry.npmjs.org/prompt-choices/-/prompt-choices-4.1.0.tgz",
770 | "integrity": "sha512-ZNYLv6rW9z9n0WdwCkEuS+w5nUAGzRgtRt6GQ5aFNFz6MIcU7nHFlHOwZtzy7RQBk80KzUGPSRQphvMiQzB8pg==",
771 | "dev": true,
772 | "requires": {
773 | "arr-flatten": "^1.1.0",
774 | "arr-swap": "^1.0.1",
775 | "choices-separator": "^2.0.0",
776 | "clone-deep": "^4.0.0",
777 | "collection-visit": "^1.0.0",
778 | "define-property": "^2.0.2",
779 | "is-number": "^6.0.0",
780 | "kind-of": "^6.0.2",
781 | "koalas": "^1.0.2",
782 | "log-utils": "^0.2.1",
783 | "pointer-symbol": "^1.0.0",
784 | "radio-symbol": "^2.0.0",
785 | "set-value": "^3.0.0",
786 | "strip-color": "^0.1.0",
787 | "terminal-paginator": "^2.0.2",
788 | "toggle-array": "^1.0.1"
789 | },
790 | "dependencies": {
791 | "clone-deep": {
792 | "version": "4.0.1",
793 | "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
794 | "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
795 | "dev": true,
796 | "requires": {
797 | "is-plain-object": "^2.0.4",
798 | "kind-of": "^6.0.2",
799 | "shallow-clone": "^3.0.0"
800 | }
801 | },
802 | "define-property": {
803 | "version": "2.0.2",
804 | "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
805 | "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
806 | "dev": true,
807 | "requires": {
808 | "is-descriptor": "^1.0.2",
809 | "isobject": "^3.0.1"
810 | }
811 | },
812 | "kind-of": {
813 | "version": "6.0.3",
814 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
815 | "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
816 | "dev": true
817 | },
818 | "shallow-clone": {
819 | "version": "3.0.1",
820 | "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
821 | "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
822 | "dev": true,
823 | "requires": {
824 | "kind-of": "^6.0.2"
825 | }
826 | }
827 | }
828 | },
829 | "prompt-confirm": {
830 | "version": "1.2.0",
831 | "resolved": "https://registry.npmjs.org/prompt-confirm/-/prompt-confirm-1.2.0.tgz",
832 | "integrity": "sha512-r7XZxI5J5/oPtUskN0ZYO+lkv/WJHMQgfd1GTKAuxnHuViQShiFHdUnj6DamL4gQExaKAX7rnIcTKoRSpVVquA==",
833 | "dev": true,
834 | "requires": {
835 | "debug": "^2.6.8",
836 | "prompt-base": "^4.0.1"
837 | }
838 | },
839 | "prompt-question": {
840 | "version": "5.0.2",
841 | "resolved": "https://registry.npmjs.org/prompt-question/-/prompt-question-5.0.2.tgz",
842 | "integrity": "sha512-wreaLbbu8f5+7zXds199uiT11Ojp59Z4iBi6hONlSLtsKGTvL2UY8VglcxQ3t/X4qWIxsNCg6aT4O8keO65v6Q==",
843 | "dev": true,
844 | "requires": {
845 | "clone-deep": "^1.0.0",
846 | "debug": "^3.0.1",
847 | "define-property": "^1.0.0",
848 | "isobject": "^3.0.1",
849 | "kind-of": "^5.0.2",
850 | "koalas": "^1.0.2",
851 | "prompt-choices": "^4.0.5"
852 | },
853 | "dependencies": {
854 | "debug": {
855 | "version": "3.2.6",
856 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
857 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
858 | "dev": true,
859 | "requires": {
860 | "ms": "^2.1.1"
861 | }
862 | },
863 | "kind-of": {
864 | "version": "5.1.0",
865 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
866 | "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
867 | "dev": true
868 | },
869 | "ms": {
870 | "version": "2.1.2",
871 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
872 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
873 | "dev": true
874 | }
875 | }
876 | },
877 | "radio-symbol": {
878 | "version": "2.0.0",
879 | "resolved": "https://registry.npmjs.org/radio-symbol/-/radio-symbol-2.0.0.tgz",
880 | "integrity": "sha1-eqm/xQSFY21S3XbWqOYxspB5muE=",
881 | "dev": true,
882 | "requires": {
883 | "ansi-gray": "^0.1.1",
884 | "ansi-green": "^0.1.1",
885 | "is-windows": "^1.0.1"
886 | }
887 | },
888 | "readline-ui": {
889 | "version": "2.2.3",
890 | "resolved": "https://registry.npmjs.org/readline-ui/-/readline-ui-2.2.3.tgz",
891 | "integrity": "sha512-ix7jz0PxqQqcIuq3yQTHv1TOhlD2IHO74aNO+lSuXsRYm1d+pdyup1yF3zKyLK1wWZrVNGjkzw5tUegO2IDy+A==",
892 | "dev": true,
893 | "requires": {
894 | "component-emitter": "^1.2.1",
895 | "debug": "^2.6.8",
896 | "readline-utils": "^2.2.1",
897 | "string-width": "^2.0.0"
898 | }
899 | },
900 | "readline-utils": {
901 | "version": "2.2.3",
902 | "resolved": "https://registry.npmjs.org/readline-utils/-/readline-utils-2.2.3.tgz",
903 | "integrity": "sha1-b4R9a48ZFcORtYHDZ81HhzhiNRo=",
904 | "dev": true,
905 | "requires": {
906 | "arr-flatten": "^1.1.0",
907 | "extend-shallow": "^2.0.1",
908 | "is-buffer": "^1.1.5",
909 | "is-number": "^3.0.0",
910 | "is-windows": "^1.0.1",
911 | "koalas": "^1.0.2",
912 | "mute-stream": "0.0.7",
913 | "strip-color": "^0.1.0",
914 | "window-size": "^1.1.0"
915 | },
916 | "dependencies": {
917 | "is-number": {
918 | "version": "3.0.0",
919 | "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
920 | "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
921 | "dev": true,
922 | "requires": {
923 | "kind-of": "^3.0.2"
924 | }
925 | }
926 | }
927 | },
928 | "serverless-finch": {
929 | "version": "2.5.2",
930 | "resolved": "https://registry.npmjs.org/serverless-finch/-/serverless-finch-2.5.2.tgz",
931 | "integrity": "sha512-fNxedBed7CL8nK2QJ2oxGgDzxHeChva3nvGmxTHMx9SXX2te6RqHFEX0sG6RNt+4VrteazRYA8/j0vArJem6cw==",
932 | "dev": true,
933 | "requires": {
934 | "is_js": "^0.9.0",
935 | "mime": "^1.2.11",
936 | "minimatch": "^3.0.4",
937 | "prompt-confirm": "^1.2.0"
938 | }
939 | },
940 | "set-getter": {
941 | "version": "0.1.0",
942 | "resolved": "https://registry.npmjs.org/set-getter/-/set-getter-0.1.0.tgz",
943 | "integrity": "sha1-12nBgsnVpR9AkUXy+6guXoboA3Y=",
944 | "dev": true,
945 | "requires": {
946 | "to-object-path": "^0.3.0"
947 | }
948 | },
949 | "set-value": {
950 | "version": "3.0.2",
951 | "resolved": "https://registry.npmjs.org/set-value/-/set-value-3.0.2.tgz",
952 | "integrity": "sha512-npjkVoz+ank0zjlV9F47Fdbjfj/PfXyVhZvGALWsyIYU/qrMzpi6avjKW3/7KeSU2Df3I46BrN1xOI1+6vW0hA==",
953 | "dev": true,
954 | "requires": {
955 | "is-plain-object": "^2.0.4"
956 | }
957 | },
958 | "shallow-clone": {
959 | "version": "1.0.0",
960 | "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-1.0.0.tgz",
961 | "integrity": "sha512-oeXreoKR/SyNJtRJMAKPDSvd28OqEwG4eR/xc856cRGBII7gX9lvAqDxusPm0846z/w/hWYjI1NpKwJ00NHzRA==",
962 | "dev": true,
963 | "requires": {
964 | "is-extendable": "^0.1.1",
965 | "kind-of": "^5.0.0",
966 | "mixin-object": "^2.0.1"
967 | },
968 | "dependencies": {
969 | "kind-of": {
970 | "version": "5.1.0",
971 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
972 | "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
973 | "dev": true
974 | }
975 | }
976 | },
977 | "static-extend": {
978 | "version": "0.1.2",
979 | "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
980 | "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=",
981 | "dev": true,
982 | "requires": {
983 | "define-property": "^0.2.5",
984 | "object-copy": "^0.1.0"
985 | },
986 | "dependencies": {
987 | "define-property": {
988 | "version": "0.2.5",
989 | "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
990 | "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
991 | "dev": true,
992 | "requires": {
993 | "is-descriptor": "^0.1.0"
994 | }
995 | },
996 | "is-accessor-descriptor": {
997 | "version": "0.1.6",
998 | "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
999 | "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
1000 | "dev": true,
1001 | "requires": {
1002 | "kind-of": "^3.0.2"
1003 | },
1004 | "dependencies": {
1005 | "kind-of": {
1006 | "version": "3.2.2",
1007 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
1008 | "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
1009 | "dev": true,
1010 | "requires": {
1011 | "is-buffer": "^1.1.5"
1012 | }
1013 | }
1014 | }
1015 | },
1016 | "is-data-descriptor": {
1017 | "version": "0.1.4",
1018 | "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
1019 | "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
1020 | "dev": true,
1021 | "requires": {
1022 | "kind-of": "^3.0.2"
1023 | },
1024 | "dependencies": {
1025 | "kind-of": {
1026 | "version": "3.2.2",
1027 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
1028 | "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
1029 | "dev": true,
1030 | "requires": {
1031 | "is-buffer": "^1.1.5"
1032 | }
1033 | }
1034 | }
1035 | },
1036 | "is-descriptor": {
1037 | "version": "0.1.6",
1038 | "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
1039 | "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
1040 | "dev": true,
1041 | "requires": {
1042 | "is-accessor-descriptor": "^0.1.6",
1043 | "is-data-descriptor": "^0.1.4",
1044 | "kind-of": "^5.0.0"
1045 | }
1046 | },
1047 | "kind-of": {
1048 | "version": "5.1.0",
1049 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
1050 | "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
1051 | "dev": true
1052 | }
1053 | }
1054 | },
1055 | "string-width": {
1056 | "version": "2.1.1",
1057 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
1058 | "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
1059 | "dev": true,
1060 | "requires": {
1061 | "is-fullwidth-code-point": "^2.0.0",
1062 | "strip-ansi": "^4.0.0"
1063 | }
1064 | },
1065 | "strip-ansi": {
1066 | "version": "4.0.0",
1067 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
1068 | "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
1069 | "dev": true,
1070 | "requires": {
1071 | "ansi-regex": "^3.0.0"
1072 | }
1073 | },
1074 | "strip-color": {
1075 | "version": "0.1.0",
1076 | "resolved": "https://registry.npmjs.org/strip-color/-/strip-color-0.1.0.tgz",
1077 | "integrity": "sha1-EG9l09PmotlAHKwOsM6LinArT3s=",
1078 | "dev": true
1079 | },
1080 | "success-symbol": {
1081 | "version": "0.1.0",
1082 | "resolved": "https://registry.npmjs.org/success-symbol/-/success-symbol-0.1.0.tgz",
1083 | "integrity": "sha1-JAIuSG878c3KCUKDt2nEctO3KJc=",
1084 | "dev": true
1085 | },
1086 | "terminal-paginator": {
1087 | "version": "2.0.2",
1088 | "resolved": "https://registry.npmjs.org/terminal-paginator/-/terminal-paginator-2.0.2.tgz",
1089 | "integrity": "sha512-IZMT5ECF9p4s+sNCV8uvZSW9E1+9zy9Ji9xz2oee8Jfo7hUFpauyjxkhfRcIH6Lu3Wdepv5D1kVRc8Hx74/LfQ==",
1090 | "dev": true,
1091 | "requires": {
1092 | "debug": "^2.6.6",
1093 | "extend-shallow": "^2.0.1",
1094 | "log-utils": "^0.2.1"
1095 | }
1096 | },
1097 | "time-stamp": {
1098 | "version": "1.1.0",
1099 | "resolved": "https://registry.npmjs.org/time-stamp/-/time-stamp-1.1.0.tgz",
1100 | "integrity": "sha1-dkpaEa9QVhkhsTPztE5hhofg9cM=",
1101 | "dev": true
1102 | },
1103 | "to-object-path": {
1104 | "version": "0.3.0",
1105 | "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
1106 | "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=",
1107 | "dev": true,
1108 | "requires": {
1109 | "kind-of": "^3.0.2"
1110 | }
1111 | },
1112 | "toggle-array": {
1113 | "version": "1.0.1",
1114 | "resolved": "https://registry.npmjs.org/toggle-array/-/toggle-array-1.0.1.tgz",
1115 | "integrity": "sha1-y/WEB5K9UJfzMReugkyTKv/ofVg=",
1116 | "dev": true,
1117 | "requires": {
1118 | "isobject": "^3.0.0"
1119 | }
1120 | },
1121 | "warning-symbol": {
1122 | "version": "0.1.0",
1123 | "resolved": "https://registry.npmjs.org/warning-symbol/-/warning-symbol-0.1.0.tgz",
1124 | "integrity": "sha1-uzHdEbeg+dZ6su2V9Fe2WCW7rSE=",
1125 | "dev": true
1126 | },
1127 | "window-size": {
1128 | "version": "1.1.1",
1129 | "resolved": "https://registry.npmjs.org/window-size/-/window-size-1.1.1.tgz",
1130 | "integrity": "sha512-5D/9vujkmVQ7pSmc0SCBmHXbkv6eaHwXEx65MywhmUMsI8sGqJ972APq1lotfcwMKPFLuCFfL8xGHLIp7jaBmA==",
1131 | "dev": true,
1132 | "requires": {
1133 | "define-property": "^1.0.0",
1134 | "is-number": "^3.0.0"
1135 | },
1136 | "dependencies": {
1137 | "is-number": {
1138 | "version": "3.0.0",
1139 | "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
1140 | "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
1141 | "dev": true,
1142 | "requires": {
1143 | "kind-of": "^3.0.2"
1144 | }
1145 | }
1146 | }
1147 | }
1148 | }
1149 | }
1150 |
--------------------------------------------------------------------------------
/frontend/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "frontend",
3 | "description": "",
4 | "version": "0.1.0",
5 | "dependencies": {},
6 | "devDependencies": {
7 | "serverless-finch": "^2.5.2"
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/frontend/serverless.yml:
--------------------------------------------------------------------------------
1 | # Welcome to Serverless!
2 | #
3 | # This file is the main config file for your service.
4 | # It's very minimal at this point and uses default values.
5 | # You can always add more config options for more control.
6 | # We've included some commented out config examples here.
7 | # Just uncomment any of them to get that config option.
8 | #
9 | # For full config options, check the docs:
10 | # docs.serverless.com
11 | #
12 | # Happy Coding!
13 |
14 | service: frontend
15 | # app and org for use with dashboard.serverless.com
16 | #app: your-app-name
17 | #org: your-org-name
18 |
19 | # You can pin your service to only deploy with a specific Serverless version
20 | # Check out our docs for more details
21 | # frameworkVersion: "=X.X.X"
22 |
23 | provider:
24 | name: aws
25 | runtime: nodejs12.x
26 |
27 | # you can overwrite defaults here
28 | # stage: dev
29 | # region: us-east-1
30 |
31 | # you can add statements to the Lambda function's IAM Role here
32 | # iamRoleStatements:
33 | # - Effect: "Allow"
34 | # Action:
35 | # - "s3:ListBucket"
36 | # Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] }
37 | # - Effect: "Allow"
38 | # Action:
39 | # - "s3:PutObject"
40 | # Resource:
41 | # Fn::Join:
42 | # - ""
43 | # - - "arn:aws:s3:::"
44 | # - "Ref" : "ServerlessDeploymentBucket"
45 | # - "/*"
46 |
47 | # you can define service wide environment variables here
48 | # environment:
49 | # variable1: value1
50 |
51 | # you can add packaging information here
52 | #package:
53 | # include:
54 | # - include-me.js
55 | # - include-me-dir/**
56 | # exclude:
57 | # - exclude-me.js
58 | # - exclude-me-dir/**
59 |
60 | functions:
61 | hello:
62 | handler: handler.hello
63 | # The following are a few example events you can configure
64 | # NOTE: Please make sure to change your handler code to work with those events
65 | # Check the event documentation for details
66 | # events:
67 | # - http:
68 | # path: users/create
69 | # method: get
70 | # - websocket: $connect
71 | # - s3: ${env:BUCKET}
72 | # - schedule: rate(10 minutes)
73 | # - sns: greeter-topic
74 | # - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000
75 | # - alexaSkill: amzn1.ask.skill.xx-xx-xx-xx
76 | # - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx
77 | # - iot:
78 | # sql: "SELECT * FROM 'some_topic'"
79 | # - cloudwatchEvent:
80 | # event:
81 | # source:
82 | # - "aws.ec2"
83 | # detail-type:
84 | # - "EC2 Instance State-change Notification"
85 | # detail:
86 | # state:
87 | # - pending
88 | # - cloudwatchLog: '/aws/lambda/hello'
89 | # - cognitoUserPool:
90 | # pool: MyUserPool
91 | # trigger: PreSignUp
92 | # - alb:
93 | # listenerArn: arn:aws:elasticloadbalancing:us-east-1:XXXXXX:listener/app/my-load-balancer/50dc6c495c0c9188/
94 | # priority: 1
95 | # conditions:
96 | # host: example.com
97 | # path: /hello
98 |
99 | # Define function environment variables here
100 | # environment:
101 | # variable2: value2
102 |
103 | # you can add CloudFormation resource templates here
104 | #resources:
105 | # Resources:
106 | # NewResource:
107 | # Type: AWS::S3::Bucket
108 | # Properties:
109 | # BucketName: my-new-bucket
110 | # Outputs:
111 | # NewOutput:
112 | # Description: "Description for the output"
113 | # Value: "Some output value"
114 | custom:
115 | client:
116 | bucketName: serverless-search-engine
117 | plugins:
118 | - serverless-finch
119 |
--------------------------------------------------------------------------------
/query-service/.gitignore:
--------------------------------------------------------------------------------
1 | # package directories
2 | node_modules
3 | jspm_packages
4 |
5 | # Serverless directories
6 | .serverless
--------------------------------------------------------------------------------
/query-service/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "query-service",
3 | "version": "0.1.0",
4 | "lockfileVersion": 1,
5 | "requires": true,
6 | "dependencies": {
7 | "serverless-pseudo-parameters": {
8 | "version": "2.5.0",
9 | "resolved": "https://registry.npmjs.org/serverless-pseudo-parameters/-/serverless-pseudo-parameters-2.5.0.tgz",
10 | "integrity": "sha512-A/O49AR8LL6jlnPSmnOTYgL1KqVgskeRla4sVDeS/r5dHFJlwOU5MgFilc7aaQP8NWAwRJANaIS9oiSE3I+VUA==",
11 | "dev": true
12 | }
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/query-service/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "query-service",
3 | "description": "",
4 | "version": "0.1.0",
5 | "dependencies": {},
6 | "devDependencies": {
7 | "serverless-pseudo-parameters": "^2.5.0"
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/query-service/serverless.yml:
--------------------------------------------------------------------------------
1 | # Welcome to Serverless!
2 | #
3 | # This file is the main config file for your service.
4 | # It's very minimal at this point and uses default values.
5 | # You can always add more config options for more control.
6 | # We've included some commented out config examples here.
7 | # Just uncomment any of them to get that config option.
8 | #
9 | # For full config options, check the docs:
10 | # docs.serverless.com
11 | #
12 | # Happy Coding!
13 |
14 | service: query-service
15 | # app and org for use with dashboard.serverless.com
16 | app: search-engine
17 | org: serverlesssearchengine
18 |
19 | # You can pin your service to only deploy with a specific Serverless version
20 | # Check out our docs for more details
21 | # frameworkVersion: "=X.X.X"
22 |
23 | provider:
24 | name: aws
25 | runtime: nodejs12.x
26 |
27 | # you can overwrite defaults here
28 | # stage: dev
29 | # region: us-east-1
30 |
31 | # you can add statements to the Lambda function's IAM Role here
32 | iamRoleStatements:
33 | - Effect: "Allow"
34 | Action:
35 | - dynamodb:Query
36 | Resource:
37 | - arn:aws:dynamodb:#{AWS::Region}:#{AWS::AccountId}:table/${self:service}-keywordsTable-${opt:stage, 'dev'}
38 | # iamRoleStatements:
39 | # - Effect: "Allow"
40 | # Action:
41 | # - "s3:ListBucket"
42 | # Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] }
43 | # - Effect: "Allow"
44 | # Action:
45 | # - "s3:PutObject"
46 | # Resource:
47 | # Fn::Join:
48 | # - ""
49 | # - - "arn:aws:s3:::"
50 | # - "Ref" : "ServerlessDeploymentBucket"
51 | # - "/*"
52 |
53 | # you can define service wide environment variables here
54 | environment:
55 | DYNAMODB_KEYWORD_TABLE: ${self:service}-keywordsTable-${opt:stage, 'dev'}
56 | # environment:
57 | # variable1: value1
58 |
59 | # you can add packaging information here
60 | #package:
61 | # include:
62 | # - include-me.js
63 | # - include-me-dir/**
64 | # exclude:
65 | # - exclude-me.js
66 | # - exclude-me-dir/**
67 | functions:
68 | addKeywords:
69 | handler: src/functions/addKeywords.addKeywords
70 | # events:
71 | # - sns: ${outputs:spider-service.keywordsTopic}
72 | queryFunction:
73 | handler: src/functions/query.query
74 | events:
75 | - http:
76 | path: v1/query
77 | method: post
78 | cors: true
79 | # The following are a few example events you can configure
80 | # NOTE: Please make sure to change your handler code to work with those events
81 | # Check the event documentation for details
82 | # events:
83 | # - http:
84 | # path: users/create
85 | # method: get
86 | # - websocket: $connect
87 | # - s3: ${env:BUCKET}
88 | # - schedule: rate(10 minutes)
89 | # - sns: greeter-topic
90 | # - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000
91 | # - alexaSkill: amzn1.ask.skill.xx-xx-xx-xx
92 | # - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx
93 | # - iot:
94 | # sql: "SELECT * FROM 'some_topic'"
95 | # - cloudwatchEvent:
96 | # event:
97 | # source:
98 | # - "aws.ec2"
99 | # detail-type:
100 | # - "EC2 Instance State-change Notification"
101 | # detail:
102 | # state:
103 | # - pending
104 | # - cloudwatchLog: '/aws/lambda/hello'
105 | # - cognitoUserPool:
106 | # pool: MyUserPool
107 | # trigger: PreSignUp
108 | # - alb:
109 | # listenerArn: arn:aws:elasticloadbalancing:us-east-1:XXXXXX:listener/app/my-load-balancer/50dc6c495c0c9188/
110 | # priority: 1
111 | # conditions:
112 | # host: example.com
113 | # path: /hello
114 |
115 | # Define function environment variables here
116 | # environment:
117 | # variable2: value2
118 |
119 | # you can add CloudFormation resource templates here
120 | resources:
121 | Resources:
122 | keywordsTable:
123 | Type: AWS::DynamoDB::Table
124 | Properties:
125 | AttributeDefinitions:
126 | - AttributeName: pk
127 | AttributeType: S
128 | - AttributeName: sk
129 | AttributeType: S
130 | BillingMode: PAY_PER_REQUEST
131 | KeySchema:
132 | - AttributeName: pk
133 | KeyType: HASH
134 | - AttributeName: sk
135 | KeyType: RANGE
136 | TableName: ${self:service}-keywordsTable-${opt:stage, 'dev'}
137 | # NewResource:
138 | # Type: AWS::S3::Bucket
139 | # Properties:
140 | # BucketName: my-new-bucket
141 | # Outputs:
142 | # NewOutput:
143 | # Description: "Description for the output"
144 | # Value: "Some output value"
145 |
146 | plugins:
147 | - serverless-pseudo-parameters
148 |
--------------------------------------------------------------------------------
/query-service/src/functions/addKeywords.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 | const AWS = require('aws-sdk')
3 |
4 | module.exports.addKeywords = async event => {
5 | const dynamodb = new AWS.DynamoDB.DocumentClient()
6 | console.log('event', event)
7 | await event.Records.forEach(record => {
8 | const message = JSON.parse(record.Sns.Message)
9 | const keywordsArr = message.keywords.split(' ')
10 | const secondaryKey = keywordsArr[1] + '|' + keywordsArr[2]
11 | const putParams = {
12 | TableName: process.env.DYNAMODB_KEYWORD_TABLE,
13 | Item: {
14 | 'pk': keywordsArr[0],
15 | 'sk': secondaryKey
16 | }
17 | }
18 | dynamodb.put(putParams).promise()
19 | })
20 |
21 | return true
22 | // {
23 | // "keywords":"serverless framework awesome",
24 | // "url": ["serverless.com"]
25 | // }
26 |
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/query-service/src/functions/query.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 | const AWS = require('aws-sdk')
3 |
4 | module.exports.query = async event => {
5 | const body = JSON.parse(event.body)
6 | const keywords = body.search
7 | const keywordsArr = keywords.split(' ')
8 | const secondaryKey = keywordsArr[1] + '|' + keywordsArr[2]
9 |
10 | const queryParams = {
11 | TableName: process.env.DYNAMODB_KEYWORD_TABLE,
12 | KeyConditionExpression: '#HashKey = :hkey and #RangeKey = :rkey',
13 | ExpressionAttributeValues: {
14 | ':hkey': keywordsArr[0],
15 | ':rkey': secondaryKey
16 | },
17 | ExpressionAttributeNames: {
18 | '#HashKey': 'pk',
19 | '#RangeKey': 'sk'
20 | }
21 | }
22 | try {
23 | const dynamodb = new AWS.DynamoDB.DocumentClient()
24 | const queryResult = await dynamodb.query(queryParams).promise()
25 | console.log('queryParams', queryParams)
26 | console.log('queryResult', queryResult)
27 | const items = []
28 | for (const itemResult of queryResult.Items) {
29 | console.log(itemResult)
30 | items.push(itemResult.url)
31 | }
32 | return {
33 | statusCode: 200,
34 | body: JSON.stringify({
35 | items: items
36 | }),
37 | headers: {
38 | 'Access-Control-Allow-Origin': '*'
39 | }
40 | }
41 | } catch (queryError) {
42 | console.log('There was an error querying the database')
43 | console.log('queryError', queryError)
44 | console.log('queryParams', queryParams)
45 | return {
46 | statusCode: 500,
47 | body: JSON.stringify({
48 | error: 'There was an error querying the database'
49 | }),
50 | headers: {
51 | 'Access-Control-Allow-Origin': '*'
52 | }
53 | }
54 | }
55 | };
56 |
--------------------------------------------------------------------------------
/spider-service/.gitignore:
--------------------------------------------------------------------------------
1 | # package directories
2 | node_modules
3 | jspm_packages
4 |
5 | # Serverless directories
6 | .serverless
--------------------------------------------------------------------------------
/spider-service/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "spider-service",
3 | "version": "0.1.0",
4 | "lockfileVersion": 1,
5 | "requires": true,
6 | "dependencies": {
7 | "@types/node": {
8 | "version": "14.0.1",
9 | "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.1.tgz",
10 | "integrity": "sha512-FAYBGwC+W6F9+huFIDtn43cpy7+SzG+atzRiTfdp3inUKL2hXnd4rG8hylJLIh4+hqrQy1P17kvJByE/z825hA=="
11 | },
12 | "aws-sdk": {
13 | "version": "2.683.0",
14 | "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.683.0.tgz",
15 | "integrity": "sha512-mi1175pJMbQhCWhBEQ5ccQ3SDE+SJzbapuAQtcb7tdLLV7dPKf4zQXhpqK1uG0dysm8NhsNtgxwA2diYOKWlTg==",
16 | "dev": true,
17 | "requires": {
18 | "buffer": "4.9.1",
19 | "events": "1.1.1",
20 | "ieee754": "1.1.13",
21 | "jmespath": "0.15.0",
22 | "querystring": "0.2.0",
23 | "sax": "1.2.1",
24 | "url": "0.10.3",
25 | "uuid": "3.3.2",
26 | "xml2js": "0.4.19"
27 | }
28 | },
29 | "axios": {
30 | "version": "0.19.2",
31 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz",
32 | "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==",
33 | "requires": {
34 | "follow-redirects": "1.5.10"
35 | }
36 | },
37 | "base64-js": {
38 | "version": "1.3.1",
39 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
40 | "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==",
41 | "dev": true
42 | },
43 | "boolbase": {
44 | "version": "1.0.0",
45 | "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
46 | "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
47 | },
48 | "buffer": {
49 | "version": "4.9.1",
50 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
51 | "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
52 | "dev": true,
53 | "requires": {
54 | "base64-js": "^1.0.2",
55 | "ieee754": "^1.1.4",
56 | "isarray": "^1.0.0"
57 | }
58 | },
59 | "cheerio": {
60 | "version": "1.0.0-rc.3",
61 | "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
62 | "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
63 | "requires": {
64 | "css-select": "~1.2.0",
65 | "dom-serializer": "~0.1.1",
66 | "entities": "~1.1.1",
67 | "htmlparser2": "^3.9.1",
68 | "lodash": "^4.15.0",
69 | "parse5": "^3.0.1"
70 | }
71 | },
72 | "css-select": {
73 | "version": "1.2.0",
74 | "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
75 | "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
76 | "requires": {
77 | "boolbase": "~1.0.0",
78 | "css-what": "2.1",
79 | "domutils": "1.5.1",
80 | "nth-check": "~1.0.1"
81 | }
82 | },
83 | "css-what": {
84 | "version": "2.1.3",
85 | "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
86 | "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
87 | },
88 | "debug": {
89 | "version": "3.1.0",
90 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
91 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
92 | "requires": {
93 | "ms": "2.0.0"
94 | }
95 | },
96 | "dom-serializer": {
97 | "version": "0.1.1",
98 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
99 | "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
100 | "requires": {
101 | "domelementtype": "^1.3.0",
102 | "entities": "^1.1.1"
103 | }
104 | },
105 | "domelementtype": {
106 | "version": "1.3.1",
107 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
108 | "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
109 | },
110 | "domhandler": {
111 | "version": "2.4.2",
112 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
113 | "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
114 | "requires": {
115 | "domelementtype": "1"
116 | }
117 | },
118 | "domutils": {
119 | "version": "1.5.1",
120 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
121 | "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
122 | "requires": {
123 | "dom-serializer": "0",
124 | "domelementtype": "1"
125 | }
126 | },
127 | "entities": {
128 | "version": "1.1.2",
129 | "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
130 | "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
131 | },
132 | "events": {
133 | "version": "1.1.1",
134 | "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
135 | "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
136 | "dev": true
137 | },
138 | "follow-redirects": {
139 | "version": "1.5.10",
140 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
141 | "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
142 | "requires": {
143 | "debug": "=3.1.0"
144 | }
145 | },
146 | "htmlparser2": {
147 | "version": "3.10.1",
148 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
149 | "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
150 | "requires": {
151 | "domelementtype": "^1.3.1",
152 | "domhandler": "^2.3.0",
153 | "domutils": "^1.5.1",
154 | "entities": "^1.1.1",
155 | "inherits": "^2.0.1",
156 | "readable-stream": "^3.1.1"
157 | }
158 | },
159 | "ieee754": {
160 | "version": "1.1.13",
161 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
162 | "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==",
163 | "dev": true
164 | },
165 | "inherits": {
166 | "version": "2.0.4",
167 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
168 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
169 | },
170 | "isarray": {
171 | "version": "1.0.0",
172 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
173 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
174 | "dev": true
175 | },
176 | "jmespath": {
177 | "version": "0.15.0",
178 | "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
179 | "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=",
180 | "dev": true
181 | },
182 | "lodash": {
183 | "version": "4.17.15",
184 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
185 | "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
186 | },
187 | "ms": {
188 | "version": "2.0.0",
189 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
190 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
191 | },
192 | "nth-check": {
193 | "version": "1.0.2",
194 | "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
195 | "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
196 | "requires": {
197 | "boolbase": "~1.0.0"
198 | }
199 | },
200 | "parse5": {
201 | "version": "3.0.3",
202 | "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
203 | "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
204 | "requires": {
205 | "@types/node": "*"
206 | }
207 | },
208 | "punycode": {
209 | "version": "1.3.2",
210 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
211 | "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
212 | "dev": true
213 | },
214 | "querystring": {
215 | "version": "0.2.0",
216 | "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
217 | "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
218 | "dev": true
219 | },
220 | "readable-stream": {
221 | "version": "3.6.0",
222 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
223 | "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
224 | "requires": {
225 | "inherits": "^2.0.3",
226 | "string_decoder": "^1.1.1",
227 | "util-deprecate": "^1.0.1"
228 | }
229 | },
230 | "safe-buffer": {
231 | "version": "5.2.1",
232 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
233 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
234 | },
235 | "sax": {
236 | "version": "1.2.1",
237 | "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
238 | "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=",
239 | "dev": true
240 | },
241 | "serverless-pseudo-parameters": {
242 | "version": "2.5.0",
243 | "resolved": "https://registry.npmjs.org/serverless-pseudo-parameters/-/serverless-pseudo-parameters-2.5.0.tgz",
244 | "integrity": "sha512-A/O49AR8LL6jlnPSmnOTYgL1KqVgskeRla4sVDeS/r5dHFJlwOU5MgFilc7aaQP8NWAwRJANaIS9oiSE3I+VUA=="
245 | },
246 | "string_decoder": {
247 | "version": "1.3.0",
248 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
249 | "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
250 | "requires": {
251 | "safe-buffer": "~5.2.0"
252 | }
253 | },
254 | "url": {
255 | "version": "0.10.3",
256 | "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
257 | "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
258 | "dev": true,
259 | "requires": {
260 | "punycode": "1.3.2",
261 | "querystring": "0.2.0"
262 | }
263 | },
264 | "util-deprecate": {
265 | "version": "1.0.2",
266 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
267 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
268 | },
269 | "uuid": {
270 | "version": "3.3.2",
271 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
272 | "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
273 | "dev": true
274 | },
275 | "xml2js": {
276 | "version": "0.4.19",
277 | "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
278 | "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
279 | "dev": true,
280 | "requires": {
281 | "sax": ">=0.6.0",
282 | "xmlbuilder": "~9.0.1"
283 | }
284 | },
285 | "xmlbuilder": {
286 | "version": "9.0.7",
287 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
288 | "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=",
289 | "dev": true
290 | }
291 | }
292 | }
293 |
--------------------------------------------------------------------------------
/spider-service/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "spider-service",
3 | "description": "",
4 | "version": "0.1.0",
5 | "dependencies": {
6 | "axios": "^0.19.2",
7 | "cheerio": "^1.0.0-rc.3",
8 | "serverless-pseudo-parameters": "^2.5.0"
9 | },
10 | "devDependencies": {
11 | "aws-sdk": "^2.683.0"
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/spider-service/serverless.yml:
--------------------------------------------------------------------------------
1 | # Welcome to Serverless!
2 | #
3 | # This file is the main config file for your service.
4 | # It's very minimal at this point and uses default values.
5 | # You can always add more config options for more control.
6 | # We've included some commented out config examples here.
7 | # Just uncomment any of them to get that config option.
8 | #
9 | # For full config options, check the docs:
10 | # docs.serverless.com
11 | #
12 | # Happy Coding!
13 |
14 | service: spider-service
15 | # app and org for use with dashboard.serverless.com
16 | app: search-engine
17 | org: serverlesssearchengine
18 | custom:
19 | accountId: !Ref AWS::AccountId
20 | stage: ${{opt:stage, 'dev'}}
21 | # You can pin your service to only deploy with a specific Serverless version
22 | # Check out our docs for more details
23 | # frameworkVersion: "=X.X.X"
24 | provider:
25 | name: aws
26 | runtime: nodejs12.x
27 | region: us-east-1
28 | variableSyntax: '\${{([\s\S]+?)}}'
29 | # you can overwrite defaults here
30 | # stage: dev
31 | # region: us-east-1
32 |
33 | # you can add statements to the Lambda function's IAM Role here
34 | iamRoleStatements:
35 | - Effect: "Allow"
36 | Action:
37 | - sns:Publish
38 | Resource:
39 | - Fn::Sub: arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${{self:service}}-sse-scrapePageTopic-${{self:custom.stage}}
40 | - Effect: "Allow"
41 | Action:
42 | - dynamodb:PutItem
43 | - dynamodb:Query
44 | Resource:
45 | - Fn::Sub: arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${{self:service}}-urls-${{opt:stage, 'dev'}}
46 | # - Effect: "Allow"
47 | # Action:
48 | # - "s3:ListBucket"
49 | # Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] }
50 | # - Effect: "Allow"
51 | # Action:
52 | # - "s3:PutObject"
53 | # Resource:
54 | # Fn::Join:
55 | # - ""
56 | # - - "arn:aws:s3:::"
57 | # - "Ref" : "ServerlessDeploymentBucket"
58 | # - "/*"
59 |
60 | # you can define service wide environment variables here
61 | environment:
62 | DYNAMODB_URLS_TABLE: ${{self:service}}-urls-${{opt:stage, 'dev'}}
63 | SCRAPER_BATCH_SIZE: ${param:SCRAPER_BATCH_SIZE}
64 | SNS_SCRAPE_PAGE_TOPIC:
65 | Fn::Sub: arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${{self:service}}-sse-scrapePageTopic-${{self:custom.stage}}
66 | # SNS_SCRAPE_PAGE_TOPIC: arn:aws:sns:${self:provider.region}:*:${self:service}-sse-scrapePageTopic-${opt:stage, 'dev'}
67 | # variable1: value1
68 |
69 | # you can add packaging information here
70 | #package:
71 | # include:
72 | # - include-me.js
73 | # - include-me-dir/**
74 | # exclude:
75 | # - exclude-me.js
76 | # - exclude-me-dir/**
77 |
78 | functions:
79 | scraper:
80 | handler: src/functions/scraper.scraper
81 | events:
82 | - sns: ${{self:service}}-sse-scrapePageTopic-${{opt:stage, 'dev'}}
83 | timeout: 30
84 | startScraping:
85 | handler: src/functions/startScraping.startScraping
86 | events:
87 | - schedule:
88 | rate: rate(1 hour)
89 | enabled: true
90 | # The following are a few example events you can configure
91 | # NOTE: Please make sure to change your handler code to work with those events
92 | # Check the event documentation for details
93 | # events:
94 | # - http:
95 | # path: users/create
96 | # method: get
97 | # - websocket: $connect
98 | # - s3: ${env:BUCKET}
99 | # - schedule: rate(10 minutes)
100 | # - sns: greeter-topic
101 | # - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000
102 | # - alexaSkill: amzn1.ask.skill.xx-xx-xx-xx
103 | # - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx
104 | # - iot:
105 | # sql: "SELECT * FROM 'some_topic'"
106 | # - cloudwatchEvent:
107 | # event:
108 | # source:
109 | # - "aws.ec2"
110 | # detail-type:
111 | # - "EC2 Instance State-change Notification"
112 | # detail:
113 | # state:
114 | # - pending
115 | # - cloudwatchLog: '/aws/lambda/hello'
116 | # - cognitoUserPool:
117 | # pool: MyUserPool
118 | # trigger: PreSignUp
119 | # - alb:
120 | # listenerArn: arn:aws:elasticloadbalancing:us-east-1:XXXXXX:listener/app/my-load-balancer/50dc6c495c0c9188/
121 | # priority: 1
122 | # conditions:
123 | # host: example.com
124 | # path: /hello
125 |
126 | # Define function environment variables here
127 | # environment:
128 | # variable2: value2
129 |
130 | # you can add CloudFormation resource templates here
131 |
132 | resources:
133 | Resources:
134 | urlsTable:
135 | Type: AWS::DynamoDB::Table
136 | Properties:
137 | AttributeDefinitions:
138 | - AttributeName: url
139 | AttributeType: S
140 | BillingMode: PAY_PER_REQUEST
141 | KeySchema:
142 | - AttributeName: url
143 | KeyType: HASH
144 | TableName: ${{self:service}}-urls-${{opt:stage, 'dev'}}
145 | #resources:
146 | # Resources:
147 | # NewResource:
148 | # Type: AWS::S3::Bucket
149 | # Properties:
150 | # BucketName: my-new-bucket
151 | # Outputs:
152 | # NewOutput:
153 | # Description: "Description for the output"
154 | # Value: "Some output value"
155 |
156 | plugins:
157 | - serverless-pseudo-parameters
158 |
--------------------------------------------------------------------------------
/spider-service/src/functions/scraper.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 | const cheerio = require('cheerio')
3 | const axios = require('axios')
4 | const AWS = require('aws-sdk')
5 |
6 | module.exports.scraper = async event => {
7 | if (!Array.isArray(event.Records) ||
8 | event.Records.length <= 0) {
9 | console.log('Event object is not an array')
10 | return new Error('Event object is not an array')
11 | }
12 | const url = JSON.parse(event.Records[0].Sns.Message).url
13 | const page = await axios(url)
14 |
15 | try {
16 | const $ = await cheerio.load(page.data)
17 | const links = $('a')
18 | const dynamodb = new AWS.DynamoDB.DocumentClient()
19 | try {
20 | const linkPromises = links.map(function (link) {
21 | console.log('link', link)
22 | console.log('$(this).attr(\'href\').toString()', $(this).attr('href').toString())
23 | const putParam = {
24 | TableName: process.env.DYNAMODB_URLS_TABLE,
25 | Item: {
26 | url: $(this).attr('href').toString()
27 | }
28 | }
29 | return dynamodb.put(putParam).promise()
30 | })
31 | await Promise.all(linkPromises)
32 | } catch (putError) {
33 | console.log('putError', putError)
34 | return putError
35 | }
36 |
37 | const heading1 = $('h1')
38 | const headingText = Object.values(await heading1.map(function(heading) {
39 | return $(this).text()
40 | }))
41 |
42 |
43 | } catch (pageLoadError) {
44 | console.log('There was a problem loading the page')
45 | console.log('pageLoadError', pageLoadError)
46 | }
47 | return true
48 | };
49 |
--------------------------------------------------------------------------------
/spider-service/src/functions/startScraping.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 | const AWS = require('aws-sdk')
3 | module.exports.startScraping = async event => {
4 | const batchSize = process.env.SCRAPER_BATCH_SIZE
5 | const sns = new AWS.SNS()
6 | const messageParams = {
7 | TopicArn: process.env.SNS_SCRAPE_PAGE_TOPIC,
8 | Message: JSON.stringify({
9 | url: 'http://serverless.com'
10 | })
11 | }
12 | try {
13 | const result = await sns.publish(messageParams).promise()
14 | console.log('result', result)
15 | return true
16 | } catch(snsError) {
17 | console.log('There was an error using SNS')
18 | console.log('snsError', snsError)
19 | console.log('messageParams', messageParams)
20 | return snsError
21 | }
22 | }
23 |
--------------------------------------------------------------------------------