├── .babelrc
├── .eslintrc
├── .gitignore
├── README.md
├── config
    └── .gitkeep
├── functions
    ├── alerting
    │   └── stats-health.js
    ├── api
    │   ├── alexa-ranking.js
    │   ├── color-bar.js
    │   ├── color-detection.js
    │   ├── css-stats.js
    │   ├── html-stats.js
    │   ├── http-headers.js
    │   ├── latest-screenshot.js
    │   ├── lighthouse.js
    │   ├── optimize-image.js
    │   └── technology-detection.js
    └── daily
    │   ├── alexa-ranking.js
    │   ├── color-bar.js
    │   ├── color-detection.js
    │   ├── convert-images.js
    │   ├── css-stats.js
    │   ├── html-stats.js
    │   ├── http-headers.js
    │   ├── lighthouse.js
    │   └── technology-detection.js
├── layers
    ├── canvas-lib64-layer.zip
    └── curl-settings
    │   └── nodejs
    │       ├── curl-format.json
    │       ├── package-lock.json
    │       └── package.json
├── package-lock.json
├── package.json
├── serverless.yml
├── utils
    └── index.js
└── webpack.config.js


/.babelrc:
--------------------------------------------------------------------------------
1 | {
2 |   "presets": [
3 |     ["@babel/env", {"modules": false}]
4 |   ]
5 | }


--------------------------------------------------------------------------------
/.eslintrc:
--------------------------------------------------------------------------------
  1 | {
  2 |   "rules": {
  3 |     // common
  4 |     "indent": ["warn", 2, {"SwitchCase": 1}], // specify tab or space width for your code
  5 |     "quotes": ["error", "single", { "allowTemplateLiterals": true }], // specify whether backticks, double or single quotes should be used
  6 |     "linebreak-style": ["error", "unix"], // disallow mixed 'LF' and 'CRLF' as linebreaks
  7 |     "semi": ["error", "always"], // require or disallow use of semicolons instead of ASI
  8 |     // errors
  9 |     "no-extra-parens": ["error", "functions"], // disallow unnecessary parentheses
 10 |     "no-unexpected-multiline": ["error"], // Avoid code that looks like two expressions but is actually one
 11 |     "valid-jsdoc": ["warn"], // Ensure JSDoc comments are valid
 12 |     // best practices
 13 |     "block-scoped-var": ["error"], // treat var statements as if they were block scoped
 14 |     "curly": ["error", "all"], // specify curly brace conventions for all control statements
 15 |     "default-case": ["error"], // require default case in switch statements
 16 |     "dot-notation": ["error"], // encourages use of dot notation whenever possible
 17 |     "dot-location": ["error", "property"], // enforces consistent newlines before or after dots
 18 |     "eqeqeq": ["error", "smart"], // require the use of === and !==
 19 |     "guard-for-in": ["error"], // make sure for-in loops have an if statement
 20 |     "no-alert": ["error"], // disallow the use of alert, confirm, and prompt
 21 |     "no-caller": ["error"], // disallow use of arguments.caller or arguments.callee
 22 |     "no-eq-null": ["error"], // disallow comparisons to null without a type-checking operator
 23 |     "no-eval": ["error"], // disallow use of eval()
 24 |     "no-extend-native": ["error"], // disallow adding to native types
 25 |     "no-extra-bind": ["error"], // disallow unnecessary function binding
 26 |     "no-fallthrough": ["error"], // disallow fallthrough of case statements (recommended)
 27 |     "no-floating-decimal": ["error"], // disallow the use of leading or trailing decimal points in numeric literals
 28 |     "no-implied-eval": ["error"], // disallow use of eval()-like methods
 29 |     "no-iterator": ["error"], // disallow usage of __iterator__ property
 30 |     "no-labels": ["error"], // disallow use of labeled statements
 31 |     "no-lone-blocks": ["error"], // disallow unnecessary nested blocks
 32 |     "no-loop-func": ["error"], // disallow creation of functions within loops
 33 |     "no-multi-spaces": ["error"], // disallow use of multiple spaces
 34 |     "no-multi-str": ["error"], // disallow use of multiline strings
 35 |     "no-native-reassign": ["error"], // disallow reassignments of native objects
 36 |     "no-new-func": ["error"], // disallow use of new operator for Function object
 37 |     "no-new-wrappers": ["error"], // disallows creating new instances of String,Number, and Boolean
 38 |     "no-new": ["error"], // disallow use of the new operator when not part of an assignment or comparison
 39 |     "no-octal-escape": ["error"], // disallow use of octal escape sequences in string literals, such as var foo = "Copyright \"error"51";
 40 |     "no-octal": ["error"], // disallow use of octal literals (recommended)
 41 |     "no-param-reassign": ["warn", {"props": false}], // disallow reassignment of function parameters
 42 |     "no-proto": ["error"], // disallow usage of __proto__ property
 43 |     "no-redeclare": ["error", {"builtinGlobals": true}], // disallow declaring the same variable more than once (recommended)
 44 |     "no-return-assign": ["error"], // disallow use of assignment in return statement
 45 |     "no-script-url": ["error"], // disallow use of javascript: urls.
 46 |     "no-self-compare": ["error"], // disallow comparisons where both sides are exactly the same
 47 |     "no-sequences": ["error"], // disallow use of the comma operator
 48 |     "no-throw-literal": ["error"], // restrict what can be thrown as an exception
 49 |     "no-unused-expressions": ["error"], // disallow usage of expressions in statement position
 50 |     "no-useless-call": ["error"], // disallow unnecessary .call() and .apply()
 51 |     "no-useless-concat": ["error"], // disallow unnecessary concatenation of literals or template literals
 52 |     "no-void": ["error"], // disallow use of the void operator
 53 |     "no-with": ["error"], // disallow use of the with statement
 54 |     "radix": ["error"], // require use of the second argument for parseInt()
 55 |     "wrap-iife": ["error"], // require immediate function invocation to be wrapped in parentheses
 56 |     // Variables
 57 |     "no-delete-var": ["error"], // disallow deletion of variables (recommended)
 58 |     "no-label-var": ["error"], // disallow labels that share a name with a variable
 59 |     "no-shadow-restricted-names": ["error"], // disallow shadowing of names such as arguments
 60 |     "no-undef-init": ["error"], // disallow use of undefined when initializing variables
 61 |     "no-undef": ["error"], // disallow use of undeclared variables unless mentioned in a /*global */ block (recommended)
 62 |     "no-unused-vars": ["error"], // disallow declaration of variables that are not used in the code (recommended)
 63 |     // nodejs
 64 |     "callback-return": ["error", ["callback", "cb", "next"]], // enforce return after a callback
 65 |     "handle-callback-err": ["error", "^(err\\d?|error\\d?|^.+Err$|^.+Error$)$"], // enforce error handling in callbacks
 66 |     "no-mixed-requires": ["error", false], // disallow mixing regular variable and require declarations
 67 |     "no-new-require": ["error"], // disallow use of new operator with the require function
 68 |     "no-path-concat": ["error"], // disallow string concatenation with __dirname and __filename
 69 |     "no-sync": ["error"], // disallow use of synchronous methods
 70 |     // Stylistic
 71 |     "array-bracket-spacing": ["warn", "never"], // enforce spacing inside array brackets
 72 |     "block-spacing": ["warn", "never"], // disallow or enforce spaces inside of single line blocks
 73 |     "brace-style": ["warn", "1tbs", { "allowSingleLine": false }], // enforce one true brace style
 74 |     "camelcase": ["warn", {"properties": "always"}], // require camel case names
 75 |     "comma-spacing": ["warn", {"before": false, "after": true}], // enforce spacing before and after comma
 76 |     "comma-style": ["warn", "last"], // enforce one true comma style
 77 |     "computed-property-spacing": ["warn", "never"], // require or disallow padding inside computed properties
 78 |     "consistent-this": ["warn", "self"], // enforce consistent naming when capturing the current execution context
 79 |     "eol-last": ["off"], // enforce newline at the end of file, with no multiple empty lines
 80 |     "key-spacing": ["warn", {"beforeColon": false, "afterColon": true}], // enforce spacing between keys and values in object literal properties
 81 |     "max-nested-callbacks": ["warn", 6], // specify the maximum depth callbacks can be nested
 82 |     "new-cap": ["warn", {"capIsNewExceptions": ["Router"]}], // require a capital letter for constructors
 83 |     "new-parens": ["warn"], // disallow the omission of parentheses when invoking a constructor with no arguments
 84 |     "newline-after-var": ["off", "always"], // require or disallow an empty newline after variable declarations
 85 |     "no-array-constructor": ["warn"], // disallow use of the Array constructor
 86 |     "no-lonely-if": ["warn"], // disallow if as the only statement in an else block
 87 |     "no-mixed-spaces-and-tabs": ["error"], // disallow mixed spaces and tabs for indentation (recommended)
 88 |     "no-multiple-empty-lines": ["warn", {"max": 1}], // disallow multiple empty lines
 89 |     "no-nested-ternary": ["warn"], // disallow nested ternary expressions
 90 |     "no-new-object": ["warn"], // disallow the use of the Object constructor
 91 |     "no-spaced-func": ["warn"], // disallow space between function identifier and application
 92 |     "no-trailing-spaces": ["warn", { "skipBlankLines": false }], // disallow trailing whitespace at the end of lines
 93 |     "no-unneeded-ternary": ["error"], // disallow the use of Boolean literals in conditional expressions
 94 |     "operator-linebreak": ["warn", "after"], // enforce operators to be placed before or after line breaks
 95 |     "quote-props": ["error", "as-needed"], // require quotes around object literal property names
 96 |     "semi-spacing": ["warn", {"before": false, "after": true}], // enforce spacing before and after semicolons
 97 |     "space-in-parens": ["warn", "never"], // require or disallow spaces inside parentheses
 98 |     "space-unary-ops": ["warn", {"words": true, "nonwords": false}], // require or disallow spaces before/after unary operators
 99 |     "spaced-comment": ["warn", "always"], // require or disallow a space immediately following the // or /* in a comment
100 |     "wrap-regex": ["warn"], // require regex literals to be wrapped in parentheses
101 |     "no-console": ["off"]
102 |   },
103 |   "parserOptions": {
104 |     "ecmaVersion": 2018,
105 |     "sourceType": "module"
106 |   },
107 |   "env": {
108 |     "node": true,
109 |     "es6": true,
110 |     "mocha": true
111 |   },
112 |   "extends": "eslint:recommended",
113 |   "globals": {
114 |     "should": true
115 |   }
116 | }


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Logs
 2 | logs
 3 | *.log
 4 | npm-debug.log*
 5 | yarn-debug.log*
 6 | yarn-error.log*
 7 | lerna-debug.log*
 8 | 
 9 | # Diagnostic reports (https://nodejs.org/api/report.html)
10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
11 | 
12 | # Runtime data
13 | pids
14 | *.pid
15 | *.seed
16 | *.pid.lock
17 | 
18 | # Directory for instrumented libs generated by jscoverage/JSCover
19 | lib-cov
20 | 
21 | # Coverage directory used by tools like istanbul
22 | coverage
23 | *.lcov
24 | 
25 | # nyc test coverage
26 | .nyc_output
27 | 
28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
29 | .grunt
30 | 
31 | # Bower dependency directory (https://bower.io/)
32 | bower_components
33 | 
34 | # node-waf configuration
35 | .lock-wscript
36 | 
37 | # Compiled binary addons (https://nodejs.org/api/addons.html)
38 | build/Release
39 | 
40 | # Dependency directories
41 | node_modules/
42 | jspm_packages/
43 | 
44 | # TypeScript v1 declaration files
45 | typings/
46 | 
47 | # TypeScript cache
48 | *.tsbuildinfo
49 | 
50 | # Optional npm cache directory
51 | .npm
52 | 
53 | # Optional eslint cache
54 | .eslintcache
55 | 
56 | # Optional REPL history
57 | .node_repl_history
58 | 
59 | # Output of 'npm pack'
60 | *.tgz
61 | 
62 | # Yarn Integrity file
63 | .yarn-integrity
64 | 
65 | # dotenv environment variables file
66 | .env
67 | .env.test
68 | 
69 | # parcel-bundler cache (https://parceljs.org/)
70 | .cache
71 | 
72 | # next.js build output
73 | .next
74 | 
75 | # nuxt.js build output
76 | .nuxt
77 | 
78 | # vuepress build output
79 | .vuepress/dist
80 | 
81 | # Serverless directories
82 | .serverless/
83 | 
84 | # FuseBox cache
85 | .fusebox/
86 | 
87 | # DynamoDB Local files
88 | .dynamodb/
89 | 
90 | labs/
91 | tmp/
92 | 
93 | config/*
94 | 
95 | # keep serverless configuration folder
96 | !config/.gitkeep
97 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # webcodex
 2 | A set of functions to collect data and metrics from websites.
 3 | 
 4 | ## Architecture
 5 | The idea behind all functions are to act as thin layers to accommodate some input parameters, do some action and transform the result to satify some output (or to have a specific side-effect, like daily functions). Most of the functions are designed to focus on a single metric but there are few expections (like the color detection).
 6 | ### HTTP functions
 7 | 
 8 | <p align="center">
 9 |   <img width="750" src="https://imgur.com/N6wjNwk.png">
10 | </p>
11 | 
12 | ### Daily functions
13 | These functions are persisting the metrics into DynamoDB and are triggered by an SNS message originated externally.
14 | <p align="center">
15 |   <img width="750" src="https://imgur.com/2DheX42.png">
16 | </p>
17 | 
18 | ## Project structure
19 | There are two important types of functions included in this project: functions that run on a daily basis and functions that are triggered by HTTP requests. The goal is to be able to re-use as much shared logic as possible between these two different types. The `utils` module is where the shared logic between these two live. In order to bundle just the dependencies needed for a given function the project uses Webpack and defines the necessary imports on a per-function basis, passing these dependencies back to the `utils` module.
20 | 
21 | ## Metrics
22 | ### Colors defined in the CSS
23 | To get the data for this metric a request is made to the website and all the stylesheets are aggregated. After that, the following  regular expression is used to match hexadecimal (e.g. both `#AAA` and `#AAAAAA), rgb and hsla definitions:
24 | ```
25 | /(#([\da-f]{3}){1,2}|(rgb|hsl)a\((\d{1,3}%?,\s?){3}(1|0?\.\d+)\)|(rgb|hsl)\(\d{1,3}%?(,\s?\d{1,3}%?){2}\))/ig
26 | ```
27 | Finally after the list of colors is obtained the list is sorted using [color-sorter](https://github.com/bartveneman/color-sorter).
28 | 
29 | ### Color detection
30 | This function uses a headless browser to capture a screenshot and utilizes imagemagick to extract predominant colors from a 50x50 matrix.
31 | ### HTML
32 | This function traverses the DOM tree and aggregates into DOM elements frequency and attributes frequency.
33 | ### CSS
34 | The function first tries to retrieve the CSS content of the website using [axios]() and if that fails then retries using cURL with an specific set of headers (due to browser fringerprinting). After getting the whole CSS content of the website the function uses Project Wallace's [css-analyzer](https://github.com/projectwallace/css-analyzer) to get the CSS metrics.
35 | ### HTTP Headers
36 | The purpose of this function is to collect the HTTP headers using cURL.
37 | ### Lighthouse
38 | The function spins a headless browser to then connect Lighthouse to it and run basic audits. After getting the results back, it uploads the results to Github's gists and returns the 
39 | ### Technologies detection
40 | The function focuses on identifyng libraries or frameworks that are loaded in a website. To achieve this, the function runs a headless browser and collects all the `<script>` links **after the website is fully loaded**. Once the complete list of JS resources is obtained the function downloads every resource, concatenates them and finds matches with regular expressions.
41 | 
42 | Why not use cheerio for that? Some popular websites started modifying the `<head>` element dynamically adding JavaScript bundles on-the-fly, making the task of grouping all the JS resources while not "running" any JavaScript within cheerio very difficult.
43 | 
44 | 
45 | ## Utility functions
46 | ### Convert-images
47 | The function is built to convert screenshot images from PNG format to webp (and crops the image too) so that an up-to-date screenshot for every designated website is easily accessed.
48 | ### Optimize image
49 | The function can receive an image url and will return the same image converted to webp format in a base64 encoding format.
50 | 
51 | ## Data store
52 | The metrics collected daily are being stored in DynamoDB. Every update to DynamoDB is designed to replace data at the property level, in a way similar to what an update with `:set` in MongoDB would do.
53 | 
54 | ## Next steps
55 | There is a low percentage of errors on current executions and although the project does not aim to be able to target 100% of the websites out there, it would be good to know why those failures happen and where is the limitation.


--------------------------------------------------------------------------------
/config/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/a0viedo/webcodex/68243a456efd4e5b964789d6775e199250a89712/config/.gitkeep


--------------------------------------------------------------------------------
/functions/alerting/stats-health.js:
--------------------------------------------------------------------------------
 1 | import { getDateForToday, scanDatabase2 } from '../../utils';
 2 | import { format } from 'date-fns';
 3 | exports.handler = async function handler(event, context) {
 4 |   const items = await scanDatabase2({
 5 |     date: getDateForToday(format)
 6 |   });
 7 |   console.log(`got ${items.length} results`);
 8 |   const fields = process.env.STATS_FIELDS.split(',');
 9 | 
10 |   let totalFailures = 0;
11 |   const failuresByField = {};
12 |   fields.forEach(field => {
13 |     failuresByField[field] = 0;
14 |     items.forEach(item => {
15 |       if(item[field] === '' || !item[field]) {
16 |         failuresByField[field]++;
17 |         totalFailures++;
18 |       }
19 |     });
20 |   });
21 |   console.log('total failures:', totalFailures);
22 |   console.log('failures by stat:', failuresByField);
23 | 
24 |   // TODO: if totalFailures is greater than X, send a telegram message
25 |   // TODO: if any property in failuresByField is greater than X%, send a telegram message
26 | };


--------------------------------------------------------------------------------
/functions/api/alexa-ranking.js:
--------------------------------------------------------------------------------
 1 | import { httpResponse, getAlexaRanking } from '../../utils';
 2 | import * as axios from 'axios';
 3 | import { JSDOM } from 'jsdom';
 4 | exports.handler = async (event, context) => {
 5 |   const domain = `${event.pathParameters.domain}`;
 6 |   global.console.log(`getting alexa ranking 2 for domain ${domain}`);
 7 |   try {
 8 |     const rank = await getAlexaRanking(axios, JSDOM, domain);
 9 |     return httpResponse(200, {
10 |       alexaRank: rank
11 |     });
12 |   } catch(err) {
13 |     global.console.log(err);
14 |     return httpResponse(500);
15 |   }
16 | };


--------------------------------------------------------------------------------
/functions/api/color-bar.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import * as colorSort from 'color-sorter';
 3 | import * as axios from 'axios';
 4 | import * as allSettled from 'promise-all-settled';
 5 | import { httpResponse, getColorBar } from '../../utils';
 6 | 
 7 | exports.handler = async (event, context) => {
 8 |   const domain = `${event.pathParameters.domain}`;
 9 |   try {
10 |     console.log(`getting color bar for domain ${domain}`);
11 |     const svg = await getColorBar(getCss, colorSort, domain, axios, allSettled, event.queryStringParameters && event.queryStringParameters.width);
12 |     return httpResponse(200, svg);
13 |   } catch(err) {
14 |     console.log(err);
15 |     return httpResponse(500);
16 |   }
17 | };


--------------------------------------------------------------------------------
/functions/api/color-detection.js:
--------------------------------------------------------------------------------
 1 | import * as chromeLauncher from 'chrome-launcher';
 2 | import * as chrome from 'chrome-aws-lambda';
 3 | import * as puppeteer from 'puppeteer-core';
 4 | import { httpResponse, getColorDetection, uploadToS3, getDateForToday, deleteFile } from '../../utils';
 5 | import { readFile as readFileCallback } from 'fs';
 6 | import { promisify } from 'util';
 7 | import {format} from 'date-fns';
 8 | import * as axios from 'axios';
 9 | 
10 | const readFile = promisify(readFileCallback);
11 | 
12 | exports.handler = async (event, context) => {
13 |   const domain = `${event.pathParameters.domain}`;
14 |   console.log('domain', domain);
15 |   try {
16 |     const { colors, screenshotPath } = await getColorDetection(chromeLauncher, chrome, puppeteer, axios, domain);
17 |     console.log('colors', colors);
18 |     console.log(screenshotPath);
19 | 
20 |     const { 0: { Location } } = await Promise.all([
21 |       uploadToS3(process.env.S3_SCREENSHOT_BUCKET, `${encodeURIComponent(domain)}/${getDateForToday(format)}.png`, await readFile(screenshotPath)),
22 |     ]);
23 |     console.log('Location', Location);
24 | 
25 |     await deleteFile(screenshotPath);
26 |     return httpResponse(200, {
27 |       colors,
28 |       screenshotURL: Location
29 |     });
30 | 
31 |   } catch (error) {
32 |     console.log(error);
33 |     return httpResponse(500);
34 |   }
35 | };
36 | 


--------------------------------------------------------------------------------
/functions/api/css-stats.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import * as analyze from '@projectwallace/css-analyzer';
 3 | import * as axios from 'axios';
 4 | import { httpResponse, getCSSAnalysis } from '../../utils';
 5 | 
 6 | exports.handler = async (event, context) => {
 7 |   const domain = `${event.pathParameters.domain}`;
 8 |   try {
 9 |     const result = await getCSSAnalysis(getCss, analyze, axios, domain);
10 |     global.console.log('css result', result);
11 |     return httpResponse(200, result);
12 |   } catch(err) {
13 |     global.console.log('error', err);
14 |     return httpResponse(500);
15 |   }
16 | };
17 | 


--------------------------------------------------------------------------------
/functions/api/html-stats.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import { JSDOM } from 'jsdom';
 3 | import { httpResponse, getHTMLAnalysis } from '../../utils';
 4 | import fs from 'fs';
 5 | 
 6 | exports.handler = async (event, context) => {
 7 |   const domain = decodeURIComponent(event.pathParameters.domain);
 8 |   global.console.log('domain', domain);
 9 |   try {
10 |     console.log('content of /usr/lib64');
11 |     console.log(fs.readdirSync("/usr/lib64").filter(p => p.match(/\.so/)).sort().join("\n"));
12 |     const result = await getHTMLAnalysis(getCss, JSDOM, domain);
13 |     global.console.log('html result', result);
14 |     return httpResponse(200, result);
15 |   } catch(err) {
16 |     global.console.log('error', err);
17 |     return httpResponse(500);
18 |   }
19 | };


--------------------------------------------------------------------------------
/functions/api/http-headers.js:
--------------------------------------------------------------------------------
 1 | import { httpResponse, getHTTPHeaders } from '../../utils';
 2 | 
 3 | exports.handler = async (event, context) => {
 4 |   const domain = `${event.pathParameters.domain}`;
 5 | 
 6 |   try {
 7 |     const result = await getHTTPHeaders(domain);
 8 |     return httpResponse(200, result);
 9 |   } catch(err) {
10 |     console.log(err);
11 |     return httpResponse(500);
12 |   }
13 | };


--------------------------------------------------------------------------------
/functions/api/latest-screenshot.js:
--------------------------------------------------------------------------------
 1 | import { httpResponse, getLatestScreenshot } from '../../utils';
 2 | 
 3 | exports.handler = async (event, context) => {
 4 |   const domain = event.pathParameters.domain;
 5 |   try {
 6 |     const url = await getLatestScreenshot(domain);
 7 |     return httpResponse(200, url);
 8 |   } catch(err) {
 9 |     global.console.log(err);
10 |     return httpResponse(500);
11 |   }
12 | };


--------------------------------------------------------------------------------
/functions/api/lighthouse.js:
--------------------------------------------------------------------------------
 1 | import * as chromeLauncher from 'chrome-launcher';
 2 | import * as chrome from 'chrome-aws-lambda';
 3 | import * as lighthouse from 'lighthouse';
 4 | import { format } from 'date-fns';
 5 | import * as axios from 'axios';
 6 | import { httpResponse, getLighthouse2, uploadGist, getDateForToday } from '../../utils';
 7 | global.URL = global.URL || require('whatwg-url').URL;
 8 | 
 9 | exports.handler = async (event, context) => {
10 |   const domain = `${event.pathParameters.domain}`;
11 | 
12 |   try {
13 |     const result = await getLighthouse2(chromeLauncher, chrome, lighthouse, domain);
14 |     console.log('result', result);
15 |     const { id } = await uploadGist(axios, `Report generated for ${domain} on ${getDateForToday(format)}`, JSON.stringify(result));
16 |     return httpResponse(200, {
17 |       lighthouse: result,
18 |       viewReportURL: `${process.env.LIGHTHOUSE_REPORT_VIEWER_URL}${id}`
19 |     });
20 |   } catch(err) {
21 |     console.log(err);
22 |     return httpResponse(500);
23 |   }
24 | };
25 | 


--------------------------------------------------------------------------------
/functions/api/optimize-image.js:
--------------------------------------------------------------------------------
 1 | import * as fs from 'fs';
 2 | import * as imagemin from 'imagemin';
 3 | import * as imageminWebp from 'imagemin-webp';
 4 | import { httpResponse } from '../../utils';
 5 | import * as axios from 'axios';
 6 | 
 7 | exports.handler = async (event, context) => {
 8 |   const url = decodeURIComponent(event.queryStringParameters.url);
 9 |   try {
10 |     global.console.log('url', url);
11 |     const { data: streamResponse } = await axios({
12 |       url,
13 |       method: 'GET',
14 |       responseType: 'stream'
15 |     });
16 | 
17 |     const filename = Math.round(Math.random()*10e6);
18 |     const fileStream = fs.createWriteStream(`/tmp/${filename}.png`);
19 | 
20 |     streamResponse.pipe(fileStream);
21 | 
22 |     const waitForDownload = function () {
23 |       return new Promise((resolve, reject) => {
24 |         fileStream.on('finish', () => {
25 |           global.console.log('finished');
26 |           resolve();
27 |         });
28 |         fileStream.on('error', () => {
29 |           global.console.log('error');
30 |           reject();
31 |         });
32 |       });
33 |     };
34 | 
35 |     await waitForDownload();
36 | 
37 |     global.console.log('file downloaded');
38 | 
39 |     const files = await imagemin([`/tmp/${filename}.png`], {
40 |       plugins: [
41 |         imageminWebp({
42 |           quality: Number(process.env.IMAGE_QUALITY),
43 |           crop: {
44 |             x: 0,
45 |             y: 0,
46 |             width: Number(process.env.IMAGE_WIDTH),
47 |             height: Number(process.env.IMAGE_HEIGHT)
48 |           }
49 |         })
50 |       ]
51 |     });
52 | 
53 |     global.console.log('conversion finished', files);
54 |     return {
55 |       statusCode: 200,
56 |       headers: { 'Content-Type': 'image/webp' },
57 |       body: files[0].data.toString('base64'),
58 |       isBase64Encoded: true
59 |     };
60 |   } catch(err) {
61 |     global.console.log('error', err);
62 |     return httpResponse(500);
63 |   }
64 | };
65 | 


--------------------------------------------------------------------------------
/functions/api/technology-detection.js:
--------------------------------------------------------------------------------
 1 | import * as chrome from 'chrome-aws-lambda';
 2 | import * as puppeteer from 'puppeteer-core';
 3 | import * as axios from 'axios';
 4 | import * as allSettled from 'promise-all-settled';
 5 | import { decompress } from 'brotli';
 6 | import { httpResponse, getTechnologiesUsed } from '../../utils';
 7 | 
 8 | exports.handler = async (event, context) => {
 9 |   const domain = `${event.pathParameters.domain}`;
10 |   try {
11 |     const result = await getTechnologiesUsed(chrome, puppeteer, axios, allSettled, decompress, domain);
12 |     global.console.log(result);
13 |     return httpResponse(200, result);
14 |   } catch(err) {
15 |     console.log(err);
16 |     return httpResponse(500);
17 |   }
18 | };


--------------------------------------------------------------------------------
/functions/daily/alexa-ranking.js:
--------------------------------------------------------------------------------
 1 | import * as axios from 'axios';
 2 | import { JSDOM } from 'jsdom';
 3 | import { format } from 'date-fns';
 4 | import { getAlexaRanking, getSNSPayload, updateDocument, getDateForToday } from '../../utils';
 5 | 
 6 | exports.handler = async function handler(event, context) {
 7 |   const { domain } = getSNSPayload(event);
 8 |   global.console.log(`getting alexa ranking for domain ${domain}`);
 9 |   const rank = await getAlexaRanking(axios, JSDOM, domain);
10 |   await updateDocument(domain, getDateForToday(format), { alexaRank: rank });
11 | };


--------------------------------------------------------------------------------
/functions/daily/color-bar.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import * as colorSort from 'color-sorter';
 3 | import * as axios from 'axios';
 4 | import * as allSettled from 'promise-all-settled';
 5 | import { format } from 'date-fns';
 6 | import { getColorBar, getSNSPayload, updateDocument, getDateForToday } from '../../utils';
 7 | 
 8 | exports.handler = async function handler(event, context) {
 9 |   const { domain } = getSNSPayload(event);
10 |   global.console.log(`getting color bar for domain ${domain}`);
11 |   const svg = await getColorBar(getCss, colorSort, domain, axios, allSettled, event.queryStringParameters && event.queryStringParameters.width);
12 |   await updateDocument(domain, getDateForToday(format), {
13 |     colorBar: svg
14 |   });
15 | };


--------------------------------------------------------------------------------
/functions/daily/color-detection.js:
--------------------------------------------------------------------------------
 1 | import * as chromeLauncher from 'chrome-launcher';
 2 | import * as chrome from 'chrome-aws-lambda';
 3 | import * as puppeteer from 'puppeteer-core';
 4 | import * as axios from 'axios';
 5 | import { readFile as readFileCallback } from 'fs';
 6 | import { promisify } from 'util';
 7 | import {format} from 'date-fns';
 8 | import { getColorDetection2, getSNSPayload, updateDocument, getDateForToday, uploadToS3, getMD5, runCommand } from '../../utils';
 9 | const readFile = promisify(readFileCallback);
10 | 
11 | exports.handler = async (event, context) => {
12 |   const { domain } = await getSNSPayload(event);
13 | 
14 |   try {
15 |     const { colors, screenshotPath } = await getColorDetection2(chromeLauncher, chrome, puppeteer, axios, domain);
16 |     console.log('colors', colors);
17 |     console.log(screenshotPath);
18 |     const date = getDateForToday(format);
19 |     const file = await readFile(screenshotPath);
20 |     const [{ Location }] = await Promise.all([
21 |       await uploadToS3(process.env.S3_SCREENSHOT_BUCKET, `${encodeURIComponent(domain)}/${date}.png`, file),
22 |       await uploadToS3(process.env.S3_SCREENSHOT_BUCKET, `${getMD5(domain)}.png`, file)
23 |     ]);
24 |     console.log('Location', Location);
25 |     await Promise.all([
26 |       updateDocument(domain, date, {
27 |         colorCount: colors,
28 |         screenshotURL: Location
29 |       }),
30 |       runCommand('rm -r /tmp/*')
31 |     ]);
32 |   } catch (error) {
33 |     console.log(error);
34 |   }
35 | };


--------------------------------------------------------------------------------
/functions/daily/convert-images.js:
--------------------------------------------------------------------------------
 1 | import * as AWS from 'aws-sdk';
 2 | import { uploadToS3, runCommand } from '../../utils';
 3 | import * as fs from 'fs';
 4 | import * as path from 'path';
 5 | import * as imagemin from 'imagemin';
 6 | import * as imageminWebp from 'imagemin-webp';
 7 | import * as util from 'util';
 8 | 
 9 | const s3 = new AWS.S3();
10 | const writeFile = util.promisify(fs.writeFile);
11 | 
12 | exports.handler = async function handler(event, context) {
13 |   const filesInBucket = await listFilesInBucket(process.env.S3_BUCKET);
14 | 
15 |   console.log('files in bucket', filesInBucket);
16 | 
17 |   const imageFiles = filesInBucket.filter(x=> path.extname(x.Key) === '.png');
18 | 
19 |   console.log('image files', imageFiles);
20 | 
21 |   await Promise.all(imageFiles.map(file => downloadFile(`/tmp/${file.Key}`, process.env.S3_BUCKET, file.Key)));
22 | 
23 |   console.log('downloaded image files');
24 |   const files = await imagemin(['/tmp/*.{jpg,png}'], {
25 |     plugins: [
26 |       imageminWebp({
27 |         quality: Number(process.env.IMAGE_QUALITY),
28 |         crop: {
29 |           x: 0,
30 |           y: 0,
31 |           width: Number(process.env.IMAGE_WIDTH),
32 |           height: Number(process.env.IMAGE_HEIGHT)
33 |         }
34 |       })
35 |     ]
36 |   });
37 | 
38 |   // uplood files
39 |   const result = await Promise.all(files.map(file => uploadFile(file)));
40 |   console.log('finished optimizing images', result);
41 | 
42 |   await runCommand('rm -r /tmp/*');
43 | };
44 | 
45 | async function uploadFile(file) {
46 |   return uploadToS3(process.env.S3_BUCKET, path.basename(file.sourcePath).replace(/\.png/, '.webp'), file.data);
47 | }
48 | 
49 | async function listFilesInBucket(bucket) {
50 |   var params = {
51 |     Bucket: bucket,
52 |     Delimiter: '/',
53 |   };
54 |   const { Contents } = await s3.listObjectsV2(params).promise();
55 |   return Contents;
56 | }
57 | 
58 | async function downloadFile(filePath, bucketName, key){
59 |   const params = {
60 |     Bucket: bucketName,
61 |     Key: key
62 |   };
63 |   const data = await s3.getObject(params).promise();
64 |   return writeFile(filePath, data.Body);
65 | }


--------------------------------------------------------------------------------
/functions/daily/css-stats.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import * as analyze from '@projectwallace/css-analyzer';
 3 | import { getCSSAnalysis, updateDocument, getDateForToday, getSNSPayload } from '../../utils';
 4 | import { format } from 'date-fns';
 5 | import * as axios from 'axios';
 6 | 
 7 | exports.handler = async function handler(event, context) {
 8 |   const { domain } = getSNSPayload(event);
 9 |   global.console.log(`getting css stats for domain ${domain}`);
10 | 
11 |   const result = await getCSSAnalysis(getCss, analyze, axios, domain);
12 |   await updateDocument(domain, getDateForToday(format), {
13 |     cssStats: result
14 |   });
15 | };


--------------------------------------------------------------------------------
/functions/daily/html-stats.js:
--------------------------------------------------------------------------------
 1 | import * as getCss from 'get-css';
 2 | import { JSDOM } from 'jsdom';
 3 | import { format } from 'date-fns';
 4 | import { getHTMLAnalysis, getSNSPayload, updateDocument, getDateForToday } from '../../utils';
 5 | 
 6 | exports.handler = async function (event, context) {
 7 |   const { domain } = await getSNSPayload(event);
 8 |   global.console.log(`getting html stats for domain ${domain}`);
 9 |   const result = await getHTMLAnalysis(getCss, JSDOM, domain);
10 |   await updateDocument(domain, getDateForToday(format), {
11 |     htmlStats: result
12 |   });
13 | };


--------------------------------------------------------------------------------
/functions/daily/http-headers.js:
--------------------------------------------------------------------------------
 1 | import { getHTTPHeaders, updateDocument, getSNSPayload, getDateForToday } from '../../utils';
 2 | import { format } from 'date-fns';
 3 | 
 4 | exports.handler = async function (event, context) {
 5 |   const { domain } = await getSNSPayload(event);
 6 |   global.console.log(`getting http headers for domain ${domain}`);
 7 |   const result = await getHTTPHeaders(domain);
 8 |   await updateDocument(domain, getDateForToday(format), {
 9 |     httpHeaders: result
10 |   });
11 | };


--------------------------------------------------------------------------------
/functions/daily/lighthouse.js:
--------------------------------------------------------------------------------
 1 | import * as chromeLauncher from 'chrome-launcher';
 2 | import * as chrome from 'chrome-aws-lambda';
 3 | import * as lighthouse from 'lighthouse';
 4 | import * as axios from 'axios';
 5 | import { getLighthouse2, updateDocument, getSNSPayload, getDateForToday, uploadGist } from '../../utils';
 6 | import { format } from 'date-fns';
 7 | 
 8 | global.URL = global.URL || require('whatwg-url').URL;
 9 | 
10 | exports.handler = async function (event, context) {
11 |   const { domain } = await getSNSPayload(event);
12 |   global.console.log(`getting lighthouse report for domain ${domain}`);
13 |   const result = await getLighthouse2(chromeLauncher, chrome, lighthouse, domain);
14 |   console.log('result', result);
15 |   const { id, url } = await uploadGist(axios, `Report generated for ${domain} on ${getDateForToday(format)}`, JSON.stringify(result));
16 | 
17 |   const lighthouseResults = {
18 |     categories: {
19 |       performance: result.categories.performance.score,
20 |       pwa: result.categories.pwa.score,
21 |       accessibility: result.categories.accessibility.score,
22 |       seo: result.categories.seo.score,
23 |       'best-practices': result.categories['best-practices'].score
24 |     },
25 |     audits: {
26 |       'first-contentful-paint': result.audits['first-contentful-paint'].numericValue,
27 |       'time-to-interactive': result.audits.interactive.numericValue,
28 |       'first-cpu-idle': result.audits['first-cpu-idle'].numericValue,
29 |       'first-meaningful-paint': result.audits['first-meaningful-paint'].numericValue,
30 |       'time-to-first-byte': result.audits['time-to-first-byte'].numericValue
31 |     }
32 |   };
33 |   await updateDocument(domain, getDateForToday(format), {
34 |     lighthouse: lighthouseResults,
35 |     lighthouseReportURL: url,
36 |     lighthouseViewReportURL: `${process.env.LIGHTHOUSE_REPORT_VIEWER_URL}${id}`
37 |   });
38 | };


--------------------------------------------------------------------------------
/functions/daily/technology-detection.js:
--------------------------------------------------------------------------------
 1 | import * as chrome from 'chrome-aws-lambda';
 2 | import * as puppeteer from 'puppeteer-core';
 3 | import * as axios from 'axios';
 4 | import * as allSettled from 'promise-all-settled';
 5 | import { format } from 'date-fns';
 6 | import { decompress } from 'brotli';
 7 | import { getTechnologiesUsed, updateDocument, getSNSPayload, getDateForToday } from '../../utils';
 8 | 
 9 | exports.handler = async function (event, context) {
10 |   const { domain } = await getSNSPayload(event);
11 |   global.console.log(`getting technologies for domain ${domain}`);
12 |   const technologies = await getTechnologiesUsed(chrome, puppeteer, axios, allSettled, decompress, domain);
13 |   await updateDocument(domain, getDateForToday(format), {
14 |     technologies
15 |   });
16 | };


--------------------------------------------------------------------------------
/layers/canvas-lib64-layer.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/a0viedo/webcodex/68243a456efd4e5b964789d6775e199250a89712/layers/canvas-lib64-layer.zip


--------------------------------------------------------------------------------
/layers/curl-settings/nodejs/curl-format.json:
--------------------------------------------------------------------------------
 1 | {
 2 | "time_namelookup": %{time_namelookup},
 3 | "time_connect": %{time_connect},
 4 | "time_appconnect": %{time_appconnect},
 5 | "time_pretransfer": %{time_pretransfer},
 6 | "time_redirect": %{time_redirect},
 7 | "time_starttransfer": %{time_starttransfer},
 8 | "time_total": %{time_total},
 9 | "speed_download": %{speed_download},
10 | "speed_upload": %{speed_upload},
11 | "remote_ip": "%{remote_ip}",
12 | "remote_port": "%{remote_port}",
13 | "local_ip": "%{local_ip}",
14 | "local_port": "%{local_port}",
15 | "size_download": "%{size_download}",
16 | "size_header": "%{size_header}",
17 | "size_request": "%{size_request}",
18 | "http_version": "%{http_version}",
19 | "http_code": "%{http_code}",
20 | "proxy_ssl_verify_result": "%{proxy_ssl_verify_result}"
21 | }
22 | 


--------------------------------------------------------------------------------
/layers/curl-settings/nodejs/package-lock.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "nodejs",
 3 |   "version": "1.0.0",
 4 |   "lockfileVersion": 1,
 5 |   "requires": true,
 6 |   "dependencies": {
 7 |     "curl-headers-to-json": {
 8 |       "version": "1.0.1",
 9 |       "resolved": "https://registry.npmjs.org/curl-headers-to-json/-/curl-headers-to-json-1.0.1.tgz",
10 |       "integrity": "sha512-PNbfCuu4UdsTmSAHwN4R0m34KzvOvDjP3mJ6XNfHzOdeTskUb/eaVV110y/SnT6h6HDA7fisCNgQvRP/vH9obQ=="
11 |     }
12 |   }
13 | }
14 | 


--------------------------------------------------------------------------------
/layers/curl-settings/nodejs/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "nodejs",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "keywords": [],
10 |   "author": "",
11 |   "license": "ISC",
12 |   "dependencies": {
13 |     "curl-headers-to-json": "^1.0.1"
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "webcodex",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "lint": "eslint functions/"
 8 |   },
 9 |   "keywords": [],
10 |   "author": "Alejandro Oviedo <alejandro.oviedo.g@gmail.com",
11 |   "license": "ISC",
12 |   "dependencies": {
13 |     "@projectwallace/css-analyzer": "^2.6.0",
14 |     "axios": "^0.21.1",
15 |     "brotli": "^1.3.2",
16 |     "chrome-launcher": "^0.11.2",
17 |     "color-sorter": "^4.0.0",
18 |     "curl-headers-to-json": "^1.0.1",
19 |     "date-fns": "^1.30.1",
20 |     "get-css": "^3.3.1",
21 |     "imagemin": "^7.0.0",
22 |     "imagemin-webp": "^5.1.0",
23 |     "jsdom": "^15.1.1",
24 |     "json-stringify-safe": "^5.0.1",
25 |     "lighthouse": "https://github.com/a0viedo/lighthouse",
26 |     "promise-all-settled": "^1.1.0",
27 |     "puppeteer": "^1.19.0",
28 |     "puppeteer-core": "^1.19.0",
29 |     "puppeteer-extra": "^2.1.3",
30 |     "puppeteer-extra-plugin-anonymize-ua": "^2.1.4",
31 |     "puppeteer-extra-plugin-stealth": "^2.2.2",
32 |     "whatwg-url": "^7.0.0"
33 |   },
34 |   "devDependencies": {
35 |     "@babel/core": "^7.5.5",
36 |     "@babel/node": "^7.5.5",
37 |     "@babel/preset-env": "^7.5.5",
38 |     "aws-sdk": "^2.503.0",
39 |     "chrome-aws-lambda": "^3.1.1",
40 |     "dotyaml": "^1.0.0",
41 |     "eslint": "^6.5.1",
42 |     "esm": "^3.2.25",
43 |     "rollup-plugin-babel": "^4.3.3",
44 |     "rollup-plugin-node-resolve": "^5.2.0",
45 |     "serverless-apigw-binary": "^0.4.4",
46 |     "serverless-apigwy-binary": "^1.0.0",
47 |     "serverless-dotenv-plugin": "^2.1.1",
48 |     "serverless-iam-roles-per-function": "^2.0.1",
49 |     "serverless-pseudo-parameters": "^2.4.0",
50 |     "serverless-scriptable-plugin": "^1.0.2",
51 |     "serverless-webpack": "^5.3.1",
52 |     "webpack": "^4.38.0",
53 |     "webpack-cli": "^3.3.7",
54 |     "webpack-node-externals": "^1.7.2",
55 |     "yaml": "^1.6.0"
56 |   }
57 | }
58 | 


--------------------------------------------------------------------------------
/serverless.yml:
--------------------------------------------------------------------------------
  1 | service: codex
  2 | 
  3 | provider:
  4 |   name: aws
  5 |   stage: ${opt:stage, env:NODE_ENV, "dev"}
  6 |   runtime: nodejs12.x
  7 |   environment: ${file(./config/${self:provider.stage}.yml)}
  8 |   memorySize: 256
  9 |   timeout: 10
 10 |   region: us-east-1
 11 | 
 12 | plugins:
 13 |   - serverless-pseudo-parameters
 14 |   - serverless-iam-roles-per-function
 15 |   - serverless-webpack
 16 |   - serverless-scriptable-plugin
 17 |   - serverless-apigw-binary
 18 | 
 19 | package:
 20 |   individually: true
 21 | 
 22 | custom:
 23 |   apigwBinary:
 24 |     types:
 25 |       - 'image/webp'
 26 |   scriptHooks:
 27 |     # before:package:createDeploymentArtifacts: npm run build
 28 |   webpack:
 29 |     webpackConfig: 'webpack.config.js'
 30 |     includeModules:
 31 |       forceExclude:
 32 |         - aws-sdk
 33 |         - chrome-aws-lambda
 34 |     packager: 'npm'
 35 | 
 36 | layers:
 37 |   curl:
 38 |     path: layers/curl-settings
 39 |     description: 'utility functions'
 40 |     compatibleRuntimes:
 41 |       - nodejs8.10
 42 |       - nodejs10.x
 43 |       - nodejs12.x
 44 |     licenseInfo: MIT
 45 |   canvas:
 46 |     package:
 47 |       artifact: layers/canvas-lib64-layer.zip
 48 | 
 49 | # TODO: move SNS access into read and write access
 50 | sqsAccess: &sqsAccess
 51 |   Effect: Allow
 52 |   Action:
 53 |     - sqs:*
 54 | snsAccess: &snsAccess
 55 |   Effect: Allow
 56 |   Action:
 57 |     - sns:*
 58 |   Resource: arn:aws:sns:${opt:region, self:provider.region}:#{AWS::AccountId}:${self:provider.environment.AWS_SNS}
 59 | dynamodbWriteAccess: &dynamodbWriteAccess
 60 |   Effect: Allow
 61 |   Action:
 62 |     - dynamodb:GetItem
 63 |     - dynamodb:PutItem
 64 |     - dynamodb:UpdateItem
 65 |   Resource: arn:aws:dynamodb:${opt:region, self:provider.region}:#{AWS::AccountId}:table/${self:provider.environment.AWS_DYNAMODB_TABLE}
 66 | dynamodbReadAccess: &dynamodbReadAccess
 67 |   Effect: Allow
 68 |   Action:
 69 |     - dynamodb:GetItem
 70 |     - dynamodb:Scan
 71 |     - dynamodb:Query
 72 |   Resource: arn:aws:dynamodb:${opt:region, self:provider.region}:#{AWS::AccountId}:table/${self:provider.environment.AWS_DYNAMODB_TABLE}
 73 | s3Access: &s3Access
 74 |   Effect: Allow
 75 |   Action:
 76 |     - s3:*
 77 |   Resource: 
 78 |     - "arn:aws:s3:::${self:provider.environment.S3_SCREENSHOT_BUCKET}/*"
 79 |     - "arn:aws:s3:::${self:provider.environment.S3_SCREENSHOT_BUCKET}"
 80 | 
 81 | snsFunctionTemplate: &snsFunctionTemplate
 82 |   iamRoleStatements:
 83 |   - *snsAccess
 84 |   - *dynamodbWriteAccess
 85 |   events:
 86 |     - sns: arn:aws:sns:${opt:region, self:provider.region}:#{AWS::AccountId}:${self:provider.environment.AWS_SNS}
 87 | 
 88 | httpFunctionTemplate: &httpFunctionTemplate
 89 |   iamRoleStatements:
 90 |   - *snsAccess
 91 |   - *dynamodbReadAccess
 92 | 
 93 | 
 94 | functions:
 95 | #  --------------------------------- daily functions -------------------------------------
 96 |   sns-color-detection:
 97 |     <<: *snsFunctionTemplate
 98 |     memorySize: 2048
 99 |     timeout: 75
100 |     environment: ${file(./config/fn.http-color-detection.yml)}
101 |     handler: functions/daily/color-detection.handler
102 |     layers:
103 |       - arn:aws:lambda:us-east-1:332013964457:layer:chrome-aws-lambda:1
104 |     iamRoleStatements:
105 |     - *snsAccess
106 |     - *dynamodbWriteAccess
107 |     - *s3Access
108 | 
109 |   sns-alexa-rank:
110 |     <<: *snsFunctionTemplate
111 |     timeout: 20
112 |     handler: functions/daily/alexa-ranking.handler
113 |     environment: ${file(./config/fn.alexa-ranking.yml)}
114 | 
115 |   sns-technology-detection:
116 |     <<: *snsFunctionTemplate
117 |     environment: ${file(./config/fn.technology-detection.yml)}
118 |     handler: functions/daily/technology-detection.handler
119 |     memorySize: 2048
120 |     timeout: 75
121 |     layers:
122 |       - arn:aws:lambda:us-east-1:332013964457:layer:chrome-aws-lambda:1
123 | 
124 |   sns-color-bar:
125 |     <<: *snsFunctionTemplate
126 |     environment: ${file(./config/fn.color-bar.yml)}
127 |     handler: functions/daily/color-bar.handler
128 |     timeout: 15
129 | 
130 |   sns-html-stats:
131 |     <<: *snsFunctionTemplate
132 |     timeout: 20
133 |     handler: functions/daily/html-stats.handler
134 |     environment: ${file(./config/fn.html-stats.yml)}
135 | 
136 |   sns-lighthouse:
137 |     <<: *snsFunctionTemplate
138 |     memorySize: 2048
139 |     timeout: 60
140 |     handler: functions/daily/lighthouse.handler
141 |     layers:
142 |       - arn:aws:lambda:us-east-1:332013964457:layer:chrome-aws-lambda:1
143 |     environment: ${file(./config/fn.lighthouse.yml)}
144 | 
145 |   sns-css-stats:
146 |     <<: *snsFunctionTemplate
147 |     timeout: 50
148 |     memorySize: 1024
149 |     handler: functions/daily/css-stats.handler
150 | 
151 |   sns-http-headers:
152 |     <<: *snsFunctionTemplate
153 |     handler: functions/daily/http-headers.handler
154 | 
155 |   convert-images:
156 |     timeout: 60
157 |     memorySize: 2048
158 |     handler: functions/daily/convert-images.handler
159 |     environment: ${file(./config/fn.convert-images.yml)}
160 |     iamRoleStatements:
161 |       - *s3Access
162 | #  ---------------------------------------- HTTP Functions ---------------------------------------
163 | 
164 |   http-color-detection:
165 |     environment: ${file(./config/fn.http-color-detection.yml)}
166 |     timeout: 60
167 |     memorySize: 1536
168 |     layers:
169 |       - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:10
170 |     handler: functions/api/color-detection.handler
171 |     iamRoleStatements:
172 |       - *snsAccess
173 |       - *dynamodbReadAccess
174 |       - Effect: Allow
175 |         Action:
176 |           - s3:*
177 |         Resource: 
178 |           - "arn:aws:s3:::webcodex-temporal/*"
179 |           - "arn:aws:s3:::webcodex-temporal"
180 |     events:
181 |       - http:
182 |           path: /try/color-detection/{domain}
183 |           method: GET
184 |           cors: true
185 | 
186 |   http-color-bar:
187 |     timeout: 15
188 |     environment: ${file(./config/fn.color-bar.yml)}
189 |     handler: functions/api/color-bar.handler
190 |     events:
191 |       - http:
192 |           path: /try/color-bar/{domain}
193 |           method: GET
194 |           cors: true
195 | 
196 |   http-technology-detection:
197 |     memorySize: 2048
198 |     timeout: 60
199 |     environment: ${file(./config/fn.technology-detection.yml)}
200 |     handler: functions/api/technology-detection.handler
201 |     layers:
202 |       - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:10
203 |     events:
204 |       - http:
205 |           path: /try/technology-detection/{domain}
206 |           method: GET
207 |           cors: true
208 | 
209 |   http-html-stats:
210 |     timeout: 20
211 |     handler: functions/api/html-stats.handler
212 |     layers:
213 |       - {Ref: CanvasLambdaLayer}
214 |     events:
215 |       - http:
216 |           path: /try/html-stats/{domain}
217 |           method: GET
218 |           cors: true
219 | 
220 |   http-alexa-ranking:
221 |     handler: functions/api/alexa-ranking.handler
222 |     environment: ${file(./config/fn.alexa-ranking.yml)}
223 |     layers:
224 |       - {Ref: CanvasLambdaLayer}
225 |     events:
226 |       - http:
227 |           path: /try/alexa-ranking/{domain}
228 |           method: GET
229 |           cors: true
230 | 
231 |   http-http-headers:
232 |     handler: functions/api/http-headers.handler
233 |     events:
234 |       - http:
235 |           path: /try/http-headers/{domain}
236 |           method: GET
237 |           cors: true
238 | 
239 |   http-css-stats:
240 |     timeout: 50
241 |     memorySize: 1024
242 |     handler: functions/api/css-stats.handler
243 |     events:
244 |       - http:
245 |           path: /try/css-stats/{domain}
246 |           method: GET
247 |           cors: true
248 | 
249 |   http-lighthouse:
250 |     memorySize: 1536
251 |     timeout: 60
252 |     layers:
253 |       - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:10
254 |     handler: functions/api/lighthouse.handler
255 |     environment: ${file(./config/fn.lighthouse.yml)}
256 |     events:
257 |       - http:
258 |           path: /try/lighthouse/{domain}
259 |           method: GET
260 |           cors: true
261 | 
262 |   http-latest-screenshot:
263 |     handler: functions/api/latest-screenshot.handler
264 |     environment: ${file(./config/fn.lighthouse.yml)}
265 |     iamRoleStatements:
266 |     - *snsAccess
267 |     - *dynamodbReadAccess
268 |     - *s3Access
269 |     events:
270 |       - http:
271 |           path: /try/latest-screenshot/{domain}
272 |           method: GET
273 |           cors: true
274 | 
275 |   http-optimize-image:
276 |     handler: functions/api/optimize-image.handler
277 |     environment: ${file(./config/fn.http-optimize-image.yml)}
278 |     events:
279 |       - http:
280 |           path: /try/optimize-image
281 |           method: GET
282 |           cors: true
283 | 


--------------------------------------------------------------------------------
/utils/index.js:
--------------------------------------------------------------------------------
  1 | import { spawn } from 'child_process';
  2 | import * as AWS from 'aws-sdk';
  3 | import * as url from 'url';
  4 | import * as crypto from 'crypto';
  5 | import * as fs from 'fs';
  6 | import * as util from 'util';
  7 | import * as path from 'path';
  8 | 
  9 | const fsUnlink = util.promisify(fs.unlink);
 10 | const dynamoDb = new AWS.DynamoDB.DocumentClient();
 11 | const sns = new AWS.SNS({});
 12 | 
 13 | export async function getFromDatabase(query) {
 14 |   global.console.log(`table: ${process.env.AWS_DYNAMODB_TABLE}`);
 15 |   global.console.log('query', query);
 16 |   const params = {
 17 |     TableName: process.env.AWS_DYNAMODB_TABLE,
 18 |     Key: query
 19 |   };
 20 | 
 21 |   const { Item } = await dynamoDb.get(params).promise();
 22 |   return Item;
 23 | }
 24 | 
 25 | export async function getFromDatabaseBetween(query) {
 26 |   if(!query.domain) {
 27 |     return exports.scanDatabase(query);
 28 |   }
 29 |   const params = {
 30 |     TableName: process.env.AWS_DYNAMODB_TABLE,
 31 |     KeyConditionExpression: '#primaryKey = :domain AND #sortKey BETWEEN :from AND :to',
 32 |     ExpressionAttributeNames: {
 33 |       '#sortKey': 'date',
 34 |       '#primaryKey': 'domain'
 35 |     },
 36 |     ExpressionAttributeValues: {
 37 |       ':domain': query.domain,
 38 |       ':from': query.from,
 39 |       ':to': query.to
 40 |     }
 41 |   };
 42 | 
 43 |   const result = await dynamoDb.query(params).promise();
 44 |   console.log(result);
 45 |   return result.Items;
 46 | }
 47 | 
 48 | export async function scanDatabase2({ date, exclusiveStartKey }) {
 49 |   const params = {
 50 |     TableName: process.env.AWS_DYNAMODB_TABLE,
 51 |     FilterExpression: '#sortKey = :date',
 52 |     ExclusiveStartKey: exclusiveStartKey,
 53 |     ExpressionAttributeNames: {
 54 |       '#sortKey': 'date',
 55 |     },
 56 |     ExpressionAttributeValues: { ':date': date }
 57 |   };
 58 | 
 59 |   const result = await dynamoDb.scan(params).promise();
 60 |   if(result.LastEvaluatedKey) {
 61 |     return result.Items.concat(await exports.scanDatabase2({
 62 |       date,
 63 |       exclusiveStartKey: result.LastEvaluatedKey
 64 |     }));
 65 |   }
 66 |   return result.Items;
 67 | }
 68 | 
 69 | export async function scanDatabase({ from, to, exclusiveStartKey = null }) {
 70 |   const params = {
 71 |     TableName: process.env.AWS_DYNAMODB_TABLE,
 72 |     FilterExpression: '#sortKey BETWEEN :from AND :to',
 73 |     ExclusiveStartKey: exclusiveStartKey,
 74 |     ExpressionAttributeNames: {
 75 |       '#sortKey': 'date',
 76 |     },
 77 |     ExpressionAttributeValues: { ':from': from, ':to': to }
 78 |   };
 79 | 
 80 |   const result = await dynamoDb.scan(params).promise();
 81 |   console.log('items', result.Items.length);
 82 |   if(result.LastEvaluatedKey) {
 83 |     return result.Items.concat(await exports.scanDatabase({
 84 |       from,
 85 |       to,
 86 |       exclusiveStartKey: result.LastEvaluatedKey
 87 |     }));
 88 |   }
 89 |   return result.Items;
 90 | }
 91 | 
 92 | export async function sendSNSMessage(params) {
 93 |   console.log('sending SNS message', params);
 94 |   return sns.publish(params).promise();
 95 | }
 96 | 
 97 | export function getSNSPayload(event){
 98 |   return JSON.parse(event.Records[0].Sns.Message);
 99 | }
100 | 
101 | export async function updateDocument(domain, date, document) {
102 |   const params = {
103 |     TableName: process.env.AWS_DYNAMODB_TABLE,
104 |     Key: {
105 |       domain,
106 |       date
107 |     },
108 |     ReturnValues: 'ALL_NEW',
109 |     UpdateExpression: 'set ',
110 |     ExpressionAttributeValues: {}
111 |   };
112 | 
113 |   for(const [key, value] of Object.entries(document)) {
114 |     params.UpdateExpression += `${key} = :${key},`;
115 |     params.ExpressionAttributeValues[`:${key}`] = value;
116 |   }
117 | 
118 |   params.UpdateExpression = params.UpdateExpression.slice(0, -1);
119 |   console.log('params', params);
120 |   await dynamoDb.update(params).promise();
121 | }
122 | 
123 | export function getDateForToday(formatFn) {
124 |   return formatFn(new Date(), 'YYYY-MM-DD');
125 | }
126 | 
127 | export function domainToURL(domain) {
128 |   const decodedDomain = decodeURIComponent(domain);
129 |   return decodedDomain.includes('https://') ? decodedDomain : `https://${decodedDomain}`;
130 | }
131 | 
132 | export function httpResponse(statusCode, body, stringify) {
133 |   const response = {
134 |     statusCode,
135 |     headers: {
136 |       'Access-Control-Allow-Origin': '*',
137 |       'Access-Control-Allow-Credentials': true,
138 |     }
139 |   };
140 | 
141 |   if(body) {
142 |     if(stringify) {
143 |       response.body = stringify(body);
144 |     } else {
145 |       response.body = JSON.stringify(body);
146 |     }
147 |   }
148 | 
149 |   return response;
150 | }
151 | 
152 | export async function getAlexaRanking(axios, JSDOM, domain) {
153 |   // TODO: this URL should be an environment variable
154 |   const { data: html } = await axios.get(`https://www.alexa.com/siteinfo/${domain}`);
155 |   const dom = new JSDOM(html);
156 |   const domElement = dom.window.document.querySelector(process.env.ALEXA_RANK_DOM_SELECTOR);
157 |   if(!domElement) {
158 |     throw new Error(`no element found for selector ${process.env.ALEXA_RANK_DOM_SELECTOR}`);
159 |   }
160 |   const rank = domElement.textContent.trim().replace('#', '');
161 |   return rank;
162 | }
163 | 
164 | export async function getHTTPHeaders(domain) {
165 |   const curlOutput = await runCommand(`curl -A "${process.env.USER_AGENT}" -sSL -D - ${domain} -o /dev/null`);
166 | 
167 |   if(!curlOutput) {
168 |     throw new Error('empty curl output');
169 |   }
170 | 
171 |   console.log('curlOutput', curlOutput);
172 |   const result = formatCurlOutput(curlOutput);
173 |   console.log(result);
174 | 
175 |   return result;
176 | }
177 | 
178 | export async function runCommand(cmd) {
179 |   const p = spawn('sh', ['-c', cmd]);
180 |   let cmdOutput = '';
181 |   p.stdout.setEncoding('utf8');
182 |   p.stdout.on('data', (data) => {
183 |     cmdOutput += data;
184 |   });
185 |   await new Promise(resolve => p.on('exit', resolve));
186 |   return cmdOutput;
187 | }
188 | 
189 | export async function getCurlData(domain) {
190 |   // TODO: this path should be an environment variable
191 |   const curlOutput = await runCommand(`curl -w "@/opt/nodejs/curl-format.json" -o /dev/null -s ${domainToURL(domain)}`);
192 | 
193 |   global.console.log('output', curlOutput);
194 |   if(curlOutput === '') {
195 |     throw new Error('empty output from cURL');
196 |   }
197 | 
198 |   return JSON.parse(curlOutput);
199 | }
200 | 
201 | export async function getCSSAnalysis(getCss, analyze, axios, domain) {
202 |   let css;
203 |   try {
204 |     const result = await new Promise(async (resolve, reject) => {
205 |       let resolved = false;
206 |       setTimeout(() => {
207 |         if(resolved) {
208 |           return;
209 |         }
210 |         reject(Error(`it didn't finished`));
211 |       }, Number(process.env.TIMEOUT));
212 | 
213 |       // for some reason this timeout is not working, we need to enforce it
214 |       const result = await getCss(domainToURL(domain), { timeout: Number(process.env.TIMEOUT) });
215 |       resolved = true;
216 |       resolve(result);
217 |     });
218 |     css = result.css;
219 |   } catch(err) {
220 |     console.log('there was an error', err);
221 |     console.log('trying to get the css with curl');
222 |     css = await getCSSUsingCurl(axios, domainToURL(domain));
223 |   } finally {
224 |     const result = await analyze(css);
225 |     var reducedResult = { };
226 | 
227 |     // sort for frequency and get top 5 for each stat
228 |     for(const [key, value] of Object.entries(result)) {
229 |       if(Array.isArray(value)) {
230 |         if(value.length !== 0) {
231 |           reducedResult[key] = value.sort((a,b) => {
232 |             if(a.count < b.count){
233 |               return 1;
234 |             }
235 |             return -1;
236 |           }).slice(0, 5);
237 |         }
238 |       } else {
239 |         reducedResult[key] = value;
240 |       }
241 |     }
242 |   
243 |     return reducedResult;
244 |   }
245 | }
246 | 
247 | // TODO: update curl-headers-to-json package to export a method and use that
248 | function formatCurlOutput(data) {
249 |   let parts = data.split('HTTP/1.1 200 OK\r\n');
250 |   if(parts.length === 1) {
251 |     parts = data.split('HTTP/2 200');
252 |   }
253 |   console.log('parts', parts);
254 |   const lines = parts[1].split(/\r?\n|\r/g);
255 |   const result = {};
256 |   for(let line of lines) {
257 |     if(line.trim() === '') {
258 |       continue;
259 |     }
260 |     const [key, value] = line.split(': ');
261 |     result[key] = value.replace(/\\r/g, '');
262 |   }
263 |   return result;
264 | }
265 | 
266 | export async function getHTMLAnalysis(getCss, JSDOM, domain) {
267 |   let html;
268 |   try {
269 |     const result = await new Promise(async (resolve, reject) => {
270 |       let resolved = false;
271 |       setTimeout(() => {
272 |         if(resolved) {
273 |           return;
274 |         }
275 |         reject(Error(`it didn't finished`));
276 |       }, Number(process.env.TIMEOUT));
277 | 
278 |       // for some reason this timeout is not working, we need to enforce it
279 |       const result = await getCss(domainToURL(domain), { timeout: Number(process.env.TIMEOUT) });
280 |       resolved = true;
281 |       resolve(result);
282 |     });
283 |     html = result.html;
284 |   } catch(err) {
285 |     console.log('there was an error', err);
286 |     console.log('trying to get the html using curl');
287 |     html = await getHTMLUsingCurl(domainToURL(domain));
288 |   } finally {
289 |     global.console.log('got html');
290 |     const dom = new JSDOM(html);
291 |     const result = {attributes: {}};
292 |     global.console.log('started traversing');
293 |     traverseHTMLNode(dom.window.document.body, result);
294 |     global.console.log('finished traversing');
295 |     return result;
296 |   }
297 | }
298 | 
299 | function traverseHTMLNode(node, result){
300 |   if(node.children && node.children.length !== 0){
301 |     for(const child of node.children) {
302 |       traverseHTMLNode(child, result);
303 |     }
304 |   }
305 | 
306 |   const tagName = node.tagName.toLowerCase();
307 | 
308 |   if(!result[tagName]) {
309 |     result[tagName] = {
310 |       count: 1,
311 |       attributes: {}
312 |     };
313 | 
314 |     // if(node.attributes && node.attributes.length !== 0) {
315 |     //   for(const { name } of node.attributes) {
316 |     //     result[tagName].attributes[name] = 1;
317 |     //     if(!result.attributes[name]) {
318 |     //       result.attributes[name] = 1;
319 |     //     } else {
320 |     //       result.attributes[name]++;
321 |     //     }
322 |     //   }
323 |     // }
324 |   } else {
325 |     result[tagName].count++;
326 |   }
327 | 
328 |   if(node.attributes && node.attributes.length !== 0) {
329 |     for(const { name } of node.attributes) {
330 |       if(!result[tagName].attributes[name]) {
331 |         result[tagName].attributes[name] = 1;
332 |       } else {
333 |         result[tagName].attributes[name]++;
334 |       }
335 |       if(!result.attributes[name]) {
336 |         result.attributes[name] = 1;
337 |       } else {
338 |         result.attributes[name]++;
339 |       }
340 |     }
341 |   }
342 | }
343 | 
344 | function allCharsAreEqual(color) {
345 |   return color[1] === color[2] && color[2] === color[3] && color[3] === color[4] && color[4] === color[5];
346 | }
347 | function relativePathToAbsolute(input, domain) {
348 |   if(input[0] === '/') {
349 |     if(input.substr(0, 2) === '//') {
350 |       return `https:${input}`;
351 |     } else {
352 |       return `https://${url.parse(domain).host}${input}`;
353 |     }
354 |   } else if(input.substr(0, 4) !== 'http') {
355 |     return `https://${url.parse(domain).host}/${input}`;
356 |   } else {
357 |     return input;
358 |   }
359 | }
360 | 
361 | async function getCSSUsingCurl(axios, domain){
362 |   const links = await runCommand(`curl '${domain}' -L ${process.env.CURL_PARAMS} 2> /dev/null | grep stylesheet`);
363 |   const regex = /href=(["'])(.*?)\1/g;
364 |   let match;
365 |   const hrefs = [];
366 |   while (match = regex.exec(links)) {
367 |     hrefs.push(match[2]);
368 |   }
369 | 
370 |   const results = await Promise.all(hrefs.map(href => axios.get(relativePathToAbsolute(href, domain))));
371 |   return results.map(x => x.data).join('');
372 | }
373 | 
374 | function getHTMLUsingCurl(domain){
375 |   return runCommand(`curl '${domain}' -L ${process.env.CURL_PARAMS} 2> /dev/null`);
376 | }
377 | 
378 | export async function getColorBar(getCss, colorSort, domain, axios, allSettled, width) {
379 |   let css;
380 |   try {
381 | 
382 |     // TODO: remove these headers as are usually uneffective against CDNs and bloat the function...
383 |     const { css: cssResponse, links} = await getCss(domainToURL(domain), {
384 |       headers: {
385 |         authority: 'codepen.io',
386 |         accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
387 |         'accept-language': 'en-US,en;q=0.9,es-419;q=0.8,es;q=0.7,gl;q=0.6',
388 |         'cache-control': 'no-cache',
389 |         pragma: 'no-cache',
390 |         dnt: 1,
391 |         'sec-ch-ua': 'Google Chrome 77',
392 |         'sec-fetch-dest': 'document',
393 |         'sec-fetch-mode': 'navigate',
394 |         'sec-fetch-site': 'none',
395 |         'sec-fetch-user': '?1',
396 |         'sec-origin-policy': '0',
397 |         'upgrade-insecure-requests': '1',
398 |         'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
399 |         'accept-encoding': 'gzip, deflate'
400 |       },
401 |       timeout: process.env.TIMEOUT
402 |     });
403 |     css = cssResponse;
404 |     if(css === '' && links.length !== 0) {
405 |       const responses = await allSettled(links.map(link => axios.get(link.url)));
406 |       const cssContent = responses.filter(x=> x.state === 'fulfilled').map(x => x.result.data).join('');
407 |       css = cssContent;
408 |     }
409 |     console.log('css', css);
410 |   } catch(err) {
411 |     console.log('there was an error', err);
412 |     console.log('trying to get the css with curl');
413 |     css = await getCSSUsingCurl(axios, domainToURL(domain));
414 |   } finally {
415 |     const matches = css.match(/(#([\da-f]{3}){1,2}|(rgb|hsl)a\((\d{1,3}%?,\s?){3}(1|0?\.\d+)\)|(rgb|hsl)\(\d{1,3}%?(,\s?\d{1,3}%?){2}\))/ig);
416 |     if(!matches) {
417 |       console.log('there were no matches, which is super weird friend');
418 | 
419 |       // TODO: try with curl, there's no way this is can be empty
420 |       return [];
421 |     }
422 | 
423 |     global.console.log('matches', matches.length);
424 |     const result = [...countColorsFrequency(matches)];
425 | 
426 |     global.console.log('colors frequency', result);
427 | 
428 |     if(result && result.length === 0) {
429 |       return [];
430 |     }
431 | 
432 |     const sorted = colorSort(result.map(x => x.color));
433 |     return sorted;
434 |   }
435 | }
436 | 
437 | function countColorsFrequency(colors) {
438 |   const map = {};
439 |   colors.forEach(color => {
440 |     let normalized;
441 | 
442 |     if(color.startsWith('rgb') || color.startsWith('hsl')) {
443 |       normalized = color.replace(/ /g, '');
444 |     } else if(color.length === 4) {
445 |       normalized = threeDigitHexaToSixDigit(color).toUpperCase();
446 |     } else {
447 |       normalized = color.toUpperCase();
448 |     }
449 | 
450 |     if(map[normalized]) {
451 |       map[normalized]++;
452 |     } else {
453 |       map[normalized] = 1;
454 |     }
455 |   });
456 | 
457 |   return Object.entries(map).map(([key, value]) => ({ color: key, count: value }));
458 | }
459 | 
460 | function threeDigitHexaToSixDigit(color) {
461 |   return `#${color[1].repeat(2)}${color[2].repeat(2)}${color[3].repeat(2)}`;
462 | }
463 | 
464 | export async function getTechnologiesUsed(chrome, puppeteer, axios, allSettled, decompress, domain) {
465 |   console.log('technologies used', domain);
466 |   const browser = await puppeteer.launch({
467 |     args: chrome.args,
468 |     defaultViewport: chrome.defaultViewport,
469 |     executablePath: await chrome.executablePath,
470 |     headless: chrome.headless,
471 |   });
472 |   const page = await browser.newPage();
473 |   await page.setViewport({ width: 1920, height: 1080 });
474 |   await page.setUserAgent(process.env.USER_AGENT);
475 |   await page.goto(domainToURL(domain), {
476 |     waitUntil: process.env.PUPPETEER_WAIT_UNTIL_EVENT,
477 |   });
478 | 
479 |   global.console.log('loaded domain', domain);
480 |   if(process.env.PUPPETEER_WAIT_FOR_MILLISECONDS) {
481 |     await page.waitFor(Number(process.env.PUPPETEER_WAIT_FOR_MILLISECONDS));
482 |   }
483 | 
484 |   const scripts = await page.evaluate(domain => {
485 |     const result = [];
486 |     const elements = document.getElementsByTagName('script');
487 |     for(const script of elements) {
488 |       if(!script.attributes.src) {
489 |         continue;
490 |       }
491 |       result.push(script.src);
492 |     }
493 | 
494 |     const links = document.getElementsByTagName('links');
495 |     for(const link of links) {
496 |       if(link.rel && link.href && link.rel === 'preload' && link.href.match(/\.js/)) {
497 |         result.push(link.href);
498 |       }
499 |     }
500 |     return result;
501 |   }, domain);
502 | 
503 |   global.console.log('scripts', scripts);
504 | 
505 |   const scriptResults = await allSettled(scripts.map(script => {
506 | 
507 |     // TODO: axios won't be able to get some resources on CDNs
508 |     return axios.get(script, {
509 |       responseType: 'arraybuffer',
510 |       timeout: 5000
511 |     });
512 |   }));
513 | 
514 |   console.log('finished downloading scripts');
515 |   const scriptContents = scriptResults.filter(x=> x.state === 'fulfilled').map(({ result: response })=> {
516 |     if(response.headers['content-encoding'] === 'br') {
517 |       console.log('decompressing brotli file');
518 |       return Buffer.from(decompress(response.data)).toString();
519 |     } else if(response.headers['content-encoding'] === 'gzip') {
520 |       console.log('gzipped script');
521 |       return response.data;
522 |     }
523 |     return Buffer.from(response.data).toString();
524 |   });
525 |   // TODO: investigate why some scripts would fail
526 |   scriptResults.filter(x=> x.state === 'rejected').forEach(e => console.log('script failed', e.error));
527 |   const mergedJavaScriptFiles = scriptContents.join('');
528 |   const technologiesConfigured = process.env.TECHNOLOGIES.split(',');
529 |   const technologies = [];
530 | 
531 |   for(const tech of technologiesConfigured) {
532 |     const identifiers = process.env[`${tech}_IDENTIFIERS`].split(',');
533 |     for(const identifier of identifiers) {
534 |       const match = mergedJavaScriptFiles.match(new RegExp(identifier));
535 |       if(match) {
536 |         technologies.push(mapTechnologyNames(tech));
537 |         break;
538 |       }
539 |     }
540 |   }
541 |   console.log(`the domain ${(domain)} is using:`, technologies);
542 | 
543 |   await page.close();
544 |   await browser.close();
545 |   return technologies;
546 | }
547 | 
548 | function mapTechnologyNames(str) {
549 |   switch (str) {
550 |     case 'REACTJS':
551 |       return 'React.js';
552 |     case 'VUEJS':
553 |       return 'Vue.js';
554 |     case 'VUEX':
555 |       return 'Vuex';
556 |     case 'ANGULAR':
557 |       return 'Angular';
558 |     case 'SIZZLE':
559 |       return 'Sizzle.js';
560 |     case 'JQUERY':
561 |       return 'jQuery.js';
562 |     case 'LODASH':
563 |       return 'lodash';
564 |     case 'PREACT':
565 |       return 'Preact';
566 |     case 'WEBPACK':
567 |       return 'Webpack';
568 |     case 'REQUIREJS':
569 |       return 'Require.js';
570 |     case 'REDUX':
571 |       return 'Redux';
572 |     case 'MOMENTJS':
573 |       return 'Moment.js';
574 |     case 'REACTROUTER':
575 |       return 'react-router';
576 |     case 'VUEROUTER':
577 |       return 'vue-router';
578 |     case 'NEXTJS':
579 |       return 'Next.js';
580 |     case 'NUXTJS':
581 |       return 'Nuxt.js';
582 |     default:
583 |       return str.toLowerCase();
584 |   }
585 | }
586 | 
587 | export async function getLighthouse2(chromeLauncher, chrome, lighthouse, domain) {
588 |   const chromeInstance = await chromeLauncher.launch({
589 |     chromeFlags: [...chrome.args, '--disable-web-security', '--disable-gpu', '--headless', '--single-process'],
590 |     chromePath: await chrome.executablePath,
591 |     logLevel: 'verbose'
592 |   });
593 | 
594 |   console.log('chromeInstance', chromeInstance);
595 | 
596 |   const lighthouseConfig = {
597 |     extends: 'lighthouse:default',
598 |     settings: {
599 |       onlyCategories: ['performance', 'accessibility', 'pwa', 'accessibility', 'seo', 'best-practices']
600 |     }
601 |   };
602 | 
603 |   const { lhr } = await lighthouse(domainToURL(domain), { port: chromeInstance.port }, lighthouseConfig);
604 | 
605 |   return lhr;
606 | }
607 | 
608 | export async function uploadToS3(bucket, filename, content) {
609 | 
610 |   const s3bucket = new AWS.S3({params: {Bucket: bucket}});
611 |   const params = {
612 |     Key: filename,
613 |     Body: content
614 |   };
615 | 
616 |   switch(path.extname(filename)) {
617 |     case '.png':
618 |       params.ContentType = 'image/png';
619 |       break;
620 |     case '.jpg':
621 |     case '.jpeg':
622 |       params.ContentType = 'image/jpg';
623 |       break;
624 | 
625 |     case '.webp':
626 |       params.ContentType = 'image/webp';
627 |       break;
628 | 
629 |     default:
630 |   }
631 |   return s3bucket.upload(params).promise();
632 | }
633 | 
634 | export async function uploadGist(axios, description, content) {
635 |   const { data: result } = await axios.post(process.env.GH_GISTS_URL, {
636 |     description,
637 |     public: false,
638 |     files: {
639 |       'report.json': { content }}
640 |   }, {
641 |     headers: {
642 |       Authorization: `Token ${process.env.GH_TOKEN}`,
643 |       'Content-Type': 'application/json'
644 |     }
645 |   });
646 |   return {
647 |     url: result.url,
648 |     id: result.id
649 |   };
650 | }
651 | 
652 | export function getMD5(str) {
653 |   return crypto.createHash('md5').update(str).digest('hex');
654 | }
655 | 
656 | export async function getColorDetection(chromeLauncher, chrome, puppeteer, axios, domain) {
657 |   const lsResult = await runCommand('df -k /tmp ; ls -al /tmp');
658 |   console.log(lsResult);
659 | 
660 |   let chromeInstance;
661 |   try {
662 |     chromeInstance = await chromeLauncher.launch({
663 |       chromeFlags: [...chrome.args, '--disable-web-security', '--disable-gpu', '--headless', '--single-process'],
664 |       chromePath: await chrome.executablePath,
665 |       logLevel: 'verbose'
666 |     });
667 | 
668 |     console.log('launched chrome instance', chromeInstance);
669 | 
670 |     const { data: chromeInstanceJSON } = await axios.get(`http://localhost:${chromeInstance.port}/json/version`);
671 |     const browser = await puppeteer.connect({browserWSEndpoint: chromeInstanceJSON.webSocketDebuggerUrl});
672 | 
673 |     console.log('connected puppeteer to chrome. launching new page...');
674 |     const page = await browser.newPage();
675 | 
676 |     // TODO: debate if viewport should be this or not
677 |     await page.setViewport({ width: 1920, height: 1080 });
678 |     await page.setUserAgent(process.env.USER_AGENT);
679 | 
680 |     console.log(`browsing to ${domain}`);
681 |     await page.goto(domainToURL(domain), {
682 |       waitUntil: process.env.PUPPETEER_WAIT_UNTIL_EVENT,
683 |       timeout: 0
684 |     });
685 | 
686 |     if(process.env.PUPPETEER_WAIT_FOR_MILLISECONDS) {
687 |       await page.waitFor(Number(process.env.PUPPETEER_WAIT_FOR_MILLISECONDS));
688 |     }
689 | 
690 |     const screenshotPath = `/tmp/${getMD5(domain)}.png`;
691 |     await page.screenshot({path: screenshotPath, fullPage: true });
692 | 
693 |     console.log('screenshot taken');
694 |     const convertOutput = await runCommand(`convert /tmp/${getMD5(domain)}.png -scale 50x50! -depth 8 +dither -colors 8 -format "%c" histogram:info: | sed -n 's/^[ ]*\\(.*\\):.*[#]\\([0-9a-fA-F]*\\) .*$/\\1,#\\2/p' | sort -r -n -k 1 -t ","`);
695 | 
696 |     console.log('convert output', convertOutput);
697 |     const lines = convertOutput.split(/\r?\n|\r/g);
698 |     const result = {
699 |       screenshotPath,
700 |       colors: []
701 |     };
702 | 
703 |     for(const line of lines) {
704 |       if(line.trim() === '') {
705 |         continue;
706 |       }
707 |       const [count, color] = line.split(',');
708 |       result.colors.push({ count, color });
709 |     }
710 | 
711 |     console.log('closing chrome and cleaning up...');
712 |     await page.close();
713 |     await browser.close();
714 | 
715 |     // why is this needed? browser.close supposedly finished...why?
716 |     process.nextTick(() => chromeInstance.kill());
717 |     return result;
718 |   } catch(err) {
719 |     console.log('there was an error while trying to run puppeteer');
720 |     console.log(err);
721 |     process.nextTick(() => {
722 |       if(chromeInstance) {
723 |         chromeInstance.kill();
724 |       }
725 |     });
726 |     throw err;
727 |   }
728 | }
729 | 
730 | export async function getLatestScreenshot(domain) {
731 |   return `${process.env.SCREENSHOT_BUCKET_URL}/${getMD5(domain)}.${process.env.SCREENSHOT_EXTENSION}`;
732 | }
733 | 
734 | export function deleteFile(filePath) {
735 |   return fsUnlink(filePath);
736 | }
737 | 
738 | export async function optimizeImage(imagemin, imageminWebp, filePath) {
739 |   const { 0: result } = await imagemin([filePath], {
740 |     plugins: [
741 |       imageminWebp({
742 |         quality: Number(process.env.IMAGE_QUALITY),
743 |         crop: {
744 |           x: 0,
745 |           y: 0,
746 |           width: Number(process.env.IMAGE_WIDTH),
747 |           height: Number(process.env.IMAGE_HEIGHT)
748 |         }
749 |       })
750 |     ]
751 |   });
752 |   return result;
753 | }
754 | 


--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
 1 | const webpack = require('webpack');
 2 | const slsw = require('serverless-webpack');
 3 | const nodeExternals = require('webpack-node-externals');
 4 | 
 5 | module.exports = {
 6 |   mode: 'production',
 7 |   // mode: 'development',
 8 |   entry: slsw.lib.entries,
 9 |   target: 'node',
10 |   externals: [nodeExternals()],
11 |   plugins: [
12 |     // new webpack.IgnorePlugin(/chrome-aws-lambda/),
13 |     // new webpack.IgnorePlugin(/utils/)
14 |   ],
15 |   // devtool: '#inline-source-map',
16 |   // compress: {
17 |   //   drop_debugger: false
18 |   // }
19 | };


--------------------------------------------------------------------------------