├── LICENSE.md
├── README.md
├── archivist-cli
    ├── .gitignore
    ├── cli.js
    ├── lib.js
    ├── package-lock.json
    └── package.json
├── archivist-pinboard
    ├── .gitignore
    ├── fetch
    │   ├── assets
    │   │   └── .gitkeep
    │   ├── fetcher.js
    │   └── index.js
    ├── index.js
    ├── package-lock.json
    ├── package.json
    ├── query
    │   └── index.js
    └── scripts
    │   └── compile-freeze-dry.js
├── archivist-pinterest-crawl
    ├── .gitignore
    ├── fetch
    │   ├── crawler.js
    │   ├── fetcher.js
    │   └── index.js
    ├── index.js
    ├── package-lock.json
    ├── package.json
    └── query
    │   └── index.js
├── archivist-ui
    ├── .gitignore
    ├── README.md
    ├── assets
    │   └── Archivist.icns
    ├── package-lock.json
    ├── package.json
    ├── scripts
    │   └── install.sh
    └── src
    │   ├── main
    │       └── index.js
    │   └── renderer
    │       ├── index.html
    │       └── index.js
└── assets
    └── screenshot.png


/LICENSE.md:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Szymon Kaliski
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Archivist
 2 | 
 3 | Tool for archiving and exploring.
 4 | 
 5 | Built out of a need to get out of walled gardens of Pinterest and (much less walled) Pinboard.
 6 | 
 7 | **Alpha quality at best.**
 8 | 
 9 | <p align="center"><img src="assets/screenshot.png" /></p>
10 | 
11 | ## Modules
12 | 
13 | Archivist is built out of three interconnected parts (each package has it's own readme file):
14 | 
15 | 1. [`archivist-cli`](./archivist-cli) - command line tool for configuration, fetching and querying the data
16 | 2. [`archivist-ui`](./archivist-ui) - Electron UI built on top of `archivist-cli`
17 | 3. `archivist-*` - various crawlers, "official" ones:
18 |    - [`archivist-pinboard`](./archivist-pinboard) - API-based Pinboard archiving: screenshot and [freeze-dry](https://github.com/WebMemex/freeze-dry) of the original website
19 |    - [`archivist-pinterest-crawl`](./archivist-pinterest-crawl) - slowly crawl through Pinterest and archive pin image
20 | 
21 | ## Installation
22 | 
23 | 1. `npm install -g archivist-cli`
24 | 2. to archive pinboard: `npm install -g archivist-pinboard`
25 | 3. to archive pinterest: `npm install -g archivist-pinterest-crawl`
26 | 
27 | `archivist-ui` is not on npm (it should probably be a downloadable `dmg`, but I didn't get around to it), so to generate the `.app` and put it in `/Applications/` yourself:
28 | 
29 | 1. clone this repo
30 | 2. `cd archivist-ui && ./scripts/install.sh`
31 | 
32 | ## Configure
33 | 
34 | ```bash
35 | $ archivist config
36 | ```
37 | 
38 | Config is a JSON object of shape:
39 | 
40 | ```json
41 | {
42 |   "crawler-1": CRAWLER_1_OPTIONS,
43 |   "crawler-2": CRAWLER_2_OPTIONS,
44 |   ...
45 | }
46 | ```
47 | 
48 | Example config (assuming Pinboard and Pinterest backup):
49 | 
50 | ```json
51 | {
52 |   "archivist-pinterest-crawl": {
53 |     "loginMethod": "cookies",
54 |     "profile": "szymon_k"
55 |   },
56 |   "archivist-pinboard": {
57 |     "apiKey": "API_KEY_FOR_PINBOARD"
58 |   }
59 | }
60 | ```
61 | 
62 | `archivist-pinterest-crawl` supports two login methods: `"cookies"` (which uses cookies from local Google Chrome installation) or `"password"` which requires plaintext username and password:
63 | 
64 | ```json
65 | "archivist-pinterest-crawl": {
66 |   "loginMethod": "password",
67 |   "username": "PINTEREST_USERNAME",
68 |   "password": "PINTEREST_PASSWORD",
69 |   "profile": "szymon_k"
70 | },
71 | ```
72 | 
73 | `archivist-pinboard` requires `API Token` from https://pinboard.in/settings/password to run properly.
74 | 
75 | ## Usage
76 | 
77 | - backup data: `archivist fetch` (might take a long time depending on the size of the archive)
78 | - list everything: `archivist query`
79 |   - find everything about keyboards: `archivist query keyboard`
80 |   - `query` by default returns `ndjson`, normal JSON can be outputed using `--json`
81 | 
82 | ## References
83 | 
84 | - [kollektor](https://github.com/vorg/kollektor) - no-ui self-hosted Pinterest clone
85 | - gwern on [archiving URLs](https://www.gwern.net/Archiving-URLs)
86 | - [freeze-dry implementation notes](https://github.com/WebMemex/freeze-dry/blob/master/src/Readme.md)
87 | 
88 | 


--------------------------------------------------------------------------------
/archivist-cli/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | 


--------------------------------------------------------------------------------
/archivist-cli/cli.js:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env node
 2 | 
 3 | const fs = require("fs");
 4 | const yargs = require("yargs");
 5 | const { spawn } = require("child_process");
 6 | 
 7 | const { fetch, search } = require("./lib");
 8 | 
 9 | const args = yargs
10 |   .command("config", "open configuration file")
11 |   .command("fetch", "fetch all configured crawlers")
12 |   .command("search", "search all crawlers", (yargs) => {
13 |     yargs.option("json", { description: "output as JSON" });
14 |   })
15 |   .demandCommand(1, "you need to provide a command")
16 |   .help().argv;
17 | 
18 | const [TYPE] = args._;
19 | 
20 | if (TYPE === "config") {
21 |   const editor = process.env.EDITOR || "vim";
22 | 
23 |   if (!fs.existsSync(CONFIG_FILE)) {
24 |     fs.writeFileSync(
25 |       JSON.stringify(DEFAULT_CONFIG, null, 2),
26 |       CONFIG_FILE,
27 |       "utf-8"
28 |     );
29 |   }
30 | 
31 |   spawn(editor, [CONFIG_FILE], { stdio: "inherit" });
32 | } else if (TYPE === "fetch") {
33 |   fetch();
34 | } else if (TYPE === "search") {
35 |   search(args._[1]).then((result) => {
36 |     if (args.json) {
37 |       console.log(JSON.stringify(result.value(), null, 2));
38 |     } else {
39 |       result.forEach((d) => console.log(JSON.stringify(d))).value();
40 |     }
41 |   });
42 | }
43 | 


--------------------------------------------------------------------------------
/archivist-cli/lib.js:
--------------------------------------------------------------------------------
 1 | const async = require("async");
 2 | const envPaths = require("env-paths");
 3 | const mkdirp = require("mkdirp");
 4 | const os = require("os");
 5 | const path = require("path");
 6 | const { chain } = require("lodash");
 7 | 
 8 | const CONFIG_PATH = envPaths("archivist").config;
 9 | const CONFIG_FILE = path.join(CONFIG_PATH, "config.json");
10 | const DEFAULT_CONFIG = {};
11 | 
12 | mkdirp(CONFIG_PATH);
13 | 
14 | const loadConfig = () => {
15 |   let config;
16 | 
17 |   try {
18 |     config = require(CONFIG_FILE);
19 |   } catch (e) {
20 |     console.log(e);
21 |     process.exit(1);
22 |   }
23 | 
24 |   return config;
25 | };
26 | 
27 | const loadCrawler = (name) => {
28 |   return new Promise((resolve, reject) => {
29 |     let crawler;
30 | 
31 |     try {
32 |       crawler = require(name);
33 |     } catch (e) {
34 |       return reject(e);
35 |     }
36 | 
37 |     resolve(crawler);
38 |   });
39 | };
40 | 
41 | const fetch = () => {
42 |   return new Promise((resolve, reject) => {
43 |     async.eachLimit(
44 |       Object.entries(loadConfig()),
45 |       os.cpus().length,
46 |       ([name, config], callback) => {
47 |         loadCrawler(name).then((crawler) =>
48 |           crawler(config)
49 |             .fetch(config)
50 |             .then(callback)
51 |             .catch((e) => callback(`[${name}] fetching error ${e}`))
52 |         );
53 |       },
54 |       (err) => {
55 |         if (err) {
56 |           return reject(err);
57 |         }
58 | 
59 |         resolve();
60 |       }
61 |     );
62 |   });
63 | };
64 | 
65 | const search = (query) => {
66 |   return new Promise((resolve, reject) => {
67 |     async.mapLimit(
68 |       Object.entries(loadConfig()),
69 |       os.cpus().length,
70 |       ([name, config], callback) => {
71 |         loadCrawler(name).then((crawler) => {
72 |           crawler(config)
73 |             .query(query)
74 |             .then((result) => callback(null, result))
75 |             .catch((e) => callback(`[${name}] search error ${e}`));
76 |         });
77 |       },
78 |       (err, result) => {
79 |         if (err) {
80 |           return reject(err);
81 |         }
82 | 
83 |         const sortedResult = chain(result)
84 |           .flatten()
85 |           .sortBy((d) => new Date(d.time));
86 | 
87 |         resolve(sortedResult);
88 |       }
89 |     );
90 |   });
91 | };
92 | 
93 | module.exports = { loadConfig, loadCrawler, fetch, search };
94 | 


--------------------------------------------------------------------------------
/archivist-cli/package-lock.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "name": "archivist-cli",
  3 |   "version": "1.2.0",
  4 |   "lockfileVersion": 1,
  5 |   "requires": true,
  6 |   "dependencies": {
  7 |     "@types/color-name": {
  8 |       "version": "1.1.1",
  9 |       "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
 10 |       "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
 11 |     },
 12 |     "ansi-regex": {
 13 |       "version": "5.0.0",
 14 |       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
 15 |       "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
 16 |     },
 17 |     "ansi-styles": {
 18 |       "version": "4.2.1",
 19 |       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
 20 |       "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
 21 |       "requires": {
 22 |         "@types/color-name": "^1.1.1",
 23 |         "color-convert": "^2.0.1"
 24 |       }
 25 |     },
 26 |     "async": {
 27 |       "version": "3.2.0",
 28 |       "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz",
 29 |       "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw=="
 30 |     },
 31 |     "camelcase": {
 32 |       "version": "5.3.1",
 33 |       "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
 34 |       "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
 35 |     },
 36 |     "cliui": {
 37 |       "version": "6.0.0",
 38 |       "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz",
 39 |       "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==",
 40 |       "requires": {
 41 |         "string-width": "^4.2.0",
 42 |         "strip-ansi": "^6.0.0",
 43 |         "wrap-ansi": "^6.2.0"
 44 |       }
 45 |     },
 46 |     "color-convert": {
 47 |       "version": "2.0.1",
 48 |       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
 49 |       "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
 50 |       "requires": {
 51 |         "color-name": "~1.1.4"
 52 |       }
 53 |     },
 54 |     "color-name": {
 55 |       "version": "1.1.4",
 56 |       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
 57 |       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
 58 |     },
 59 |     "decamelize": {
 60 |       "version": "1.2.0",
 61 |       "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
 62 |       "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
 63 |     },
 64 |     "emoji-regex": {
 65 |       "version": "8.0.0",
 66 |       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
 67 |       "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
 68 |     },
 69 |     "env-paths": {
 70 |       "version": "2.2.0",
 71 |       "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz",
 72 |       "integrity": "sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA=="
 73 |     },
 74 |     "find-up": {
 75 |       "version": "4.1.0",
 76 |       "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
 77 |       "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
 78 |       "requires": {
 79 |         "locate-path": "^5.0.0",
 80 |         "path-exists": "^4.0.0"
 81 |       }
 82 |     },
 83 |     "get-caller-file": {
 84 |       "version": "2.0.5",
 85 |       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
 86 |       "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
 87 |     },
 88 |     "is-fullwidth-code-point": {
 89 |       "version": "3.0.0",
 90 |       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
 91 |       "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
 92 |     },
 93 |     "locate-path": {
 94 |       "version": "5.0.0",
 95 |       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
 96 |       "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
 97 |       "requires": {
 98 |         "p-locate": "^4.1.0"
 99 |       }
100 |     },
101 |     "lodash": {
102 |       "version": "4.17.15",
103 |       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
104 |       "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
105 |     },
106 |     "mkdirp": {
107 |       "version": "1.0.4",
108 |       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
109 |       "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
110 |     },
111 |     "p-limit": {
112 |       "version": "2.3.0",
113 |       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
114 |       "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
115 |       "requires": {
116 |         "p-try": "^2.0.0"
117 |       }
118 |     },
119 |     "p-locate": {
120 |       "version": "4.1.0",
121 |       "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
122 |       "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
123 |       "requires": {
124 |         "p-limit": "^2.2.0"
125 |       }
126 |     },
127 |     "p-try": {
128 |       "version": "2.2.0",
129 |       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
130 |       "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
131 |     },
132 |     "path-exists": {
133 |       "version": "4.0.0",
134 |       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
135 |       "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
136 |     },
137 |     "require-directory": {
138 |       "version": "2.1.1",
139 |       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
140 |       "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
141 |     },
142 |     "require-main-filename": {
143 |       "version": "2.0.0",
144 |       "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
145 |       "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg=="
146 |     },
147 |     "set-blocking": {
148 |       "version": "2.0.0",
149 |       "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
150 |       "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
151 |     },
152 |     "string-width": {
153 |       "version": "4.2.0",
154 |       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
155 |       "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
156 |       "requires": {
157 |         "emoji-regex": "^8.0.0",
158 |         "is-fullwidth-code-point": "^3.0.0",
159 |         "strip-ansi": "^6.0.0"
160 |       }
161 |     },
162 |     "strip-ansi": {
163 |       "version": "6.0.0",
164 |       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
165 |       "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
166 |       "requires": {
167 |         "ansi-regex": "^5.0.0"
168 |       }
169 |     },
170 |     "which-module": {
171 |       "version": "2.0.0",
172 |       "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
173 |       "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
174 |     },
175 |     "wrap-ansi": {
176 |       "version": "6.2.0",
177 |       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
178 |       "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
179 |       "requires": {
180 |         "ansi-styles": "^4.0.0",
181 |         "string-width": "^4.1.0",
182 |         "strip-ansi": "^6.0.0"
183 |       }
184 |     },
185 |     "y18n": {
186 |       "version": "4.0.0",
187 |       "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
188 |       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
189 |     },
190 |     "yargs": {
191 |       "version": "15.3.1",
192 |       "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz",
193 |       "integrity": "sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA==",
194 |       "requires": {
195 |         "cliui": "^6.0.0",
196 |         "decamelize": "^1.2.0",
197 |         "find-up": "^4.1.0",
198 |         "get-caller-file": "^2.0.1",
199 |         "require-directory": "^2.1.1",
200 |         "require-main-filename": "^2.0.0",
201 |         "set-blocking": "^2.0.0",
202 |         "string-width": "^4.2.0",
203 |         "which-module": "^2.0.0",
204 |         "y18n": "^4.0.0",
205 |         "yargs-parser": "^18.1.1"
206 |       }
207 |     },
208 |     "yargs-parser": {
209 |       "version": "18.1.3",
210 |       "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
211 |       "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
212 |       "requires": {
213 |         "camelcase": "^5.0.0",
214 |         "decamelize": "^1.2.0"
215 |       }
216 |     }
217 |   }
218 | }
219 | 


--------------------------------------------------------------------------------
/archivist-cli/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "archivist-cli",
 3 |   "version": "1.2.0",
 4 |   "description": "",
 5 |   "keywords": [],
 6 |   "author": "Szymon Kaliski <hi@szymonkaliski.com> (http://szymonkaliski.com)",
 7 |   "license": "MIT",
 8 |   "bin": {
 9 |     "archivist": "./cli.js"
10 |   },
11 |   "dependencies": {
12 |     "async": "^3.2.0",
13 |     "env-paths": "^2.2.0",
14 |     "lodash": "^4.17.15",
15 |     "mkdirp": "^1.0.4",
16 |     "yargs": "^15.3.1"
17 |   }
18 | }
19 | 


--------------------------------------------------------------------------------
/archivist-pinboard/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | fetch/assets/freeze-dry-browserified.js
3 | .DS_Store
4 | .env
5 | 


--------------------------------------------------------------------------------
/archivist-pinboard/fetch/assets/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/szymonkaliski/archivist/2061bc231b03437313f9f2eb41fdaec66aeebbe9/archivist-pinboard/fetch/assets/.gitkeep


--------------------------------------------------------------------------------
/archivist-pinboard/fetch/fetcher.js:
--------------------------------------------------------------------------------
  1 | const async = require("async");
  2 | const envPaths = require("env-paths");
  3 | const fs = require("fs");
  4 | const isReachable = require("is-reachable");
  5 | const md5 = require("md5");
  6 | const mkdirp = require("mkdirp");
  7 | const path = require("path");
  8 | const puppeteer = require("puppeteer");
  9 | const wayback = require("wayback-machine");
 10 | 
 11 | const DATA_PATH = envPaths("archivist-pinboard").data;
 12 | const ASSETS_PATH = path.join(DATA_PATH, "assets");
 13 | const FROZEN_PATH = path.join(DATA_PATH, "frozen");
 14 | 
 15 | mkdirp(ASSETS_PATH);
 16 | mkdirp(FROZEN_PATH);
 17 | 
 18 | const FREEZE_DRY_PATH = path.join(
 19 |   __dirname,
 20 |   "./assets/freeze-dry-browserified.js"
 21 | );
 22 | 
 23 | const FREEZE_DRY_SRC = fs.readFileSync(FREEZE_DRY_PATH, "utf-8");
 24 | 
 25 | const savePageInternal = async (browser, link, url) => {
 26 |   const screenshotPath = path.join(ASSETS_PATH, `${md5(url)}.png`);
 27 |   const frozenPath = path.join(FROZEN_PATH, `${md5(url)}.html`);
 28 | 
 29 |   let didScreenshot, didFreeze, didOpen;
 30 | 
 31 |   // don't re-download stuff
 32 |   if (fs.existsSync(screenshotPath) && fs.existsSync(frozenPath)) {
 33 |     console.log("[archivist-pinboard]", "already downloaded:", link);
 34 |     return {
 35 |       screenshot: path.basename(screenshotPath),
 36 |       frozen: path.basename(frozenPath)
 37 |     };
 38 |   }
 39 | 
 40 |   console.log("[archivist-pinboard]", "saving:", link);
 41 | 
 42 |   const page = await browser.newPage();
 43 | 
 44 |   page.on("error", async () => {
 45 |     await page.close();
 46 | 
 47 |     return null;
 48 |   });
 49 | 
 50 |   await page.setViewport({ width: 1920, height: 1080, deviceScaleRatio: 2 });
 51 | 
 52 |   try {
 53 |     // await page.goto(link, { waitUntil: "networkidle2" });
 54 |     await page.goto(link, { waitUntil: "load" });
 55 | 
 56 |     didOpen = true;
 57 |   } catch (e) {
 58 |     console.log(
 59 |       "[archivist-pinboard]",
 60 |       "error navigating:",
 61 |       link,
 62 |       e.toString()
 63 |     );
 64 | 
 65 |     didOpen = false;
 66 |   }
 67 | 
 68 |   if (!didOpen) {
 69 |     await page.close();
 70 |     return null;
 71 |   }
 72 | 
 73 |   try {
 74 |     console.log("[archivist-pinboard]", "screenshot:", link);
 75 |     await page.screenshot({ path: screenshotPath });
 76 | 
 77 |     didScreenshot = true;
 78 |   } catch (e) {
 79 |     didScreenshot = false;
 80 |   }
 81 | 
 82 |   try {
 83 |     console.log("[archivist-pinboard]", "freeze:", link);
 84 |     await page.evaluate(FREEZE_DRY_SRC);
 85 | 
 86 |     const frozen = await Promise.race([
 87 |       page.evaluate(async () => await window.freezeDry()),
 88 |       page.waitFor(5000)
 89 |     ]);
 90 | 
 91 |     fs.writeFileSync(frozenPath, frozen, "utf-8");
 92 | 
 93 |     didFreeze = true;
 94 |   } catch (e) {
 95 |     didFreeze = false;
 96 |   }
 97 | 
 98 |   await page.close();
 99 | 
100 |   return {
101 |     screenshot:
102 |       didScreenshot === true ? path.basename(screenshotPath) : undefined,
103 |     frozen: didFreeze === true ? path.basename(frozenPath) : undefined
104 |   };
105 | };
106 | 
107 | const savePage = async (browser, link) => {
108 |   const isOnline = await isReachable(link);
109 | 
110 |   if (!isOnline) {
111 |     console.log("[archivist-pinboard]", "offline, trying wayback for:", link);
112 | 
113 |     return new Promise(resolve => {
114 |       wayback.getClosest(link, (err, closest) => {
115 |         const isError = !!err;
116 |         const isClosest = closest && !!closest.available && !!closest.url;
117 | 
118 |         if (isError || !isClosest) {
119 |           console.log(
120 |             "[archivist-pinboard]",
121 |             "couldn't find wayback for:",
122 |             link
123 |           );
124 |           resolve(null);
125 |         } else {
126 |           console.log(
127 |             "[archivist-pinboard]",
128 |             "found wayback for",
129 |             link,
130 |             "->",
131 |             closest.url
132 |           );
133 | 
134 |           savePageInternal(browser, closest.url, link).then(paths =>
135 |             resolve(paths)
136 |           );
137 |         }
138 |       });
139 |     });
140 |   }
141 | 
142 |   return await savePageInternal(browser, link, link);
143 | };
144 | 
145 | const run = async links => {
146 |   const headless = true;
147 |   const browser = await puppeteer.launch({ headless, ignoreHTTPSErrors: true });
148 | 
149 |   return new Promise((resolve, reject) => {
150 |     async.mapLimit(
151 |       links,
152 |       10,
153 |       (link, callback) => {
154 |         savePage(browser, link.href)
155 |           .then(paths => {
156 |             callback(null, { ...link, paths });
157 |           })
158 |           .catch(e => {
159 |             console.log(
160 |               "[archivist-pinboard]",
161 |               "uncatched error",
162 |               link.href,
163 |               e.toString()
164 |             );
165 |             // ignoring errors for now
166 |             callback(null, null);
167 |           });
168 |       },
169 |       (err, res) => {
170 |         browser.close().then(() => {
171 |           if (err) {
172 |             reject(err);
173 |           } else {
174 |             resolve(res);
175 |           }
176 |         });
177 |       }
178 |     );
179 |   });
180 | };
181 | 
182 | module.exports = run;
183 | 


--------------------------------------------------------------------------------
/archivist-pinboard/fetch/index.js:
--------------------------------------------------------------------------------
  1 | const async = require("async");
  2 | const Database = require("better-sqlite3");
  3 | const Pinboard = require("node-pinboard");
  4 | const envPaths = require("env-paths");
  5 | const fs = require("fs");
  6 | const mkdirp = require("mkdirp");
  7 | const path = require("path");
  8 | const { isString } = require("lodash");
  9 | 
 10 | const fetcher = require("./fetcher");
 11 | 
 12 | const DATA_PATH = envPaths("archivist-pinboard").data;
 13 | const ASSETS_PATH = path.join(DATA_PATH, "assets");
 14 | const FROZEN_PATH = path.join(DATA_PATH, "frozen");
 15 | 
 16 | mkdirp(ASSETS_PATH);
 17 | mkdirp(FROZEN_PATH);
 18 | 
 19 | const CRAWLED_DATA_PATH = path.join(DATA_PATH, "crawled-links.json");
 20 | 
 21 | const processRemovedLinks = async (removedLinks) => {
 22 |   return new Promise((resolve) => {
 23 |     async.mapLimit(
 24 |       removedLinks,
 25 |       10,
 26 |       (item, callback) => {
 27 |         const screenshotPath =
 28 |           item.screenshot && path.join(ASSETS_PATH, item.screenshot);
 29 | 
 30 |         const frozenPath = item.frozen && path.join(FROZEN_PATH, item.frozen);
 31 | 
 32 |         if (screenshotPath && fs.existsSync(screenshotPath)) {
 33 |           console.log("[archivist-pinboard]", `unlinking ${screenshotPath}`);
 34 |           fs.unlinkSync(screenshotPath);
 35 |         }
 36 | 
 37 |         if (frozenPath && fs.existsSync(frozenPath)) {
 38 |           console.log("[archivist-pinboard]", `unlinking ${frozenPath}`);
 39 |           fs.unlinkSync(frozenPath);
 40 |         }
 41 | 
 42 |         callback(null, item.hash);
 43 |       },
 44 |       (err, hashes) => resolve(hashes)
 45 |     );
 46 |   });
 47 | };
 48 | 
 49 | const run = async (options) => {
 50 |   if (!options.apiKey) {
 51 |     throw new Error("apiKey not provided");
 52 |   }
 53 | 
 54 |   const pinboard = new Pinboard(options.apiKey);
 55 | 
 56 |   const crawlLinks = async () =>
 57 |     new Promise((resolve, reject) => {
 58 |       pinboard.all((err, links) => {
 59 |         if (err) {
 60 |           reject(err);
 61 |         } else {
 62 |           resolve(links);
 63 |         }
 64 |       });
 65 |     });
 66 | 
 67 |   const db = new Database(path.join(DATA_PATH, "data.db"));
 68 | 
 69 |   db.prepare(
 70 |     `
 71 |     CREATE TABLE IF NOT EXISTS data (
 72 |       href TEXT,
 73 |       hash TEXT PRIMARY KEY,
 74 |       meta TEXT,
 75 |       description TEXT,
 76 |       extended TEXT,
 77 |       tags TEXT,
 78 |       time DATETIME,
 79 |       screenshot TEXT,
 80 |       frozen TEXT
 81 |     )
 82 |     `
 83 |   ).run();
 84 | 
 85 |   const search = db.prepare(
 86 |     "SELECT count(hash) AS count FROM data WHERE hash = ?"
 87 |   );
 88 | 
 89 |   const insert = db.prepare(
 90 |     `INSERT OR REPLACE INTO data (href,   hash,  meta,  description,  extended,  tags,  time,  screenshot,  frozen)
 91 |      VALUES                      (:href, :hash, :meta, :description, :extended, :tags, :time, :screenshot, :frozen)`
 92 |   );
 93 | 
 94 |   const remove = db.prepare("DELETE FROM data WHERE hash = ?");
 95 | 
 96 |   const dbLinks = db.prepare("SELECT * FROM data").all();
 97 | 
 98 |   let crawledLinks = await crawlLinks();
 99 | 
100 |   // not sure what's going on in here really
101 |   if (isString(crawledLinks)) {
102 |     try {
103 |       crawledLinks = JSON.parse(crawledLinks.slice(1));
104 |     } catch (e) {}
105 |   }
106 | 
107 |   if (isString(crawledLinks)) {
108 |     console.log("[archivist-pinboard] unrecoverable issue with crawled links");
109 |     return;
110 |   }
111 | 
112 |   // const crawledLinks = require(CRAWLED_DATA_PATH);
113 | 
114 |   fs.writeFileSync(
115 |     CRAWLED_DATA_PATH,
116 |     JSON.stringify(crawledLinks, null, 2),
117 |     "utf-8"
118 |   );
119 |   // console.log("[archivist-pinboard]", `crawled data saved to ${CRAWLED_DATA_PATH}`);
120 | 
121 |   const newLinks = crawledLinks.filter(
122 |     (link) => search.get(link.hash).count === 0
123 |   );
124 | 
125 |   const removedLinks = dbLinks.filter(
126 |     ({ hash }) => !crawledLinks.find((l) => l.hash === hash)
127 |   );
128 | 
129 |   console.log(
130 |     "[archivist-pinboard]",
131 |     `all links: ${crawledLinks.length} / new links: ${newLinks.length} / removed links: ${removedLinks.length}`
132 |   );
133 | 
134 |   const hashesToRemove = await processRemovedLinks(removedLinks);
135 | 
136 |   const removeLinks = db.transaction((hashes) => {
137 |     hashes.forEach((hash) => remove.run(hash));
138 |   });
139 | 
140 |   removeLinks(hashesToRemove);
141 | 
142 |   const fetchedLinks = await fetcher(newLinks);
143 | 
144 |   const finalLinks = fetchedLinks
145 |     .filter((link) => link && link.paths)
146 |     .map((link) => ({
147 |       href: link.href,
148 |       hash: link.hash,
149 |       meta: link.meta,
150 |       description: link.description,
151 |       extended: link.extended,
152 |       tags: link.tags,
153 |       time: link.time,
154 |       screenshot: link.paths.screenshot,
155 |       frozen: link.paths.frozen,
156 |     }));
157 | 
158 |   const insertLinks = db.transaction((links) => {
159 |     links.forEach((link) => insert.run(link));
160 |   });
161 | 
162 |   insertLinks(finalLinks);
163 | 
164 |   console.log(
165 |     "[archivist-pinboard]",
166 |     `insterted links: ${finalLinks.length} (of ${newLinks.length} new links)`
167 |   );
168 | };
169 | 
170 | module.exports = run;
171 | 


--------------------------------------------------------------------------------
/archivist-pinboard/index.js:
--------------------------------------------------------------------------------
1 | const fetch = require("./fetch");
2 | const query = require("./query");
3 | 
4 | module.exports = options => ({
5 |   fetch: () => fetch(options),
6 |   get: () => query(options),
7 |   query: text => query(options, text)
8 | });
9 | 


--------------------------------------------------------------------------------
/archivist-pinboard/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "archivist-pinboard",
 3 |   "version": "1.2.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "postinstall": "node scripts/compile-freeze-dry.js"
 8 |   },
 9 |   "keywords": [],
10 |   "author": "Szymon Kaliski <hi@szymonkaliski.com> (http://szymonkaliski.com)",
11 |   "license": "MIT",
12 |   "dependencies": {
13 |     "async": "^3.2.0",
14 |     "better-sqlite3": "^6.0.1",
15 |     "env-paths": "^2.2.0",
16 |     "freeze-dry": "^0.2.4",
17 |     "is-reachable": "^4.0.0",
18 |     "lodash": "^4.17.15",
19 |     "md5": "^2.2.1",
20 |     "mkdirp": "^1.0.4",
21 |     "node-pinboard": "^1.0.0",
22 |     "puppeteer": "^3.0.1",
23 |     "wayback-machine": "^0.2.1"
24 |   },
25 |   "devDependencies": {
26 |     "browserify": "^16.5.1"
27 |   }
28 | }
29 | 


--------------------------------------------------------------------------------
/archivist-pinboard/query/index.js:
--------------------------------------------------------------------------------
 1 | const envPaths = require("env-paths");
 2 | const Database = require("better-sqlite3");
 3 | const path = require("path");
 4 | 
 5 | const DATA_PATH = envPaths("archivist-pinboard").data;
 6 | const ASSETS_PATH = path.join(DATA_PATH, "assets");
 7 | const FROZEN_PATH = path.join(DATA_PATH, "frozen");
 8 | 
 9 | const query = async (_, text) => {
10 |   const db = new Database(path.join(DATA_PATH, "data.db"));
11 |   let search;
12 | 
13 |   if (text) {
14 |     search = db
15 |       .prepare(
16 |         `
17 |         SELECT *
18 |         FROM data
19 |         WHERE
20 |           description LIKE :query OR
21 |           extended LIKE :query OR
22 |           tags LIKE :query OR
23 |           href LIKE :query
24 |         `
25 |       )
26 |       .all({ query: `%${text}%` });
27 |   } else {
28 |     search = db.prepare("SELECT * FROM data").all();
29 |   }
30 | 
31 |   return search.map(d => ({
32 |     img: path.join(ASSETS_PATH, d.screenshot),
33 |     link: d.href,
34 |     id: d.hash,
35 |     time: d.time,
36 | 
37 |     width: 1920,
38 |     height: 1080,
39 | 
40 |     meta: {
41 |       source: "pinboard",
42 |       title: d.description,
43 |       note: d.extended,
44 |       tags: d.tags.split(" "),
45 |       static: d.frozen ? path.join(FROZEN_PATH, d.frozen) : undefined
46 |     }
47 |   }));
48 | };
49 | 
50 | module.exports = query;
51 | 


--------------------------------------------------------------------------------
/archivist-pinboard/scripts/compile-freeze-dry.js:
--------------------------------------------------------------------------------
 1 | const path = require("path");
 2 | const browserify = require("browserify");
 3 | 
 4 | const freezeDryFunction = require("fs").createWriteStream(
 5 |   path.resolve(__dirname, "../fetch/assets/freeze-dry-browserified.js")
 6 | );
 7 | 
 8 | freezeDryFunction.write("window.freezeDry = (async () => {\n");
 9 | 
10 | browserify()
11 |   .require(path.resolve(__dirname, "../node_modules/freeze-dry/lib/index.js"), {
12 |     expose: "freeze-dry"
13 |   })
14 |   .bundle()
15 |   .on("end", () => {
16 |     freezeDryFunction.write('return await require("freeze-dry").default()})');
17 |     freezeDryFunction.end();
18 |   })
19 |   .pipe(freezeDryFunction, { end: false });
20 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | .creds.json
3 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/fetch/crawler.js:
--------------------------------------------------------------------------------
  1 | const assert = require("assert");
  2 | const async = require("async");
  3 | const chrome = require("chrome-cookies-secure");
  4 | const puppeteer = require("puppeteer");
  5 | const { chain, flatten } = require("lodash");
  6 | 
  7 | const ROOT = "https://pinterest.com";
  8 | 
  9 | const sleep = time => new Promise(resolve => setTimeout(resolve, time));
 10 | 
 11 | const crawlPin = async (browser, pinUrl) => {
 12 |   console.log("[archivist-pinterest-crawl]", "crawling pin", pinUrl);
 13 | 
 14 |   const page = await browser.newPage();
 15 |   await page.setViewport({ width: 1600, height: 900, deviceScaleRatio: 2 });
 16 | 
 17 |   await page.goto(pinUrl, { waitUntil: "networkidle2" });
 18 | 
 19 |   const { link, title, date } = await page.evaluate(() => {
 20 |     const getLink = () => {
 21 |       const link = document.querySelector(".linkModuleActionButton");
 22 |       return link ? link.href : undefined;
 23 |     };
 24 | 
 25 |     const getTitle = () => {
 26 |       const titleCard = document.querySelector(".CloseupTitleCard");
 27 |       return titleCard ? titleCard.textContent : undefined;
 28 |     };
 29 | 
 30 |     const getDate = () => {
 31 |       let date = undefined;
 32 | 
 33 |       try {
 34 |         date = Object.values(
 35 |           JSON.parse(document.getElementById("initial-state").innerText).pins
 36 |         ).map(p => p.created_at)[0];
 37 |       } catch (e) {}
 38 | 
 39 |       return date;
 40 |     };
 41 | 
 42 |     const getData = () => ({
 43 |       link: getLink(),
 44 |       title: getTitle(),
 45 |       date: getDate()
 46 |     });
 47 | 
 48 |     return new Promise(resolve => {
 49 |       const data = getData();
 50 | 
 51 |       if (data.link || data.title || data.date) {
 52 |         resolve(data);
 53 |       } else {
 54 |         // try once more and give up
 55 |         setTimeout(() => {
 56 |           resolve(getData());
 57 |         }, 1000);
 58 |       }
 59 |     });
 60 |   });
 61 | 
 62 |   await page.close();
 63 | 
 64 |   return { link, title, date };
 65 | };
 66 | 
 67 | const crawlBoard = async (page, boardUrl) => {
 68 |   console.log("[archivist-pinterest-crawl]", "crawling board", boardUrl);
 69 | 
 70 |   await page.goto(boardUrl, { waitUntil: "networkidle2" });
 71 | 
 72 |   await sleep(2000); // boards load slowly, but "networkidle0" causes timeout
 73 | 
 74 |   // scroll down to bottom (hopefully)
 75 |   const scrollResult = await page.evaluate(
 76 |     () =>
 77 |       new Promise(resolve => {
 78 |         let lastScrollPosition = 0;
 79 |         const allPins = {};
 80 | 
 81 |         const scrollDown = () => {
 82 |           window.scrollTo(0, window.scrollY + 10);
 83 | 
 84 |           setTimeout(() => {
 85 |             Array.from(document.querySelectorAll("[data-test-id=pin]")).forEach(
 86 |               pin => {
 87 |                 const a = pin.querySelector("a");
 88 |                 const img = pin.querySelector("img");
 89 | 
 90 |                 if (a && img) {
 91 |                   const url = a.href;
 92 |                   const src = img.src;
 93 |                   const srcset = img.srcset;
 94 |                   const alt = img.alt;
 95 | 
 96 |                   allPins[url] = { url, src, alt, srcset };
 97 |                 } else {
 98 |                   console.log(
 99 |                     "[archivist-pinterest-crawl]",
100 |                     "no a/img for",
101 |                     pin
102 |                   );
103 |                 }
104 |               }
105 |             );
106 | 
107 |             if (window.scrollY === lastScrollPosition) {
108 |               resolve(Object.values(allPins));
109 |             } else {
110 |               lastScrollPosition = window.scrollY;
111 |               scrollDown();
112 |             }
113 |           }, 10);
114 |         };
115 | 
116 |         scrollDown();
117 |       })
118 |   );
119 | 
120 |   return scrollResult.map(pin => {
121 |     return {
122 |       ...pin,
123 |       biggestSrc: chain(pin.srcset)
124 |         .split(",")
125 |         .last()
126 |         .trim()
127 |         .split(" ")
128 |         .first()
129 |         .value()
130 |     };
131 |   });
132 | };
133 | 
134 | const crawlProfile = async (page, profileUrl) => {
135 |   console.log("[archivist-pinterest-crawl]", "crawling profile", profileUrl);
136 | 
137 |   await page.goto(profileUrl, { waitUntil: "networkidle2" });
138 | 
139 |   const boards = await page.evaluate(() => {
140 |     return Array.from(document.querySelectorAll("[draggable=true]")).map(el => {
141 |       return el.querySelector("a").href;
142 |     });
143 |   });
144 | 
145 |   return boards;
146 | };
147 | 
148 | const loginWithCreds = async (page, email, password) => {
149 |   await page.goto(ROOT, { waitUntil: "networkidle2" });
150 |   await page.click("[data-test-id=simple-login-button] > button");
151 | 
152 |   await page.type("#email", email);
153 |   await sleep(2000);
154 |   await page.type("#password", password);
155 |   await sleep(2000);
156 | 
157 |   await page.click(".SignupButton");
158 |   await page.waitForNavigation();
159 | };
160 | 
161 | const loginWithCookiesFromChrome = async page =>
162 |   new Promise(resolve => {
163 |     chrome.getCookies(ROOT, "puppeteer", (err, cookies) => {
164 |       page.setCookie(...cookies).then(() => {
165 |         page.goto(ROOT, { waitUntil: "networkidle2" }).then(() => {
166 |           resolve();
167 |         });
168 |       });
169 |     });
170 |   });
171 | 
172 | const createBrowser = async options => {
173 |   const headless = true;
174 |   const browser = await puppeteer.launch({ headless });
175 | 
176 |   const page = await browser.newPage();
177 |   await page.setViewport({ width: 1600, height: 900, deviceScaleRatio: 2 });
178 | 
179 |   if (options.loginMethod === "cookies") {
180 |     await loginWithCookiesFromChrome(page);
181 |   } else if (options.loginMethod === "password") {
182 |     await loginWithCreds(page, options.username, options.password);
183 |   } else {
184 |     throw new Error("invalid login option");
185 |   }
186 | 
187 |   assert(options.profile, "requires profile option");
188 | 
189 |   return { browser, page };
190 | };
191 | 
192 | const crawlBoards = async options => {
193 |   const { browser, page } = await createBrowser(options);
194 | 
195 |   const boards = await crawlProfile(page, ROOT + "/" + options.profile + "/boards");
196 | 
197 |   return new Promise(resolve => {
198 |     async.mapSeries(
199 |       boards,
200 |       (board, callback) =>
201 |         crawlBoard(page, board).then(pins => {
202 |           console.log(
203 |             "[archivist-pinterest-crawl]",
204 |             "board pins:",
205 |             board,
206 |             pins.length
207 |           );
208 | 
209 |           callback(
210 |             null,
211 |             pins.map(pin => ({
212 |               ...pin,
213 |               board: chain(board)
214 |                 .split("/")
215 |                 .takeRight(2)
216 |                 .first()
217 |                 .value()
218 |             }))
219 |           );
220 |         }),
221 |       (err, res) => {
222 |         browser.close().then(() => {
223 |           resolve(flatten(res));
224 |         });
225 |       }
226 |     );
227 |   });
228 | };
229 | 
230 | const crawlPinMetadata = async (options, pins) => {
231 |   const { browser } = await createBrowser(options);
232 | 
233 |   return new Promise(resolve => {
234 |     async.mapLimit(
235 |       pins,
236 |       4,
237 |       (pin, callback) => {
238 |         crawlPin(browser, pin.url).then(({ link, title, date }) => {
239 |           callback(null, { ...pin, title, link, createdAt: date });
240 |         });
241 |       },
242 |       (err, res) => {
243 |         browser.close().then(() => {
244 |           resolve(res);
245 |         });
246 |       }
247 |     );
248 |   });
249 | };
250 | 
251 | module.exports = { crawlBoards, crawlPinMetadata };
252 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/fetch/fetcher.js:
--------------------------------------------------------------------------------
 1 | const async = require("async");
 2 | const envPaths = require("env-paths");
 3 | const fs = require("fs");
 4 | const md5 = require("md5");
 5 | const mkdirp = require("mkdirp");
 6 | const path = require("path");
 7 | const sizeOf = require("image-size");
 8 | const tmp = require("tmp");
 9 | const wget = require("node-wget");
10 | 
11 | const TMP_PATH = envPaths("archivist-pinterest").data;
12 | const DATA_PATH = envPaths("archivist-pinterest").data;
13 | const ASSETS_PATH = path.join(DATA_PATH, "assets");
14 | 
15 | mkdirp(TMP_PATH);
16 | mkdirp(DATA_PATH);
17 | mkdirp(ASSETS_PATH);
18 | 
19 | const download = async url => {
20 |   console.log("[archivist-pinterest-crawl]", "downloading", url);
21 | 
22 |   const tempPath = tmp.tmpNameSync();
23 | 
24 |   return new Promise((resolve, reject) =>
25 |     wget({ url, dest: tempPath }, (error, result, body) => {
26 |       if (error) {
27 |         return reject(error);
28 |       }
29 | 
30 |       const ext = path.extname(url);
31 |       const hash = md5(body);
32 |       const filename = `${hash}${ext}`;
33 |       const finalPath = path.join(ASSETS_PATH, filename);
34 | 
35 |       fs.renameSync(tempPath, finalPath);
36 | 
37 |       sizeOf(finalPath, (err, size) => {
38 |         if (err) {
39 |           console.log(
40 |             "[archivist-pinterest-crawl]",
41 |             `image-size error: ${err} (${finalPath})`
42 |           );
43 | 
44 |           resolve({ filename, width: 0, height: 0 });
45 |         } else {
46 |           resolve({ filename, ...size });
47 |         }
48 |       });
49 |     })
50 |   );
51 | };
52 | 
53 | module.exports = async crawledPins => {
54 |   return new Promise(resolve => {
55 |     async.mapLimit(
56 |       crawledPins,
57 |       10,
58 |       async pin => {
59 |         const { filename, width, height } = await download(pin.biggestSrc);
60 |         return { ...pin, filename, width, height };
61 |       },
62 |       (err, res) => {
63 |         resolve(res);
64 |       }
65 |     );
66 |   });
67 | };
68 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/fetch/index.js:
--------------------------------------------------------------------------------
  1 | const Database = require("better-sqlite3");
  2 | const async = require("async");
  3 | const dateFormat = require("dateformat");
  4 | const envPaths = require("env-paths");
  5 | const fs = require("fs");
  6 | const mkdirp = require("mkdirp");
  7 | const path = require("path");
  8 | const { chain } = require("lodash");
  9 | 
 10 | const { crawlBoards, crawlPinMetadata } = require("./crawler");
 11 | const fetcher = require("./fetcher");
 12 | 
 13 | const DATA_PATH = envPaths("archivist-pinterest").data;
 14 | mkdirp(DATA_PATH);
 15 | 
 16 | const CRAWLED_DATA_PATH = path.join(DATA_PATH, "crawled-pins.json");
 17 | 
 18 | const makePinId = pin => {
 19 |   return chain(pin.url)
 20 |     .split("/")
 21 |     .takeRight(2)
 22 |     .first()
 23 |     .value();
 24 | };
 25 | 
 26 | const processRemovedPins = async removedPins => {
 27 |   return new Promise(resolve => {
 28 |     async.mapLimit(
 29 |       removedPins,
 30 |       10,
 31 |       (item, callback) => {
 32 |         const filePath = item.filename && path.join(DATA_PATH, item.filename);
 33 | 
 34 |         if (filePath && fs.existsSync(filePath)) {
 35 |           console.log("[archivist-pinterest-crawl]", `unlinking ${filePath}`);
 36 |           fs.unlinkSync(filePath);
 37 |         }
 38 | 
 39 |         callback(null, item.pinid);
 40 |       },
 41 |       (err, pinids) => resolve(pinids)
 42 |     );
 43 |   });
 44 | };
 45 | 
 46 | const run = async options => {
 47 |   const db = new Database(path.join(DATA_PATH, "data.db"));
 48 | 
 49 |   db.prepare(
 50 |     `
 51 |     CREATE TABLE IF NOT EXISTS data (
 52 |       board TEXT,
 53 |       filename TEXT,
 54 |       title TEXT,
 55 |       text TEXT,
 56 |       link TEXT,
 57 |       pinurl TEXT,
 58 |       width INTEGER,
 59 |       height INTEGER,
 60 |       pinid TEXT PRIMARY KEY,
 61 |       crawldate DATETIME,
 62 |       createdat DATETIME
 63 |     )
 64 |     `
 65 |   ).run();
 66 | 
 67 |   const search = db.prepare(
 68 |     "SELECT count(pinid) AS count FROM data WHERE pinid = ?"
 69 |   );
 70 | 
 71 |   const insert = db.prepare(
 72 |     `INSERT OR REPLACE INTO data (board,   filename,  title,  text,  link,  pinurl,  pinid,  crawldate,  createdat,  width,  height)
 73 |      VALUES                      (:board, :filename, :title, :text, :link, :pinurl, :pinid, :crawldate, :createdat, :width, :height)`
 74 |   );
 75 | 
 76 |   const remove = db.prepare("DELETE FROM data WHERE pinid = ?");
 77 | 
 78 |   const dbPins = db.prepare("SELECT * FROM data").all();
 79 | 
 80 |   const crawledPins = await crawlBoards(options);
 81 |   // const crawledPins = require(CRAWLED_DATA_PATH);
 82 | 
 83 |   if (crawledPins.length === 0) {
 84 |     console.log("[archivist-pinterest-crawl]", "0 crawled pins, exiting");
 85 |     return;
 86 |   }
 87 | 
 88 |   fs.writeFileSync(
 89 |     CRAWLED_DATA_PATH,
 90 |     JSON.stringify(crawledPins, null, 2),
 91 |     "utf-8"
 92 |   );
 93 |   // console.log("[archivist-pinterest-crawl]", `crawled data saved to ${CRAWLED_DATA_PATH}`);
 94 | 
 95 |   const newPins = crawledPins.filter(pin => {
 96 |     if (!pin) {
 97 |       return false;
 98 |     }
 99 | 
100 |     const pinid = makePinId(pin);
101 |     return search.get(pinid).count === 0;
102 |   });
103 | 
104 |   const removedPins = dbPins.filter(
105 |     ({ pinid }) => !crawledPins.find(pin => makePinId(pin) === pinid)
106 |   );
107 | 
108 |   console.log(
109 |     "[archivist-pinterest-crawl]",
110 |     `all pins: ${crawledPins.length} / new pins: ${newPins.length} / removed pins: ${removedPins.length}`
111 |   );
112 | 
113 |   const pinidsToRemove = await processRemovedPins(removedPins);
114 | 
115 |   const removePins = db.transaction(pinids => {
116 |     pinids.forEach(pinid => remove.run(pinid));
117 |   });
118 | 
119 |   removePins(pinidsToRemove);
120 | 
121 |   const newPinsWithMetadata = await crawlPinMetadata(options, newPins);
122 | 
123 |   const fetchedPins = await fetcher(newPinsWithMetadata);
124 | 
125 |   const crawldate = dateFormat(new Date(), "isoDateTime");
126 | 
127 |   const finalPins = fetchedPins.map(pin => ({
128 |     board: pin.board,
129 |     filename: pin.filename,
130 |     title: pin.title,
131 |     text: pin.alt,
132 |     link: pin.link,
133 |     pinurl: pin.url,
134 |     pinid: makePinId(pin),
135 |     crawldate,
136 |     createdat: pin.createdAt
137 |       ? dateFormat(new Date(pin.createdAt), "isoDateTime")
138 |       : undefined,
139 |     width: pin.width,
140 |     height: pin.height
141 |   }));
142 | 
143 |   const insertPins = db.transaction(pins => {
144 |     pins.forEach(pin => insert.run(pin));
145 |   });
146 | 
147 |   insertPins(finalPins);
148 | 
149 |   console.log(
150 |     "[archivist-pinterest-crawl]",
151 |     `inserted pins: ${finalPins.length} (of ${newPins.length})`
152 |   );
153 | };
154 | 
155 | module.exports = run;
156 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/index.js:
--------------------------------------------------------------------------------
1 | const fetch = require("./fetch");
2 | const query = require("./query");
3 | 
4 | module.exports = options => ({
5 |   fetch: () => fetch(options),
6 |   get: () => query(options),
7 |   query: text => query(options, text)
8 | });
9 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/package-lock.json:
--------------------------------------------------------------------------------
   1 | {
   2 |   "name": "archivist-pinterest-crawl",
   3 |   "version": "1.2.0",
   4 |   "lockfileVersion": 1,
   5 |   "requires": true,
   6 |   "dependencies": {
   7 |     "@types/mime-types": {
   8 |       "version": "2.1.0",
   9 |       "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz",
  10 |       "integrity": "sha1-nKUs2jY/aZxpRmwqbM2q2RPqenM="
  11 |     },
  12 |     "@types/node": {
  13 |       "version": "12.12.14",
  14 |       "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.14.tgz",
  15 |       "integrity": "sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA=="
  16 |     },
  17 |     "@types/yauzl": {
  18 |       "version": "2.9.1",
  19 |       "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
  20 |       "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
  21 |       "optional": true,
  22 |       "requires": {
  23 |         "@types/node": "*"
  24 |       }
  25 |     },
  26 |     "abbrev": {
  27 |       "version": "1.1.1",
  28 |       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
  29 |       "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
  30 |     },
  31 |     "agent-base": {
  32 |       "version": "5.1.1",
  33 |       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
  34 |       "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
  35 |     },
  36 |     "ajv": {
  37 |       "version": "6.12.2",
  38 |       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
  39 |       "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
  40 |       "requires": {
  41 |         "fast-deep-equal": "^3.1.1",
  42 |         "fast-json-stable-stringify": "^2.0.0",
  43 |         "json-schema-traverse": "^0.4.1",
  44 |         "uri-js": "^4.2.2"
  45 |       }
  46 |     },
  47 |     "ansi-regex": {
  48 |       "version": "2.1.1",
  49 |       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
  50 |       "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
  51 |     },
  52 |     "aproba": {
  53 |       "version": "1.2.0",
  54 |       "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
  55 |       "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
  56 |     },
  57 |     "are-we-there-yet": {
  58 |       "version": "1.1.5",
  59 |       "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
  60 |       "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
  61 |       "requires": {
  62 |         "delegates": "^1.0.0",
  63 |         "readable-stream": "^2.0.6"
  64 |       }
  65 |     },
  66 |     "asn1": {
  67 |       "version": "0.2.4",
  68 |       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
  69 |       "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
  70 |       "requires": {
  71 |         "safer-buffer": "~2.1.0"
  72 |       }
  73 |     },
  74 |     "assert-plus": {
  75 |       "version": "1.0.0",
  76 |       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
  77 |       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
  78 |     },
  79 |     "async": {
  80 |       "version": "3.2.0",
  81 |       "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz",
  82 |       "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw=="
  83 |     },
  84 |     "asynckit": {
  85 |       "version": "0.4.0",
  86 |       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
  87 |       "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
  88 |     },
  89 |     "aws-sign2": {
  90 |       "version": "0.7.0",
  91 |       "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
  92 |       "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
  93 |     },
  94 |     "aws4": {
  95 |       "version": "1.9.0",
  96 |       "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.0.tgz",
  97 |       "integrity": "sha512-Uvq6hVe90D0B2WEnUqtdgY1bATGz3mw33nH9Y+dmA+w5DHvUmBgkr5rM/KCHpCsiFNRUfokW/szpPPgMK2hm4A=="
  98 |     },
  99 |     "balanced-match": {
 100 |       "version": "1.0.0",
 101 |       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
 102 |       "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
 103 |     },
 104 |     "base64-js": {
 105 |       "version": "1.3.1",
 106 |       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
 107 |       "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
 108 |     },
 109 |     "bcrypt-pbkdf": {
 110 |       "version": "1.0.2",
 111 |       "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
 112 |       "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
 113 |       "requires": {
 114 |         "tweetnacl": "^0.14.3"
 115 |       }
 116 |     },
 117 |     "better-sqlite3": {
 118 |       "version": "6.0.1",
 119 |       "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-6.0.1.tgz",
 120 |       "integrity": "sha512-4aV1zEknM9g1a6B0mVBx1oIlmYioEJ8gSS3J6EpN1b1bKYEE+N5lmpmXHKNKTi0qjHziSd7XrXwHl1kpqvEcHQ==",
 121 |       "requires": {
 122 |         "bindings": "^1.5.0",
 123 |         "integer": "^3.0.1",
 124 |         "prebuild-install": "^5.3.3",
 125 |         "tar": "4.4.10"
 126 |       }
 127 |     },
 128 |     "bindings": {
 129 |       "version": "1.5.0",
 130 |       "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
 131 |       "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
 132 |       "requires": {
 133 |         "file-uri-to-path": "1.0.0"
 134 |       }
 135 |     },
 136 |     "bl": {
 137 |       "version": "4.0.2",
 138 |       "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
 139 |       "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
 140 |       "requires": {
 141 |         "buffer": "^5.5.0",
 142 |         "inherits": "^2.0.4",
 143 |         "readable-stream": "^3.4.0"
 144 |       },
 145 |       "dependencies": {
 146 |         "readable-stream": {
 147 |           "version": "3.6.0",
 148 |           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
 149 |           "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
 150 |           "requires": {
 151 |             "inherits": "^2.0.3",
 152 |             "string_decoder": "^1.1.1",
 153 |             "util-deprecate": "^1.0.1"
 154 |           }
 155 |         }
 156 |       }
 157 |     },
 158 |     "boolbase": {
 159 |       "version": "1.0.0",
 160 |       "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
 161 |       "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
 162 |     },
 163 |     "boom": {
 164 |       "version": "4.3.1",
 165 |       "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
 166 |       "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=",
 167 |       "requires": {
 168 |         "hoek": "4.x.x"
 169 |       }
 170 |     },
 171 |     "brace-expansion": {
 172 |       "version": "1.1.11",
 173 |       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
 174 |       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
 175 |       "requires": {
 176 |         "balanced-match": "^1.0.0",
 177 |         "concat-map": "0.0.1"
 178 |       }
 179 |     },
 180 |     "buffer": {
 181 |       "version": "5.6.0",
 182 |       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
 183 |       "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
 184 |       "requires": {
 185 |         "base64-js": "^1.0.2",
 186 |         "ieee754": "^1.1.4"
 187 |       }
 188 |     },
 189 |     "buffer-crc32": {
 190 |       "version": "0.2.13",
 191 |       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
 192 |       "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
 193 |     },
 194 |     "caseless": {
 195 |       "version": "0.12.0",
 196 |       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
 197 |       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
 198 |     },
 199 |     "charenc": {
 200 |       "version": "0.0.2",
 201 |       "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
 202 |       "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
 203 |     },
 204 |     "cheerio": {
 205 |       "version": "1.0.0-rc.3",
 206 |       "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
 207 |       "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
 208 |       "requires": {
 209 |         "css-select": "~1.2.0",
 210 |         "dom-serializer": "~0.1.1",
 211 |         "entities": "~1.1.1",
 212 |         "htmlparser2": "^3.9.1",
 213 |         "lodash": "^4.15.0",
 214 |         "parse5": "^3.0.1"
 215 |       }
 216 |     },
 217 |     "chownr": {
 218 |       "version": "1.1.4",
 219 |       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
 220 |       "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
 221 |     },
 222 |     "chrome-cookies-secure": {
 223 |       "version": "1.3.2",
 224 |       "resolved": "https://registry.npmjs.org/chrome-cookies-secure/-/chrome-cookies-secure-1.3.2.tgz",
 225 |       "integrity": "sha512-HA3dmbZPiK7/IEA457UOOrZu7DYwCqAb/3HaXirg+qLNVTeyNYVcAIs1INY2mqOeEAlN5vUls1G9Ub9EJJ4NGw==",
 226 |       "requires": {
 227 |         "int": "^0.2.0",
 228 |         "keytar": "^5.0.0",
 229 |         "request": "^2.88.0",
 230 |         "sqlite3": "^4.1.1",
 231 |         "tldjs": "^1.5.1",
 232 |         "tough-cookie": "^2.3.4",
 233 |         "win-dpapi": "^0.1.0"
 234 |       }
 235 |     },
 236 |     "co": {
 237 |       "version": "4.6.0",
 238 |       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
 239 |       "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
 240 |     },
 241 |     "code-point-at": {
 242 |       "version": "1.1.0",
 243 |       "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
 244 |       "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
 245 |     },
 246 |     "combined-stream": {
 247 |       "version": "1.0.8",
 248 |       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
 249 |       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
 250 |       "requires": {
 251 |         "delayed-stream": "~1.0.0"
 252 |       }
 253 |     },
 254 |     "concat-map": {
 255 |       "version": "0.0.1",
 256 |       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
 257 |       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
 258 |     },
 259 |     "console-control-strings": {
 260 |       "version": "1.1.0",
 261 |       "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
 262 |       "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
 263 |     },
 264 |     "core-util-is": {
 265 |       "version": "1.0.2",
 266 |       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
 267 |       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
 268 |     },
 269 |     "crypt": {
 270 |       "version": "0.0.2",
 271 |       "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
 272 |       "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
 273 |     },
 274 |     "cryptiles": {
 275 |       "version": "3.1.4",
 276 |       "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.4.tgz",
 277 |       "integrity": "sha512-8I1sgZHfVwcSOY6mSGpVU3lw/GSIZvusg8dD2+OGehCJpOhQRLNcH0qb9upQnOH4XhgxxFJSg6E2kx95deb1Tw==",
 278 |       "requires": {
 279 |         "boom": "5.x.x"
 280 |       },
 281 |       "dependencies": {
 282 |         "boom": {
 283 |           "version": "5.2.0",
 284 |           "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz",
 285 |           "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==",
 286 |           "requires": {
 287 |             "hoek": "4.x.x"
 288 |           }
 289 |         }
 290 |       }
 291 |     },
 292 |     "css-select": {
 293 |       "version": "1.2.0",
 294 |       "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
 295 |       "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
 296 |       "requires": {
 297 |         "boolbase": "~1.0.0",
 298 |         "css-what": "2.1",
 299 |         "domutils": "1.5.1",
 300 |         "nth-check": "~1.0.1"
 301 |       }
 302 |     },
 303 |     "css-what": {
 304 |       "version": "2.1.3",
 305 |       "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
 306 |       "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
 307 |     },
 308 |     "dashdash": {
 309 |       "version": "1.14.1",
 310 |       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
 311 |       "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
 312 |       "requires": {
 313 |         "assert-plus": "^1.0.0"
 314 |       }
 315 |     },
 316 |     "dateformat": {
 317 |       "version": "3.0.3",
 318 |       "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
 319 |       "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q=="
 320 |     },
 321 |     "debug": {
 322 |       "version": "3.2.6",
 323 |       "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
 324 |       "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
 325 |       "requires": {
 326 |         "ms": "^2.1.1"
 327 |       }
 328 |     },
 329 |     "decompress-response": {
 330 |       "version": "4.2.1",
 331 |       "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
 332 |       "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
 333 |       "requires": {
 334 |         "mimic-response": "^2.0.0"
 335 |       }
 336 |     },
 337 |     "deep-extend": {
 338 |       "version": "0.6.0",
 339 |       "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
 340 |       "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
 341 |     },
 342 |     "delayed-stream": {
 343 |       "version": "1.0.0",
 344 |       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
 345 |       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
 346 |     },
 347 |     "delegates": {
 348 |       "version": "1.0.0",
 349 |       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
 350 |       "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
 351 |     },
 352 |     "detect-libc": {
 353 |       "version": "1.0.3",
 354 |       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
 355 |       "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
 356 |     },
 357 |     "dom-serializer": {
 358 |       "version": "0.1.1",
 359 |       "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
 360 |       "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
 361 |       "requires": {
 362 |         "domelementtype": "^1.3.0",
 363 |         "entities": "^1.1.1"
 364 |       }
 365 |     },
 366 |     "domelementtype": {
 367 |       "version": "1.3.1",
 368 |       "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
 369 |       "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
 370 |     },
 371 |     "domhandler": {
 372 |       "version": "2.4.2",
 373 |       "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
 374 |       "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
 375 |       "requires": {
 376 |         "domelementtype": "1"
 377 |       }
 378 |     },
 379 |     "domutils": {
 380 |       "version": "1.5.1",
 381 |       "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
 382 |       "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
 383 |       "requires": {
 384 |         "dom-serializer": "0",
 385 |         "domelementtype": "1"
 386 |       }
 387 |     },
 388 |     "ecc-jsbn": {
 389 |       "version": "0.1.2",
 390 |       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
 391 |       "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
 392 |       "requires": {
 393 |         "jsbn": "~0.1.0",
 394 |         "safer-buffer": "^2.1.0"
 395 |       }
 396 |     },
 397 |     "end-of-stream": {
 398 |       "version": "1.4.4",
 399 |       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
 400 |       "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
 401 |       "requires": {
 402 |         "once": "^1.4.0"
 403 |       }
 404 |     },
 405 |     "entities": {
 406 |       "version": "1.1.2",
 407 |       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
 408 |       "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
 409 |     },
 410 |     "env-paths": {
 411 |       "version": "2.2.0",
 412 |       "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.0.tgz",
 413 |       "integrity": "sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA=="
 414 |     },
 415 |     "expand-template": {
 416 |       "version": "2.0.3",
 417 |       "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
 418 |       "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="
 419 |     },
 420 |     "extend": {
 421 |       "version": "3.0.2",
 422 |       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
 423 |       "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
 424 |     },
 425 |     "extract-zip": {
 426 |       "version": "2.0.0",
 427 |       "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.0.tgz",
 428 |       "integrity": "sha512-i42GQ498yibjdvIhivUsRslx608whtGoFIhF26Z7O4MYncBxp8CwalOs1lnHy21A9sIohWO2+uiE4SRtC9JXDg==",
 429 |       "requires": {
 430 |         "@types/yauzl": "^2.9.1",
 431 |         "debug": "^4.1.1",
 432 |         "get-stream": "^5.1.0",
 433 |         "yauzl": "^2.10.0"
 434 |       },
 435 |       "dependencies": {
 436 |         "debug": {
 437 |           "version": "4.1.1",
 438 |           "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
 439 |           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
 440 |           "requires": {
 441 |             "ms": "^2.1.1"
 442 |           }
 443 |         }
 444 |       }
 445 |     },
 446 |     "extsprintf": {
 447 |       "version": "1.3.0",
 448 |       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
 449 |       "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
 450 |     },
 451 |     "fast-deep-equal": {
 452 |       "version": "3.1.1",
 453 |       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
 454 |       "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA=="
 455 |     },
 456 |     "fast-json-stable-stringify": {
 457 |       "version": "2.0.0",
 458 |       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
 459 |       "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
 460 |     },
 461 |     "fd-slicer": {
 462 |       "version": "1.1.0",
 463 |       "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
 464 |       "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
 465 |       "requires": {
 466 |         "pend": "~1.2.0"
 467 |       }
 468 |     },
 469 |     "file-uri-to-path": {
 470 |       "version": "1.0.0",
 471 |       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
 472 |       "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
 473 |     },
 474 |     "forever-agent": {
 475 |       "version": "0.6.1",
 476 |       "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
 477 |       "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
 478 |     },
 479 |     "form-data": {
 480 |       "version": "2.3.3",
 481 |       "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
 482 |       "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
 483 |       "requires": {
 484 |         "asynckit": "^0.4.0",
 485 |         "combined-stream": "^1.0.6",
 486 |         "mime-types": "^2.1.12"
 487 |       }
 488 |     },
 489 |     "fs-constants": {
 490 |       "version": "1.0.0",
 491 |       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
 492 |       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
 493 |     },
 494 |     "fs-minipass": {
 495 |       "version": "1.2.7",
 496 |       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
 497 |       "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
 498 |       "requires": {
 499 |         "minipass": "^2.6.0"
 500 |       }
 501 |     },
 502 |     "fs.realpath": {
 503 |       "version": "1.0.0",
 504 |       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
 505 |       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
 506 |     },
 507 |     "gauge": {
 508 |       "version": "2.7.4",
 509 |       "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
 510 |       "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
 511 |       "requires": {
 512 |         "aproba": "^1.0.3",
 513 |         "console-control-strings": "^1.0.0",
 514 |         "has-unicode": "^2.0.0",
 515 |         "object-assign": "^4.1.0",
 516 |         "signal-exit": "^3.0.0",
 517 |         "string-width": "^1.0.1",
 518 |         "strip-ansi": "^3.0.1",
 519 |         "wide-align": "^1.1.0"
 520 |       }
 521 |     },
 522 |     "get-stream": {
 523 |       "version": "5.1.0",
 524 |       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
 525 |       "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
 526 |       "requires": {
 527 |         "pump": "^3.0.0"
 528 |       }
 529 |     },
 530 |     "getpass": {
 531 |       "version": "0.1.7",
 532 |       "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
 533 |       "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
 534 |       "requires": {
 535 |         "assert-plus": "^1.0.0"
 536 |       }
 537 |     },
 538 |     "github-from-package": {
 539 |       "version": "0.0.0",
 540 |       "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
 541 |       "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4="
 542 |     },
 543 |     "glob": {
 544 |       "version": "7.1.6",
 545 |       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
 546 |       "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
 547 |       "requires": {
 548 |         "fs.realpath": "^1.0.0",
 549 |         "inflight": "^1.0.4",
 550 |         "inherits": "2",
 551 |         "minimatch": "^3.0.4",
 552 |         "once": "^1.3.0",
 553 |         "path-is-absolute": "^1.0.0"
 554 |       }
 555 |     },
 556 |     "har-schema": {
 557 |       "version": "2.0.0",
 558 |       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
 559 |       "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
 560 |     },
 561 |     "har-validator": {
 562 |       "version": "5.1.3",
 563 |       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
 564 |       "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
 565 |       "requires": {
 566 |         "ajv": "^6.5.5",
 567 |         "har-schema": "^2.0.0"
 568 |       }
 569 |     },
 570 |     "has-unicode": {
 571 |       "version": "2.0.1",
 572 |       "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
 573 |       "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
 574 |     },
 575 |     "hawk": {
 576 |       "version": "6.0.2",
 577 |       "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
 578 |       "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==",
 579 |       "requires": {
 580 |         "boom": "4.x.x",
 581 |         "cryptiles": "3.x.x",
 582 |         "hoek": "4.x.x",
 583 |         "sntp": "2.x.x"
 584 |       }
 585 |     },
 586 |     "hoek": {
 587 |       "version": "4.2.1",
 588 |       "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz",
 589 |       "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA=="
 590 |     },
 591 |     "htmlparser2": {
 592 |       "version": "3.10.1",
 593 |       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
 594 |       "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
 595 |       "requires": {
 596 |         "domelementtype": "^1.3.1",
 597 |         "domhandler": "^2.3.0",
 598 |         "domutils": "^1.5.1",
 599 |         "entities": "^1.1.1",
 600 |         "inherits": "^2.0.1",
 601 |         "readable-stream": "^3.1.1"
 602 |       },
 603 |       "dependencies": {
 604 |         "readable-stream": {
 605 |           "version": "3.4.0",
 606 |           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
 607 |           "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
 608 |           "requires": {
 609 |             "inherits": "^2.0.3",
 610 |             "string_decoder": "^1.1.1",
 611 |             "util-deprecate": "^1.0.1"
 612 |           }
 613 |         }
 614 |       }
 615 |     },
 616 |     "http-signature": {
 617 |       "version": "1.2.0",
 618 |       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
 619 |       "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
 620 |       "requires": {
 621 |         "assert-plus": "^1.0.0",
 622 |         "jsprim": "^1.2.2",
 623 |         "sshpk": "^1.7.0"
 624 |       }
 625 |     },
 626 |     "https-proxy-agent": {
 627 |       "version": "4.0.0",
 628 |       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
 629 |       "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
 630 |       "requires": {
 631 |         "agent-base": "5",
 632 |         "debug": "4"
 633 |       },
 634 |       "dependencies": {
 635 |         "debug": {
 636 |           "version": "4.1.1",
 637 |           "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
 638 |           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
 639 |           "requires": {
 640 |             "ms": "^2.1.1"
 641 |           }
 642 |         }
 643 |       }
 644 |     },
 645 |     "iconv-lite": {
 646 |       "version": "0.4.24",
 647 |       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
 648 |       "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
 649 |       "requires": {
 650 |         "safer-buffer": ">= 2.1.2 < 3"
 651 |       }
 652 |     },
 653 |     "ieee754": {
 654 |       "version": "1.1.13",
 655 |       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
 656 |       "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
 657 |     },
 658 |     "ignore-walk": {
 659 |       "version": "3.0.3",
 660 |       "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
 661 |       "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==",
 662 |       "requires": {
 663 |         "minimatch": "^3.0.4"
 664 |       }
 665 |     },
 666 |     "image-size": {
 667 |       "version": "0.8.3",
 668 |       "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.8.3.tgz",
 669 |       "integrity": "sha512-SMtq1AJ+aqHB45c3FsB4ERK0UCiA2d3H1uq8s+8T0Pf8A3W4teyBQyaFaktH6xvZqh+npwlKU7i4fJo0r7TYTg==",
 670 |       "requires": {
 671 |         "queue": "6.0.1"
 672 |       }
 673 |     },
 674 |     "inflight": {
 675 |       "version": "1.0.6",
 676 |       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
 677 |       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
 678 |       "requires": {
 679 |         "once": "^1.3.0",
 680 |         "wrappy": "1"
 681 |       }
 682 |     },
 683 |     "inherits": {
 684 |       "version": "2.0.4",
 685 |       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
 686 |       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 687 |     },
 688 |     "ini": {
 689 |       "version": "1.3.5",
 690 |       "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
 691 |       "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
 692 |     },
 693 |     "int": {
 694 |       "version": "0.2.0",
 695 |       "resolved": "https://registry.npmjs.org/int/-/int-0.2.0.tgz",
 696 |       "integrity": "sha1-WJ8FsDuNjAjJGMiIR4TLYqlO9H4="
 697 |     },
 698 |     "integer": {
 699 |       "version": "3.0.1",
 700 |       "resolved": "https://registry.npmjs.org/integer/-/integer-3.0.1.tgz",
 701 |       "integrity": "sha512-OqtER6W2GIJTIcnT5o2B/pWGgvurnVOYs4OZCgay40QEIbMTnNq4R0KSaIw1TZyFtPWjm5aNM+pBBMTfc3exmw==",
 702 |       "requires": {
 703 |         "bindings": "^1.5.0",
 704 |         "prebuild-install": "^5.3.3"
 705 |       }
 706 |     },
 707 |     "is-buffer": {
 708 |       "version": "1.1.6",
 709 |       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
 710 |       "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
 711 |     },
 712 |     "is-fullwidth-code-point": {
 713 |       "version": "1.0.0",
 714 |       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
 715 |       "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
 716 |       "requires": {
 717 |         "number-is-nan": "^1.0.0"
 718 |       }
 719 |     },
 720 |     "is-typedarray": {
 721 |       "version": "1.0.0",
 722 |       "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
 723 |       "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
 724 |     },
 725 |     "isarray": {
 726 |       "version": "1.0.0",
 727 |       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
 728 |       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
 729 |     },
 730 |     "isstream": {
 731 |       "version": "0.1.2",
 732 |       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
 733 |       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
 734 |     },
 735 |     "jsbn": {
 736 |       "version": "0.1.1",
 737 |       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
 738 |       "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
 739 |     },
 740 |     "json-schema": {
 741 |       "version": "0.2.3",
 742 |       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
 743 |       "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
 744 |     },
 745 |     "json-schema-traverse": {
 746 |       "version": "0.4.1",
 747 |       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
 748 |       "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
 749 |     },
 750 |     "json-stringify-safe": {
 751 |       "version": "5.0.1",
 752 |       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
 753 |       "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
 754 |     },
 755 |     "jsprim": {
 756 |       "version": "1.4.1",
 757 |       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
 758 |       "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
 759 |       "requires": {
 760 |         "assert-plus": "1.0.0",
 761 |         "extsprintf": "1.3.0",
 762 |         "json-schema": "0.2.3",
 763 |         "verror": "1.10.0"
 764 |       }
 765 |     },
 766 |     "keytar": {
 767 |       "version": "5.5.0",
 768 |       "resolved": "https://registry.npmjs.org/keytar/-/keytar-5.5.0.tgz",
 769 |       "integrity": "sha512-1d/F2qAL/qijpm25wNq8eez4mE+/J4eBvqyLfspIYIeCEHn3nttFwplowIZfbdNCjFGWh68MzGZOd2vS61Ffew==",
 770 |       "optional": true,
 771 |       "requires": {
 772 |         "nan": "2.14.0",
 773 |         "prebuild-install": "5.3.3"
 774 |       }
 775 |     },
 776 |     "lodash": {
 777 |       "version": "4.17.15",
 778 |       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
 779 |       "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
 780 |     },
 781 |     "md5": {
 782 |       "version": "2.2.1",
 783 |       "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz",
 784 |       "integrity": "sha1-U6s41f48iJG6RlMp6iP6wFQBJvk=",
 785 |       "requires": {
 786 |         "charenc": "~0.0.1",
 787 |         "crypt": "~0.0.1",
 788 |         "is-buffer": "~1.1.1"
 789 |       }
 790 |     },
 791 |     "mime": {
 792 |       "version": "2.4.4",
 793 |       "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
 794 |       "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
 795 |     },
 796 |     "mime-db": {
 797 |       "version": "1.42.0",
 798 |       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.42.0.tgz",
 799 |       "integrity": "sha512-UbfJCR4UAVRNgMpfImz05smAXK7+c+ZntjaA26ANtkXLlOe947Aag5zdIcKQULAiF9Cq4WxBi9jUs5zkA84bYQ=="
 800 |     },
 801 |     "mime-types": {
 802 |       "version": "2.1.25",
 803 |       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.25.tgz",
 804 |       "integrity": "sha512-5KhStqB5xpTAeGqKBAMgwaYMnQik7teQN4IAzC7npDv6kzeU6prfkR67bc87J1kWMPGkoaZSq1npmexMgkmEVg==",
 805 |       "requires": {
 806 |         "mime-db": "1.42.0"
 807 |       }
 808 |     },
 809 |     "mimic-response": {
 810 |       "version": "2.1.0",
 811 |       "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
 812 |       "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
 813 |     },
 814 |     "minimatch": {
 815 |       "version": "3.0.4",
 816 |       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
 817 |       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
 818 |       "requires": {
 819 |         "brace-expansion": "^1.1.7"
 820 |       }
 821 |     },
 822 |     "minimist": {
 823 |       "version": "1.2.5",
 824 |       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
 825 |       "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
 826 |     },
 827 |     "minipass": {
 828 |       "version": "2.9.0",
 829 |       "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
 830 |       "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
 831 |       "requires": {
 832 |         "safe-buffer": "^5.1.2",
 833 |         "yallist": "^3.0.0"
 834 |       }
 835 |     },
 836 |     "minizlib": {
 837 |       "version": "1.3.3",
 838 |       "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
 839 |       "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
 840 |       "requires": {
 841 |         "minipass": "^2.9.0"
 842 |       }
 843 |     },
 844 |     "mkdirp": {
 845 |       "version": "1.0.4",
 846 |       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
 847 |       "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
 848 |     },
 849 |     "mkdirp-classic": {
 850 |       "version": "0.5.2",
 851 |       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.2.tgz",
 852 |       "integrity": "sha512-ejdnDQcR75gwknmMw/tx02AuRs8jCtqFoFqDZMjiNxsu85sRIJVXDKHuLYvUUPRBUtV2FpSZa9bL1BUa3BdR2g=="
 853 |     },
 854 |     "ms": {
 855 |       "version": "2.1.2",
 856 |       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 857 |       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 858 |     },
 859 |     "nan": {
 860 |       "version": "2.14.0",
 861 |       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
 862 |       "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg=="
 863 |     },
 864 |     "napi-build-utils": {
 865 |       "version": "1.0.2",
 866 |       "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
 867 |       "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
 868 |     },
 869 |     "needle": {
 870 |       "version": "2.4.1",
 871 |       "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.1.tgz",
 872 |       "integrity": "sha512-x/gi6ijr4B7fwl6WYL9FwlCvRQKGlUNvnceho8wxkwXqN8jvVmmmATTmZPRRG7b/yC1eode26C2HO9jl78Du9g==",
 873 |       "requires": {
 874 |         "debug": "^3.2.6",
 875 |         "iconv-lite": "^0.4.4",
 876 |         "sax": "^1.2.4"
 877 |       }
 878 |     },
 879 |     "node-abi": {
 880 |       "version": "2.16.0",
 881 |       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.16.0.tgz",
 882 |       "integrity": "sha512-+sa0XNlWDA6T+bDLmkCUYn6W5k5W6BPRL6mqzSCs6H/xUgtl4D5x2fORKDzopKiU6wsyn/+wXlRXwXeSp+mtoA==",
 883 |       "requires": {
 884 |         "semver": "^5.4.1"
 885 |       }
 886 |     },
 887 |     "node-pre-gyp": {
 888 |       "version": "0.11.0",
 889 |       "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
 890 |       "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
 891 |       "requires": {
 892 |         "detect-libc": "^1.0.2",
 893 |         "mkdirp": "^0.5.1",
 894 |         "needle": "^2.2.1",
 895 |         "nopt": "^4.0.1",
 896 |         "npm-packlist": "^1.1.6",
 897 |         "npmlog": "^4.0.2",
 898 |         "rc": "^1.2.7",
 899 |         "rimraf": "^2.6.1",
 900 |         "semver": "^5.3.0",
 901 |         "tar": "^4"
 902 |       },
 903 |       "dependencies": {
 904 |         "mkdirp": {
 905 |           "version": "0.5.5",
 906 |           "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
 907 |           "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
 908 |           "requires": {
 909 |             "minimist": "^1.2.5"
 910 |           }
 911 |         }
 912 |       }
 913 |     },
 914 |     "node-wget": {
 915 |       "version": "0.4.3",
 916 |       "resolved": "https://registry.npmjs.org/node-wget/-/node-wget-0.4.3.tgz",
 917 |       "integrity": "sha512-sltt/nc4NJeI4CuncyBFZZ7W4Wz3Q1oM8h27mYEbj5OihXMsuqJpplPl0WUZTSpXy8AyGczT08jIBsXHxdCtgg==",
 918 |       "requires": {
 919 |         "request": "~2.85.0"
 920 |       },
 921 |       "dependencies": {
 922 |         "ajv": {
 923 |           "version": "5.5.2",
 924 |           "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
 925 |           "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
 926 |           "requires": {
 927 |             "co": "^4.6.0",
 928 |             "fast-deep-equal": "^1.0.0",
 929 |             "fast-json-stable-stringify": "^2.0.0",
 930 |             "json-schema-traverse": "^0.3.0"
 931 |           }
 932 |         },
 933 |         "fast-deep-equal": {
 934 |           "version": "1.1.0",
 935 |           "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
 936 |           "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
 937 |         },
 938 |         "har-validator": {
 939 |           "version": "5.0.3",
 940 |           "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
 941 |           "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
 942 |           "requires": {
 943 |             "ajv": "^5.1.0",
 944 |             "har-schema": "^2.0.0"
 945 |           }
 946 |         },
 947 |         "json-schema-traverse": {
 948 |           "version": "0.3.1",
 949 |           "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
 950 |           "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
 951 |         },
 952 |         "oauth-sign": {
 953 |           "version": "0.8.2",
 954 |           "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
 955 |           "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
 956 |         },
 957 |         "punycode": {
 958 |           "version": "1.4.1",
 959 |           "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
 960 |           "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
 961 |         },
 962 |         "request": {
 963 |           "version": "2.85.0",
 964 |           "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz",
 965 |           "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==",
 966 |           "requires": {
 967 |             "aws-sign2": "~0.7.0",
 968 |             "aws4": "^1.6.0",
 969 |             "caseless": "~0.12.0",
 970 |             "combined-stream": "~1.0.5",
 971 |             "extend": "~3.0.1",
 972 |             "forever-agent": "~0.6.1",
 973 |             "form-data": "~2.3.1",
 974 |             "har-validator": "~5.0.3",
 975 |             "hawk": "~6.0.2",
 976 |             "http-signature": "~1.2.0",
 977 |             "is-typedarray": "~1.0.0",
 978 |             "isstream": "~0.1.2",
 979 |             "json-stringify-safe": "~5.0.1",
 980 |             "mime-types": "~2.1.17",
 981 |             "oauth-sign": "~0.8.2",
 982 |             "performance-now": "^2.1.0",
 983 |             "qs": "~6.5.1",
 984 |             "safe-buffer": "^5.1.1",
 985 |             "stringstream": "~0.0.5",
 986 |             "tough-cookie": "~2.3.3",
 987 |             "tunnel-agent": "^0.6.0",
 988 |             "uuid": "^3.1.0"
 989 |           }
 990 |         },
 991 |         "tough-cookie": {
 992 |           "version": "2.3.4",
 993 |           "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",
 994 |           "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==",
 995 |           "requires": {
 996 |             "punycode": "^1.4.1"
 997 |           }
 998 |         }
 999 |       }
1000 |     },
1001 |     "noop-logger": {
1002 |       "version": "0.1.1",
1003 |       "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
1004 |       "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI="
1005 |     },
1006 |     "nopt": {
1007 |       "version": "4.0.3",
1008 |       "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
1009 |       "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
1010 |       "requires": {
1011 |         "abbrev": "1",
1012 |         "osenv": "^0.1.4"
1013 |       }
1014 |     },
1015 |     "npm-bundled": {
1016 |       "version": "1.1.1",
1017 |       "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz",
1018 |       "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==",
1019 |       "requires": {
1020 |         "npm-normalize-package-bin": "^1.0.1"
1021 |       }
1022 |     },
1023 |     "npm-normalize-package-bin": {
1024 |       "version": "1.0.1",
1025 |       "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz",
1026 |       "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA=="
1027 |     },
1028 |     "npm-packlist": {
1029 |       "version": "1.4.8",
1030 |       "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz",
1031 |       "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==",
1032 |       "requires": {
1033 |         "ignore-walk": "^3.0.1",
1034 |         "npm-bundled": "^1.0.1",
1035 |         "npm-normalize-package-bin": "^1.0.1"
1036 |       }
1037 |     },
1038 |     "npmlog": {
1039 |       "version": "4.1.2",
1040 |       "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
1041 |       "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
1042 |       "requires": {
1043 |         "are-we-there-yet": "~1.1.2",
1044 |         "console-control-strings": "~1.1.0",
1045 |         "gauge": "~2.7.3",
1046 |         "set-blocking": "~2.0.0"
1047 |       }
1048 |     },
1049 |     "nth-check": {
1050 |       "version": "1.0.2",
1051 |       "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
1052 |       "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
1053 |       "requires": {
1054 |         "boolbase": "~1.0.0"
1055 |       }
1056 |     },
1057 |     "number-is-nan": {
1058 |       "version": "1.0.1",
1059 |       "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
1060 |       "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
1061 |     },
1062 |     "oauth-sign": {
1063 |       "version": "0.9.0",
1064 |       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
1065 |       "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
1066 |     },
1067 |     "object-assign": {
1068 |       "version": "4.1.1",
1069 |       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
1070 |       "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
1071 |     },
1072 |     "once": {
1073 |       "version": "1.4.0",
1074 |       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
1075 |       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
1076 |       "requires": {
1077 |         "wrappy": "1"
1078 |       }
1079 |     },
1080 |     "os-homedir": {
1081 |       "version": "1.0.2",
1082 |       "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
1083 |       "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
1084 |     },
1085 |     "os-tmpdir": {
1086 |       "version": "1.0.2",
1087 |       "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
1088 |       "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
1089 |     },
1090 |     "osenv": {
1091 |       "version": "0.1.5",
1092 |       "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
1093 |       "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
1094 |       "requires": {
1095 |         "os-homedir": "^1.0.0",
1096 |         "os-tmpdir": "^1.0.0"
1097 |       }
1098 |     },
1099 |     "parse5": {
1100 |       "version": "3.0.3",
1101 |       "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
1102 |       "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
1103 |       "requires": {
1104 |         "@types/node": "*"
1105 |       }
1106 |     },
1107 |     "path-is-absolute": {
1108 |       "version": "1.0.1",
1109 |       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
1110 |       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
1111 |     },
1112 |     "pend": {
1113 |       "version": "1.2.0",
1114 |       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
1115 |       "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
1116 |     },
1117 |     "performance-now": {
1118 |       "version": "2.1.0",
1119 |       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
1120 |       "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
1121 |     },
1122 |     "prebuild-install": {
1123 |       "version": "5.3.3",
1124 |       "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.3.tgz",
1125 |       "integrity": "sha512-GV+nsUXuPW2p8Zy7SarF/2W/oiK8bFQgJcncoJ0d7kRpekEA0ftChjfEaF9/Y+QJEc/wFR7RAEa8lYByuUIe2g==",
1126 |       "requires": {
1127 |         "detect-libc": "^1.0.3",
1128 |         "expand-template": "^2.0.3",
1129 |         "github-from-package": "0.0.0",
1130 |         "minimist": "^1.2.0",
1131 |         "mkdirp": "^0.5.1",
1132 |         "napi-build-utils": "^1.0.1",
1133 |         "node-abi": "^2.7.0",
1134 |         "noop-logger": "^0.1.1",
1135 |         "npmlog": "^4.0.1",
1136 |         "pump": "^3.0.0",
1137 |         "rc": "^1.2.7",
1138 |         "simple-get": "^3.0.3",
1139 |         "tar-fs": "^2.0.0",
1140 |         "tunnel-agent": "^0.6.0",
1141 |         "which-pm-runs": "^1.0.0"
1142 |       },
1143 |       "dependencies": {
1144 |         "mkdirp": {
1145 |           "version": "0.5.5",
1146 |           "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
1147 |           "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
1148 |           "requires": {
1149 |             "minimist": "^1.2.5"
1150 |           }
1151 |         }
1152 |       }
1153 |     },
1154 |     "process-nextick-args": {
1155 |       "version": "2.0.1",
1156 |       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
1157 |       "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
1158 |     },
1159 |     "progress": {
1160 |       "version": "2.0.3",
1161 |       "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
1162 |       "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
1163 |     },
1164 |     "proxy-from-env": {
1165 |       "version": "1.1.0",
1166 |       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
1167 |       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
1168 |     },
1169 |     "psl": {
1170 |       "version": "1.8.0",
1171 |       "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
1172 |       "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
1173 |     },
1174 |     "pump": {
1175 |       "version": "3.0.0",
1176 |       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
1177 |       "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
1178 |       "requires": {
1179 |         "end-of-stream": "^1.1.0",
1180 |         "once": "^1.3.1"
1181 |       }
1182 |     },
1183 |     "punycode": {
1184 |       "version": "2.1.1",
1185 |       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
1186 |       "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
1187 |     },
1188 |     "puppeteer": {
1189 |       "version": "3.0.1",
1190 |       "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-3.0.1.tgz",
1191 |       "integrity": "sha512-DxNnI9n4grVHC+9irUfNK2T6YFuRECJnvG7VzdVolxpVwWC5DQqI5ho9Z0af48K5MQW4sJY5cq3qQ5g6NkAjvw==",
1192 |       "requires": {
1193 |         "@types/mime-types": "^2.1.0",
1194 |         "debug": "^4.1.0",
1195 |         "extract-zip": "^2.0.0",
1196 |         "https-proxy-agent": "^4.0.0",
1197 |         "mime": "^2.0.3",
1198 |         "mime-types": "^2.1.25",
1199 |         "progress": "^2.0.1",
1200 |         "proxy-from-env": "^1.0.0",
1201 |         "rimraf": "^3.0.2",
1202 |         "tar-fs": "^2.0.0",
1203 |         "unbzip2-stream": "^1.3.3",
1204 |         "ws": "^7.2.3"
1205 |       },
1206 |       "dependencies": {
1207 |         "debug": {
1208 |           "version": "4.1.1",
1209 |           "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
1210 |           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
1211 |           "requires": {
1212 |             "ms": "^2.1.1"
1213 |           }
1214 |         },
1215 |         "rimraf": {
1216 |           "version": "3.0.2",
1217 |           "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
1218 |           "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
1219 |           "requires": {
1220 |             "glob": "^7.1.3"
1221 |           }
1222 |         }
1223 |       }
1224 |     },
1225 |     "qs": {
1226 |       "version": "6.5.2",
1227 |       "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
1228 |       "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
1229 |     },
1230 |     "queue": {
1231 |       "version": "6.0.1",
1232 |       "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.1.tgz",
1233 |       "integrity": "sha512-AJBQabRCCNr9ANq8v77RJEv73DPbn55cdTb+Giq4X0AVnNVZvMHlYp7XlQiN+1npCZj1DuSmaA2hYVUUDgxFDg==",
1234 |       "requires": {
1235 |         "inherits": "~2.0.3"
1236 |       }
1237 |     },
1238 |     "rc": {
1239 |       "version": "1.2.8",
1240 |       "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
1241 |       "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
1242 |       "requires": {
1243 |         "deep-extend": "^0.6.0",
1244 |         "ini": "~1.3.0",
1245 |         "minimist": "^1.2.0",
1246 |         "strip-json-comments": "~2.0.1"
1247 |       }
1248 |     },
1249 |     "readable-stream": {
1250 |       "version": "2.3.7",
1251 |       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
1252 |       "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
1253 |       "requires": {
1254 |         "core-util-is": "~1.0.0",
1255 |         "inherits": "~2.0.3",
1256 |         "isarray": "~1.0.0",
1257 |         "process-nextick-args": "~2.0.0",
1258 |         "safe-buffer": "~5.1.1",
1259 |         "string_decoder": "~1.1.1",
1260 |         "util-deprecate": "~1.0.1"
1261 |       }
1262 |     },
1263 |     "request": {
1264 |       "version": "2.88.2",
1265 |       "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
1266 |       "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
1267 |       "requires": {
1268 |         "aws-sign2": "~0.7.0",
1269 |         "aws4": "^1.8.0",
1270 |         "caseless": "~0.12.0",
1271 |         "combined-stream": "~1.0.6",
1272 |         "extend": "~3.0.2",
1273 |         "forever-agent": "~0.6.1",
1274 |         "form-data": "~2.3.2",
1275 |         "har-validator": "~5.1.3",
1276 |         "http-signature": "~1.2.0",
1277 |         "is-typedarray": "~1.0.0",
1278 |         "isstream": "~0.1.2",
1279 |         "json-stringify-safe": "~5.0.1",
1280 |         "mime-types": "~2.1.19",
1281 |         "oauth-sign": "~0.9.0",
1282 |         "performance-now": "^2.1.0",
1283 |         "qs": "~6.5.2",
1284 |         "safe-buffer": "^5.1.2",
1285 |         "tough-cookie": "~2.5.0",
1286 |         "tunnel-agent": "^0.6.0",
1287 |         "uuid": "^3.3.2"
1288 |       }
1289 |     },
1290 |     "rimraf": {
1291 |       "version": "2.7.1",
1292 |       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
1293 |       "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
1294 |       "requires": {
1295 |         "glob": "^7.1.3"
1296 |       }
1297 |     },
1298 |     "safe-buffer": {
1299 |       "version": "5.1.2",
1300 |       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
1301 |       "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
1302 |     },
1303 |     "safer-buffer": {
1304 |       "version": "2.1.2",
1305 |       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
1306 |       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
1307 |     },
1308 |     "sax": {
1309 |       "version": "1.2.4",
1310 |       "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
1311 |       "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
1312 |     },
1313 |     "semver": {
1314 |       "version": "5.7.1",
1315 |       "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
1316 |       "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
1317 |     },
1318 |     "set-blocking": {
1319 |       "version": "2.0.0",
1320 |       "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
1321 |       "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
1322 |     },
1323 |     "signal-exit": {
1324 |       "version": "3.0.3",
1325 |       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
1326 |       "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
1327 |     },
1328 |     "simple-concat": {
1329 |       "version": "1.0.0",
1330 |       "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
1331 |       "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY="
1332 |     },
1333 |     "simple-get": {
1334 |       "version": "3.1.0",
1335 |       "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
1336 |       "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
1337 |       "requires": {
1338 |         "decompress-response": "^4.2.0",
1339 |         "once": "^1.3.1",
1340 |         "simple-concat": "^1.0.0"
1341 |       }
1342 |     },
1343 |     "sntp": {
1344 |       "version": "2.1.0",
1345 |       "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz",
1346 |       "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==",
1347 |       "requires": {
1348 |         "hoek": "4.x.x"
1349 |       }
1350 |     },
1351 |     "sqlite3": {
1352 |       "version": "4.1.1",
1353 |       "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.1.1.tgz",
1354 |       "integrity": "sha512-CvT5XY+MWnn0HkbwVKJAyWEMfzpAPwnTiB3TobA5Mri44SrTovmmh499NPQP+gatkeOipqPlBLel7rn4E/PCQg==",
1355 |       "requires": {
1356 |         "nan": "^2.12.1",
1357 |         "node-pre-gyp": "^0.11.0",
1358 |         "request": "^2.87.0"
1359 |       }
1360 |     },
1361 |     "sshpk": {
1362 |       "version": "1.16.1",
1363 |       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
1364 |       "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
1365 |       "requires": {
1366 |         "asn1": "~0.2.3",
1367 |         "assert-plus": "^1.0.0",
1368 |         "bcrypt-pbkdf": "^1.0.0",
1369 |         "dashdash": "^1.12.0",
1370 |         "ecc-jsbn": "~0.1.1",
1371 |         "getpass": "^0.1.1",
1372 |         "jsbn": "~0.1.0",
1373 |         "safer-buffer": "^2.0.2",
1374 |         "tweetnacl": "~0.14.0"
1375 |       }
1376 |     },
1377 |     "string-width": {
1378 |       "version": "1.0.2",
1379 |       "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
1380 |       "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
1381 |       "requires": {
1382 |         "code-point-at": "^1.0.0",
1383 |         "is-fullwidth-code-point": "^1.0.0",
1384 |         "strip-ansi": "^3.0.0"
1385 |       }
1386 |     },
1387 |     "string_decoder": {
1388 |       "version": "1.1.1",
1389 |       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
1390 |       "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
1391 |       "requires": {
1392 |         "safe-buffer": "~5.1.0"
1393 |       }
1394 |     },
1395 |     "stringstream": {
1396 |       "version": "0.0.6",
1397 |       "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.6.tgz",
1398 |       "integrity": "sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA=="
1399 |     },
1400 |     "strip-ansi": {
1401 |       "version": "3.0.1",
1402 |       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
1403 |       "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
1404 |       "requires": {
1405 |         "ansi-regex": "^2.0.0"
1406 |       }
1407 |     },
1408 |     "strip-json-comments": {
1409 |       "version": "2.0.1",
1410 |       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
1411 |       "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
1412 |     },
1413 |     "tar": {
1414 |       "version": "4.4.10",
1415 |       "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz",
1416 |       "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==",
1417 |       "requires": {
1418 |         "chownr": "^1.1.1",
1419 |         "fs-minipass": "^1.2.5",
1420 |         "minipass": "^2.3.5",
1421 |         "minizlib": "^1.2.1",
1422 |         "mkdirp": "^0.5.0",
1423 |         "safe-buffer": "^5.1.2",
1424 |         "yallist": "^3.0.3"
1425 |       },
1426 |       "dependencies": {
1427 |         "mkdirp": {
1428 |           "version": "0.5.5",
1429 |           "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
1430 |           "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
1431 |           "requires": {
1432 |             "minimist": "^1.2.5"
1433 |           }
1434 |         }
1435 |       }
1436 |     },
1437 |     "tar-fs": {
1438 |       "version": "2.0.1",
1439 |       "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz",
1440 |       "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==",
1441 |       "requires": {
1442 |         "chownr": "^1.1.1",
1443 |         "mkdirp-classic": "^0.5.2",
1444 |         "pump": "^3.0.0",
1445 |         "tar-stream": "^2.0.0"
1446 |       }
1447 |     },
1448 |     "tar-stream": {
1449 |       "version": "2.1.2",
1450 |       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz",
1451 |       "integrity": "sha512-UaF6FoJ32WqALZGOIAApXx+OdxhekNMChu6axLJR85zMMjXKWFGjbIRe+J6P4UnRGg9rAwWvbTT0oI7hD/Un7Q==",
1452 |       "requires": {
1453 |         "bl": "^4.0.1",
1454 |         "end-of-stream": "^1.4.1",
1455 |         "fs-constants": "^1.0.0",
1456 |         "inherits": "^2.0.3",
1457 |         "readable-stream": "^3.1.1"
1458 |       },
1459 |       "dependencies": {
1460 |         "readable-stream": {
1461 |           "version": "3.6.0",
1462 |           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
1463 |           "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
1464 |           "requires": {
1465 |             "inherits": "^2.0.3",
1466 |             "string_decoder": "^1.1.1",
1467 |             "util-deprecate": "^1.0.1"
1468 |           }
1469 |         }
1470 |       }
1471 |     },
1472 |     "through": {
1473 |       "version": "2.3.8",
1474 |       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
1475 |       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
1476 |     },
1477 |     "tldjs": {
1478 |       "version": "1.8.0",
1479 |       "resolved": "https://registry.npmjs.org/tldjs/-/tldjs-1.8.0.tgz",
1480 |       "integrity": "sha1-ucFr1t41e1X/y+fVvkH2s8p24/o=",
1481 |       "requires": {
1482 |         "punycode": "^1.4.1"
1483 |       },
1484 |       "dependencies": {
1485 |         "punycode": {
1486 |           "version": "1.4.1",
1487 |           "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
1488 |           "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
1489 |         }
1490 |       }
1491 |     },
1492 |     "tmp": {
1493 |       "version": "0.2.0",
1494 |       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.0.tgz",
1495 |       "integrity": "sha512-spsb5g6EiPmteS5TcOAECU3rltCMDMp4VMU2Sb0+WttN4qGobEkMAd+dkr1cubscN08JGNDX765dPbGImbG7MQ==",
1496 |       "requires": {
1497 |         "rimraf": "^3.0.0"
1498 |       },
1499 |       "dependencies": {
1500 |         "rimraf": {
1501 |           "version": "3.0.2",
1502 |           "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
1503 |           "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
1504 |           "requires": {
1505 |             "glob": "^7.1.3"
1506 |           }
1507 |         }
1508 |       }
1509 |     },
1510 |     "tough-cookie": {
1511 |       "version": "2.5.0",
1512 |       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
1513 |       "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
1514 |       "requires": {
1515 |         "psl": "^1.1.28",
1516 |         "punycode": "^2.1.1"
1517 |       }
1518 |     },
1519 |     "tunnel-agent": {
1520 |       "version": "0.6.0",
1521 |       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
1522 |       "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
1523 |       "requires": {
1524 |         "safe-buffer": "^5.0.1"
1525 |       }
1526 |     },
1527 |     "tweetnacl": {
1528 |       "version": "0.14.5",
1529 |       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
1530 |       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
1531 |     },
1532 |     "unbzip2-stream": {
1533 |       "version": "1.4.2",
1534 |       "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.2.tgz",
1535 |       "integrity": "sha512-pZMVAofMrrHX6Ik39hCk470kulCbmZ2SWfQLPmTWqfJV/oUm0gn1CblvHdUu4+54Je6Jq34x8kY6XjTy6dMkOg==",
1536 |       "requires": {
1537 |         "buffer": "^5.2.1",
1538 |         "through": "^2.3.8"
1539 |       }
1540 |     },
1541 |     "uri-js": {
1542 |       "version": "4.2.2",
1543 |       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
1544 |       "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
1545 |       "requires": {
1546 |         "punycode": "^2.1.0"
1547 |       }
1548 |     },
1549 |     "url-status-code": {
1550 |       "version": "2.0.0",
1551 |       "resolved": "https://registry.npmjs.org/url-status-code/-/url-status-code-2.0.0.tgz",
1552 |       "integrity": "sha512-ODpSV5o2OM/tldo6sq5BGl6ILQ0htv+1LSv66qQI94vG2quKMpfxYp2XDGSI0sR9CXZaLJpQ74xMGLiIkQXJzQ==",
1553 |       "requires": {
1554 |         "valid-url": "1.0.9"
1555 |       }
1556 |     },
1557 |     "util-deprecate": {
1558 |       "version": "1.0.2",
1559 |       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
1560 |       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
1561 |     },
1562 |     "uuid": {
1563 |       "version": "3.3.3",
1564 |       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz",
1565 |       "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ=="
1566 |     },
1567 |     "valid-url": {
1568 |       "version": "1.0.9",
1569 |       "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz",
1570 |       "integrity": "sha1-HBRHm0DxOXp1eC8RXkCGRHQzogA="
1571 |     },
1572 |     "verror": {
1573 |       "version": "1.10.0",
1574 |       "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
1575 |       "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
1576 |       "requires": {
1577 |         "assert-plus": "^1.0.0",
1578 |         "core-util-is": "1.0.2",
1579 |         "extsprintf": "^1.2.0"
1580 |       }
1581 |     },
1582 |     "which-pm-runs": {
1583 |       "version": "1.0.0",
1584 |       "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
1585 |       "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs="
1586 |     },
1587 |     "wide-align": {
1588 |       "version": "1.1.3",
1589 |       "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
1590 |       "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
1591 |       "requires": {
1592 |         "string-width": "^1.0.2 || 2"
1593 |       }
1594 |     },
1595 |     "win-dpapi": {
1596 |       "version": "0.1.0",
1597 |       "resolved": "https://registry.npmjs.org/win-dpapi/-/win-dpapi-0.1.0.tgz",
1598 |       "integrity": "sha512-sQ1GFRMWd4OAOUBaEMRoQvHHMkaVb0Ib5MhRc4JPWBDytnAiXJhEZTIRt5+D3zPrBmjLyErhrHflmmty2OtxPw==",
1599 |       "optional": true,
1600 |       "requires": {
1601 |         "bindings": "^1.5.0",
1602 |         "nan": "^2.13.2"
1603 |       }
1604 |     },
1605 |     "wrappy": {
1606 |       "version": "1.0.2",
1607 |       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
1608 |       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
1609 |     },
1610 |     "ws": {
1611 |       "version": "7.2.5",
1612 |       "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.5.tgz",
1613 |       "integrity": "sha512-C34cIU4+DB2vMyAbmEKossWq2ZQDr6QEyuuCzWrM9zfw1sGc0mYiJ0UnG9zzNykt49C2Fi34hvr2vssFQRS6EA=="
1614 |     },
1615 |     "yallist": {
1616 |       "version": "3.1.1",
1617 |       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
1618 |       "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
1619 |     },
1620 |     "yauzl": {
1621 |       "version": "2.10.0",
1622 |       "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
1623 |       "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
1624 |       "requires": {
1625 |         "buffer-crc32": "~0.2.3",
1626 |         "fd-slicer": "~1.1.0"
1627 |       }
1628 |     }
1629 |   }
1630 | }
1631 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "archivist-pinterest-crawl",
 3 |   "version": "1.3.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "keywords": [],
 7 |   "author": "Szymon Kaliski <hi@szymonkaliski.com> (http://szymonkaliski.com)",
 8 |   "license": "MIT",
 9 |   "dependencies": {
10 |     "async": "^3.2.0",
11 |     "better-sqlite3": "^6.0.1",
12 |     "cheerio": "^1.0.0-rc.3",
13 |     "chrome-cookies-secure": "^1.3.2",
14 |     "dateformat": "^3.0.3",
15 |     "env-paths": "^2.2.0",
16 |     "image-size": "^0.8.3",
17 |     "md5": "^2.2.1",
18 |     "mkdirp": "^1.0.4",
19 |     "node-wget": "^0.4.3",
20 |     "puppeteer": "^3.0.1",
21 |     "tmp": "^0.2.0",
22 |     "url-status-code": "^2.0.0"
23 |   }
24 | }
25 | 


--------------------------------------------------------------------------------
/archivist-pinterest-crawl/query/index.js:
--------------------------------------------------------------------------------
 1 | const envPaths = require("env-paths");
 2 | const Database = require("better-sqlite3");
 3 | const path = require("path");
 4 | 
 5 | const DATA_PATH = envPaths("archivist-pinterest").data;
 6 | const ASSETS_PATH = path.join(DATA_PATH, "assets");
 7 | 
 8 | const query = async (_, text) => {
 9 |   const db = new Database(path.join(DATA_PATH, "data.db"));
10 |   let search;
11 | 
12 |   if (text) {
13 |     search = db
14 |       .prepare(
15 |         `
16 |         SELECT *
17 |         FROM data
18 |         WHERE
19 |           board LIKE :query OR
20 |           text LIKE :query OR
21 |           title LIKE :query OR
22 |           link LIKE :query
23 |         `
24 |       )
25 |       .all({ query: `%${text}%` });
26 |   } else {
27 |     search = db.prepare("SELECT * FROM data").all();
28 |   }
29 | 
30 |   return search.map(d => ({
31 |     img: path.join(ASSETS_PATH, d.filename),
32 |     link: d.link,
33 |     id: d.pinid,
34 |     time: d.createdat || d.crawldate,
35 | 
36 |     width: d.width,
37 |     height: d.height,
38 | 
39 |     meta: {
40 |       source: "pinterest",
41 |       title: d.title,
42 |       note: d.text,
43 |       tags: [d.board]
44 |     }
45 |   }));
46 | };
47 | 
48 | module.exports = query;
49 | 


--------------------------------------------------------------------------------
/archivist-ui/.gitignore:
--------------------------------------------------------------------------------
1 | dist/
2 | node_modules/
3 | .DS_Store
4 | 


--------------------------------------------------------------------------------
/archivist-ui/README.md:
--------------------------------------------------------------------------------
1 | # archivist-ui
2 | 
3 | UI for [Archivist](../).
4 | 
5 | ## Credits
6 | 
7 | Icon by [ovtaviotti](https://www.deviantart.com/octaviotti/art/HiddenMe-789204984)
8 | 


--------------------------------------------------------------------------------
/archivist-ui/assets/Archivist.icns:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/szymonkaliski/archivist/2061bc231b03437313f9f2eb41fdaec66aeebbe9/archivist-ui/assets/Archivist.icns


--------------------------------------------------------------------------------
/archivist-ui/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "archivist-ui",
 3 |   "version": "1.1.0",
 4 |   "description": "ui for archivist-cli",
 5 |   "scripts": {
 6 |     "dev": "electron-webpack dev",
 7 |     "compile": "electron-webpack",
 8 |     "dist": "npm run compile && electron-builder"
 9 |   },
10 |   "build": {
11 |     "appId": "com.szymonkaliski.archivist-ui",
12 |     "productName": "Archivist",
13 |     "mac": {
14 |       "icon": "./assets/Archivist.icns"
15 |     }
16 |   },
17 |   "electronWebpack": {
18 |     "title": "Archivist",
19 |     "whiteListedModules": [
20 |       "use-debounce",
21 |       "react-hotkeys-hook"
22 |     ]
23 |   },
24 |   "keywords": [],
25 |   "author": "Szymon Kaliski <hi@szymonkaliski.com> (http://szymonkaliski.com)",
26 |   "license": "MIT",
27 |   "devDependencies": {
28 |     "@babel/preset-react": "^7.9.4",
29 |     "css-loader": "^3.5.3",
30 |     "electron": "^8.2.3",
31 |     "electron-builder": "^22.5.1",
32 |     "electron-webpack": "^2.8.2",
33 |     "webpack": "^4.43.0"
34 |   },
35 |   "dependencies": {
36 |     "dateformat": "^3.0.3",
37 |     "fuse.js": "^5.2.3",
38 |     "immer": "^6.0.3",
39 |     "lodash": "^4.17.15",
40 |     "react": "16.13.1",
41 |     "react-dom": "16.13.1",
42 |     "react-hotkeys-hook": "^2.1.3",
43 |     "react-virtualized": "^9.21.2",
44 |     "source-map-support": "^0.5.19",
45 |     "strip": "^3.0.0",
46 |     "tachyons": "^4.11.1",
47 |     "use-debounce": "^3.4.2"
48 |   }
49 | }
50 | 


--------------------------------------------------------------------------------
/archivist-ui/scripts/install.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 4 | cd "$DIR/.." || exit 1
 5 | 
 6 | npm run dist
 7 | rm -rf /Applications/Archivist.app
 8 | mv ./dist/mac/Archivist.app /Applications
 9 | rm -rf ./dist
10 | 


--------------------------------------------------------------------------------
/archivist-ui/src/main/index.js:
--------------------------------------------------------------------------------
 1 | const path = require("path");
 2 | const { app, BrowserWindow } = require("electron");
 3 | const { format } = require("url");
 4 | 
 5 | const IS_DEV = process.env.NODE_ENV !== "production";
 6 | 
 7 | let mainWindow;
 8 | 
 9 | const createWindow = () => {
10 |   const window = new BrowserWindow({
11 |     webPreferences: {
12 |       webSecurity: false, // otherwise we can't load images using file:/// url
13 |       nodeIntegration: true
14 |     }
15 |   });
16 | 
17 |   if (IS_DEV) {
18 |     window.webContents.openDevTools();
19 |     window.loadURL(`http://localhost:${process.env.ELECTRON_WEBPACK_WDS_PORT}`);
20 |   } else {
21 |     window.loadURL(
22 |       format({
23 |         pathname: path.join(__dirname, "index.html"),
24 |         protocol: "file",
25 |         slashes: true
26 |       })
27 |     );
28 |   }
29 | 
30 |   window.on("closed", () => {
31 |     mainWindow = null;
32 |   });
33 | 
34 |   return window;
35 | };
36 | 
37 | app.on("window-all-closed", () => {
38 |   if (process.platform !== "darwin") {
39 |     app.quit();
40 |   }
41 | });
42 | 
43 | app.on("activate", () => {
44 |   if (mainWindow === null) {
45 |     mainWindow = createWindow();
46 |   }
47 | });
48 | 
49 | app.on("ready", () => {
50 |   mainWindow = createWindow();
51 | });
52 | 


--------------------------------------------------------------------------------
/archivist-ui/src/renderer/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <meta charset="utf-8" />
 5 |     <meta name="viewport" content="width=device-width" />
 6 | 
 7 |     <title>Archivist</title>
 8 | 
 9 |     <script>
10 |       require("source-map-support/source-map-support.js").install();
11 |     </script>
12 |   </head>
13 |   <body>
14 |     <div id="app"></div>
15 |     <script>
16 |       require("./index");
17 |     </script>
18 |   </body>
19 | </html>
20 | 


--------------------------------------------------------------------------------
/archivist-ui/src/renderer/index.js:
--------------------------------------------------------------------------------
  1 | const React = require("react");
  2 | const ReactDOM = require("react-dom");
  3 | const dateFormat = require("dateformat");
  4 | const strip = require("strip");
  5 | const { chain, identity } = require("lodash");
  6 | const { produce } = require("immer");
  7 | const { shell } = require("electron");
  8 | const { spawn, spawnSync } = require("child_process");
  9 | const { useDebounce } = require("use-debounce");
 10 | const { useHotkeys } = require("react-hotkeys-hook");
 11 | 
 12 | const {
 13 |   AutoSizer,
 14 |   CellMeasurer,
 15 |   CellMeasurerCache,
 16 |   createMasonryCellPositioner,
 17 |   Masonry,
 18 | } = require("react-virtualized");
 19 | 
 20 | require("tachyons/src/tachyons.css");
 21 | require("react-virtualized/styles.css");
 22 | 
 23 | const { useEffect, useCallback, useRef, useReducer } = React;
 24 | 
 25 | const SPACER = 10;
 26 | const SHELL = process.env.SHELL || "bash";
 27 | 
 28 | // running archivist in an interactive shell to support stuff like nvm
 29 | const HAS_ARCHIVIST = !spawnSync(SHELL, ["-i", "-c", "archivist"]).error;
 30 | 
 31 | const executeCLI = async (command, args) => {
 32 |   return new Promise((resolve, reject) => {
 33 |     // running archivist in an interactive shell to support stuff like nvm
 34 |     const cmdArgs = [
 35 |       "-i",
 36 |       "-c",
 37 |       ["archivist", command, args, "--json"].filter(identity).join(" "),
 38 |     ];
 39 | 
 40 |     const process = spawn(SHELL, cmdArgs);
 41 |     let result = "";
 42 | 
 43 |     process.stdout.on("data", (data) => {
 44 |       result += data.toString();
 45 |     });
 46 | 
 47 |     process.stderr.on("data", (data) => {
 48 |       // we can have stderr AND data at the same time, this shouldn't reject the results completely...
 49 |       // reject(data);
 50 |     });
 51 | 
 52 |     process.on("exit", () => {
 53 |       // sometimes shell leaves control sequences...
 54 |       const clean = result.replace(/^.*\[/, "[");
 55 |       resolve(JSON.parse(clean));
 56 |     });
 57 |   });
 58 | };
 59 | 
 60 | const calcColumnWidth = ({ width }) => {
 61 |   return width / Math.floor(width / 400) - SPACER;
 62 | };
 63 | 
 64 | const HoverInfo = ({ meta, link, img, time, setSearchText }) => (
 65 |   <div
 66 |     className="absolute bg-dark-gray pa2 f7 white w-100"
 67 |     style={{ bottom: 0, left: 0, right: 0 }}
 68 |   >
 69 |     <a
 70 |       className="mb2 db f6 lh-title no-underline underline-hover white word-wrap truncate"
 71 |       href="#"
 72 |       onClick={() => link && shell.openExternal(link)}
 73 |     >
 74 |       {meta.title || link}
 75 |     </a>
 76 | 
 77 |     {meta.note && <div className="mb2 lh-copy">{strip(meta.note)}</div>}
 78 | 
 79 |     {meta.tags && (
 80 |       <div>
 81 |         {meta.tags.map((tag) => (
 82 |           <a
 83 |             key={tag}
 84 |             className="no-underline underline-hover light-gray mr1"
 85 |             href="#"
 86 |             onClick={() => setSearchText(tag)}
 87 |           >
 88 |             {tag}
 89 |           </a>
 90 |         ))}
 91 |       </div>
 92 |     )}
 93 | 
 94 |     <div className="mt2 flex justify-between items-center">
 95 |       <div>
 96 |         {[
 97 |           link && ["src", () => shell.openExternal(link)],
 98 |           meta.static && ["frozen", () => shell.openItem(meta.static)],
 99 |           ["img", () => shell.openItem(img)],
100 |         ]
101 |           .filter(identity)
102 |           .map(([text, callback]) => (
103 |             <a
104 |               key={text}
105 |               className="link dim br2 ph2 pv1 dib white bg-mid-gray mr1"
106 |               href="#"
107 |               onClick={callback}
108 |             >
109 |               {text}
110 |             </a>
111 |           ))}
112 |       </div>
113 |       <div className="tr gray">
114 |         {meta.source} / {dateFormat(time, "yyyy-mm-dd")}
115 |       </div>
116 |     </div>
117 |   </div>
118 | );
119 | 
120 | const createCellRenderer = ({
121 |   data,
122 |   width,
123 |   cache,
124 |   setHoveredId,
125 |   hoveredId,
126 |   setSearchText,
127 | }) => ({ index, key, parent, style }) => {
128 |   const columnWidth = calcColumnWidth({ width });
129 |   const datum = data[index] || {};
130 |   const ratio = datum.height / datum.width;
131 |   const imgPath = datum.img;
132 | 
133 |   return (
134 |     <CellMeasurer cache={cache.current} index={index} key={key} parent={parent}>
135 |       <div
136 |         style={style}
137 |         className="h-100"
138 |         onMouseEnter={() => setHoveredId(datum.id)}
139 |         onMouseLeave={() => setHoveredId(null)}
140 |       >
141 |         <div
142 |           className="h-100 relative"
143 |           style={{
144 |             height: ratio * columnWidth,
145 |             width: columnWidth,
146 |             backgroundImage: `url("file:${imgPath}")`,
147 |             backgroundSize: "contain",
148 |             backgroundRepeat: "no-repeat",
149 |             backgroundPosition: "center",
150 |             transform: "translateZ(0)",
151 |           }}
152 |         >
153 |           {hoveredId === datum.id && (
154 |             <HoverInfo setSearchText={setSearchText} {...datum} />
155 |           )}
156 |         </div>
157 |       </div>
158 |     </CellMeasurer>
159 |   );
160 | };
161 | 
162 | const SearchOverlay = React.forwardRef(
163 |   ({ searchText, setSearchText, setIsSearching }, ref) => (
164 |     <div
165 |       className="absolute flex pa2 bg-dark-gray white f7 code"
166 |       style={{ left: 0, right: 0, bottom: 0 }}
167 |     >
168 |       <div className="mr2 lh-copy gray">/</div>
169 |       <input
170 |         ref={ref}
171 |         className="w-100 bg-dark-gray white outline-0 bw0 lh-copy"
172 |         autoFocus={true}
173 |         value={searchText}
174 |         onChange={(e) => setSearchText(e.target.value)}
175 |         onKeyDown={(e) => {
176 |           // escape
177 |           if (e.keyCode === 27) {
178 |             setIsSearching(false);
179 |           }
180 |         }}
181 |       />
182 |     </div>
183 |   )
184 | );
185 | 
186 | const reducer = (state, action) => {
187 |   if (action.type === "SET_DATA") {
188 |     state.data = action.data;
189 |   }
190 | 
191 |   if (action.type === "SET_IS_SEARCHING") {
192 |     state.isSearching = action.isSearching;
193 |     state.searchText = "";
194 |   }
195 | 
196 |   if (action.type === "SET_SEARCH_TEXT") {
197 |     state.isSearching = true;
198 |     state.searchText = action.searchText;
199 |   }
200 | 
201 |   if (action.type === "SET_HOVER_ID") {
202 |     state.hoverId = action.hoverId;
203 |   }
204 | 
205 |   return state;
206 | };
207 | 
208 | const immutableReducer = produce(reducer);
209 | 
210 | const App = () => {
211 |   const [state, dispatch] = useReducer(immutableReducer, {
212 |     data: [],
213 |     searchText: "",
214 |     isSearching: false,
215 |     hoverId: null,
216 |   });
217 | 
218 |   const [debouncedSearchText] = useDebounce(state.searchText, 30);
219 | 
220 |   const searchInputRef = useRef(null);
221 |   const masonry = useRef(null);
222 | 
223 |   useHotkeys(
224 |     "/",
225 |     () => {
226 |       if (!state.isSearching) {
227 |         dispatch({ type: "SET_IS_SEARCHING", isSearching: true });
228 |       } else if (searchInputRef.current) {
229 |         searchInputRef.current.focus();
230 |       }
231 | 
232 |       return false;
233 |     },
234 |     [state.isSearching]
235 |   );
236 | 
237 |   useHotkeys(
238 |     "esc",
239 |     () => {
240 |       if (state.isSearching) {
241 |         dispatch({ type: "SET_IS_SEARCHING", isSearching: false });
242 |       }
243 | 
244 |       return false;
245 |     },
246 |     [state.isSearching]
247 |   );
248 | 
249 |   const cache = useRef(
250 |     new CellMeasurerCache({
251 |       defaultHeight: 400,
252 |       defaultWidth: 400,
253 |       fixedWidth: true,
254 |     })
255 |   );
256 | 
257 |   const cellPositioner = useRef(
258 |     createMasonryCellPositioner({
259 |       cellMeasurerCache: cache.current,
260 |       columnCount: 3,
261 |       columnWidth: 400,
262 |       spacer: SPACER,
263 |     })
264 |   );
265 | 
266 |   useEffect(() => {
267 |     executeCLI("search", debouncedSearchText)
268 |       .then((data) => {
269 |         const finalData = chain(data)
270 |           .map((d) => ({
271 |             ...d,
272 |             time: new Date(d.time),
273 |           }))
274 |           .sortBy((d) => d.time)
275 |           .reverse()
276 |           .value();
277 | 
278 |         dispatch({ type: "SET_DATA", data: finalData });
279 |       })
280 |       .catch((e) => {
281 |         console.error("archivist-cli error", e.toString());
282 |       });
283 |   }, [debouncedSearchText]);
284 | 
285 |   const onResize = useCallback(
286 |     ({ width }) => {
287 |       const columnWidth = calcColumnWidth({ width });
288 |       const columnCount = Math.floor(Math.max(width / columnWidth, 1));
289 | 
290 |       if (cache.current) {
291 |         cache.current.clearAll();
292 |       }
293 | 
294 |       if (cellPositioner.current) {
295 |         cellPositioner.current.reset({
296 |           columnCount,
297 |           columnWidth,
298 |           spacer: SPACER,
299 |         });
300 |       }
301 | 
302 |       if (masonry.current) {
303 |         masonry.current.clearCellPositions();
304 |       }
305 |     },
306 |     [cache, cellPositioner, masonry]
307 |   );
308 | 
309 |   if (!HAS_ARCHIVIST) {
310 |     return (
311 |       <div className="sans-serif w-100 vh-100 bg-light-gray code red pa4 f6">
312 |         Error: archivist cli tool not found
313 |       </div>
314 |     );
315 |   }
316 | 
317 |   return (
318 |     <div className="sans-serif w-100 vh-100 bg-light-gray">
319 |       <AutoSizer
320 |         key={debouncedSearchText + "-" + state.data.length}
321 |         onResize={onResize}
322 |         style={{ outline: "none" }}
323 |       >
324 |         {({ width, height }) =>
325 |           state.data.length > 0 ? (
326 |             <Masonry
327 |               style={{
328 |                 padding: SPACER / 2,
329 |               }}
330 |               overscanByPixels={300}
331 |               ref={masonry}
332 |               cellCount={state.data.length}
333 |               cellMeasurerCache={cache.current}
334 |               cellPositioner={cellPositioner.current}
335 |               cellRenderer={createCellRenderer({
336 |                 data: state.data,
337 |                 width,
338 |                 cache,
339 |                 setHoveredId: (hoverId) =>
340 |                   dispatch({ type: "SET_HOVER_ID", hoverId }),
341 |                 hoveredId: state.hoverId,
342 |                 setSearchText: (searchText) => {
343 |                   dispatch({ type: "SET_SEARCH_TEXT", searchText });
344 |                 },
345 |               })}
346 |               width={width}
347 |               height={height}
348 |             />
349 |           ) : (
350 |             <div />
351 |           )
352 |         }
353 |       </AutoSizer>
354 | 
355 |       {state.isSearching && (
356 |         <SearchOverlay
357 |           ref={searchInputRef}
358 |           searchText={state.searchText}
359 |           setIsSearching={(isSearching) =>
360 |             dispatch({ type: "SET_IS_SEARCHING", isSearching })
361 |           }
362 |           setSearchText={(searchText) =>
363 |             dispatch({ type: "SET_SEARCH_TEXT", searchText })
364 |           }
365 |         />
366 |       )}
367 |     </div>
368 |   );
369 | };
370 | 
371 | const rootEl = document.getElementById("app");
372 | ReactDOM.render(<App />, rootEl);
373 | 


--------------------------------------------------------------------------------
/assets/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/szymonkaliski/archivist/2061bc231b03437313f9f2eb41fdaec66aeebbe9/assets/screenshot.png


--------------------------------------------------------------------------------