├── .gitignore ├── .prettierrc.json ├── tsconfig.json ├── src ├── index.ts ├── utils.ts └── core.ts ├── .eslintrc.json ├── README.md ├── LICENSE └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | node_modules 3 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "printWidth": 100 5 | } 6 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["esnext", "dom"], 4 | "module": "commonjs", 5 | "outDir": "bin", 6 | "strict": true, 7 | "target": "esnext", 8 | "resolveJsonModule": true 9 | }, 10 | "include": ["src"] 11 | } 12 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { program } from 'commander' 4 | import { run } from './core' 5 | import { isValidURL } from './utils' 6 | import { version } from '../package.json' 7 | 8 | program.version(version, '-v, --version') 9 | program.parse(process.argv) 10 | 11 | if (!program.args.length) program.help() 12 | 13 | const [url] = program.args 14 | 15 | if (!isValidURL(url)) { 16 | console.error(`💥 Invalid URL!`) 17 | process.exit(0) 18 | } 19 | 20 | run(url) 21 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "eslint:recommended", 4 | "plugin:@typescript-eslint/eslint-recommended", 5 | "plugin:@typescript-eslint/recommended", 6 | "plugin:prettier/recommended", 7 | "prettier/@typescript-eslint" 8 | ], 9 | "parser": "@typescript-eslint/parser", 10 | "parserOptions": { 11 | "ecmaFeatures": { 12 | "sourceType": "module" 13 | } 14 | }, 15 | "rules": { 16 | "@typescript-eslint/explicit-function-return-type": "off", 17 | "@typescript-eslint/no-non-null-assertion": "error", 18 | "@typescript-eslint/no-unused-vars": "error", 19 | "prettier/prettier": "error" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

🍞 ankipan

2 |

A command line tool to save the full resources of any web page.

3 |

4 | 5 | 6 | 7 | 8 | 9 | 10 |

11 | 12 | ## Requirements 13 | 14 | - Node.js 10.12.0 (LTS) 15 | 16 | ## Installation 17 | 18 | ```bash 19 | $ npm i -g ankipan 20 | ``` 21 | 22 | ## Usage 23 | 24 | ```bash 25 | $ ankipan https://github.com 26 | github.com/index.html 27 | github.com/images/search-key-slash.svg 28 | github.com/images/modules/site/home/globe-700.jpg 29 | ... 30 | github.com/customer_stories/yyx990803/hero.jpg 31 | github.com/customer_stories/kris-nova/hero.jpg 32 | github.com/customer_stories/jessfraz/hero.jpg 33 | ✨ Done! 34 | 35 | $ npx serve github.com # serve a saved web site 36 | ``` 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 saitoeku3 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import { extname } from 'path' 2 | import { getExtension } from 'mime/lite' 3 | import type { Page } from 'puppeteer' 4 | 5 | /** 6 | * @see https://github.com/puppeteer/puppeteer/issues/305#issuecomment-385145048 7 | */ 8 | export const autoScroll = async (page: Page): Promise => { 9 | await page.evaluate(async () => { 10 | await new Promise((resolve) => { 11 | let totalHeight = 0 12 | const distance = 100 13 | const timer = setInterval(() => { 14 | const scrollHeight = document.body.scrollHeight 15 | scrollBy(0, distance) 16 | totalHeight += distance 17 | if (totalHeight >= scrollHeight) { 18 | clearInterval(timer) 19 | resolve() 20 | } 21 | }, 100) 22 | }) 23 | }) 24 | } 25 | 26 | export const getFilepath = ({ 27 | directory, 28 | pathname, 29 | mimeType, 30 | }: { 31 | directory: string 32 | pathname: string 33 | mimeType: string 34 | }): string => { 35 | const hasExtension = extname(pathname) !== '' 36 | return hasExtension 37 | ? `${directory}${pathname}` 38 | : `${directory}${pathname}.${getExtension(mimeType)}` 39 | } 40 | 41 | export const isValidURL = (value: string): boolean => { 42 | try { 43 | new URL(value) 44 | return true 45 | } catch { 46 | return false 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ankipan", 3 | "version": "0.1.0", 4 | "description": "A command line tool to save the full resources of any web page.", 5 | "license": "MIT", 6 | "bin": "bin/src/index.js", 7 | "author": "Tadao Iseki ", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/saitoeku3/ankipan" 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/saitoeku3/ankipan/issues" 14 | }, 15 | "files": [ 16 | "bin" 17 | ], 18 | "keywords": [ 19 | "cli" 20 | ], 21 | "scripts": { 22 | "dev": "ts-node -P tsconfig.json src/index.ts", 23 | "build": "tsc -b tsconfig.json", 24 | "prepublishOnly": "npm run build", 25 | "lint": "eslint \"src/**/*.ts\"", 26 | "format": "eslint \"src/**/*.ts\" --fix" 27 | }, 28 | "devDependencies": { 29 | "@types/mime": "^2.0.3", 30 | "@types/prettier": "^2.1.1", 31 | "@types/puppeteer": "^3.0.2", 32 | "@types/sleep": "0.0.8", 33 | "@typescript-eslint/eslint-plugin": "^4.4.0", 34 | "@typescript-eslint/parser": "^4.4.0", 35 | "eslint": "^7.10.0", 36 | "eslint-config-prettier": "^6.12.0", 37 | "eslint-plugin-prettier": "^3.1.4", 38 | "prettier": "^2.1.2", 39 | "ts-node": "^9.0.0", 40 | "typescript": "^4.0.3" 41 | }, 42 | "dependencies": { 43 | "commander": "^6.1.0", 44 | "mime": "^2.4.6", 45 | "puppeteer": "^5.3.1" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/core.ts: -------------------------------------------------------------------------------- 1 | import { promises as fs } from 'fs' 2 | import { dirname } from 'path' 3 | import { launch } from 'puppeteer' 4 | import { autoScroll, getFilepath } from './utils' 5 | 6 | export const run = async (url: string): Promise => { 7 | const browser = await launch({ handleSIGTERM: false }) 8 | const [page] = await browser.pages() 9 | const { host: directory, pathname: documentPath } = new URL(url) 10 | 11 | try { 12 | await fs.access(directory) 13 | await fs.rmdir(directory, { recursive: true }) 14 | } catch { 15 | // noop 16 | } finally { 17 | await fs.mkdir(directory) 18 | } 19 | 20 | page.on('response', async (res) => { 21 | const { pathname } = new URL(res.url()) 22 | const mimeType = res.headers()['content-type'] 23 | 24 | if (!pathname.startsWith('/')) return 25 | 26 | const filepath = getFilepath({ 27 | directory, 28 | pathname: pathname === documentPath ? '/index.html' : pathname, 29 | mimeType, 30 | }) 31 | 32 | try { 33 | const buffer = await res.buffer() 34 | await fs.mkdir(`${directory}${dirname(pathname)}`, { recursive: true }) 35 | await fs.writeFile(filepath, buffer) 36 | console.log(filepath) 37 | } catch { 38 | // noop 39 | } 40 | }) 41 | 42 | await page.goto(url) 43 | await autoScroll(page) 44 | 45 | await page.close() 46 | await browser.close() 47 | console.log('✨ Done!') 48 | } 49 | --------------------------------------------------------------------------------