├── dist ├── html │ └── readme.txt └── md │ └── readme.txt ├── bin └── juejinxiaoce ├── .editorconfig ├── .eslintrc.js ├── constant.js ├── package.json ├── LICENSE ├── .gitignore ├── utils.js ├── CODE_OF_CONDUCT.md ├── readme.md └── app.js /dist/html/readme.txt: -------------------------------------------------------------------------------- 1 | 爬取的 html 文件目录 2 | -------------------------------------------------------------------------------- /dist/md/readme.txt: -------------------------------------------------------------------------------- 1 | markdown 文件的输出目录 2 | -------------------------------------------------------------------------------- /bin/juejinxiaoce: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | require('../app.js') 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = false 8 | insert_final_newline = true 9 | end_of_line = crlf 10 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "extends": "eslint:recommended", 7 | "parserOptions": { 8 | "ecmaVersion": 2018 9 | }, 10 | "rules": { 11 | "indent": [ 12 | "error", 13 | 2 14 | ], 15 | "linebreak-style": [ 16 | "error", 17 | "windows" 18 | ], 19 | "quotes": [ 20 | "error", 21 | "single" 22 | ], 23 | "semi": [ 24 | "error", 25 | "never" 26 | ] 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /constant.js: -------------------------------------------------------------------------------- 1 | const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' 2 | const URL_HOSTNAME = 'juejin.im' 3 | const URL_LOGIN_EMAIL = '/auth/type/email' 4 | const URL_LOGIN_PHONENUMBER = '/auth/type/phoneNumber' 5 | 6 | const URL_BOOK_HOSTNAME = 'xiaoce-cache-api-ms.juejin.im' 7 | const URL_BOOK_LIST_SECTION = '/v1/getListSection' 8 | const URL_BOOK_SECTION = '/v1/getSection' 9 | 10 | module.exports = { 11 | USER_AGENT, 12 | URL_HOSTNAME, 13 | URL_LOGIN_EMAIL, 14 | URL_LOGIN_PHONENUMBER, 15 | URL_BOOK_HOSTNAME, 16 | URL_BOOK_LIST_SECTION, 17 | URL_BOOK_SECTION 18 | } 19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@oliyg/juejinxiaoce", 3 | "version": "2.2.4", 4 | "description": "", 5 | "main": "app.js", 6 | "scripts": { 7 | "start": "node app.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/oliyg/juejinxiaoce.git" 12 | }, 13 | "keywords": [], 14 | "bin": { 15 | "juejinxiaoce": "bin/juejinxiaoce" 16 | }, 17 | "author": "OliverYoung ", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/oliyg/juejinxiaoce/issues" 21 | }, 22 | "homepage": "https://github.com/oliyg/juejinxiaoce#readme", 23 | "devDependencies": { 24 | "eslint": "^5.12.1" 25 | }, 26 | "dependencies": { 27 | "turndown": "^5.0.3" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2019 OliverYoung 3 | 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 19 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | OR OTHER DEALINGS IN THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | .env.test 60 | 61 | # parcel-bundler cache (https://parceljs.org/) 62 | .cache 63 | 64 | # next.js build output 65 | .next 66 | 67 | # nuxt.js build output 68 | .nuxt 69 | 70 | # vuepress build output 71 | .vuepress/dist 72 | 73 | # Serverless directories 74 | .serverless/ 75 | 76 | # FuseBox cache 77 | .fusebox/ 78 | 79 | # DynamoDB Local files 80 | .dynamodb/ 81 | 82 | 83 | 84 | 85 | # dist 86 | dist/html/*.html 87 | dist/md/*.md 88 | -------------------------------------------------------------------------------- /utils.js: -------------------------------------------------------------------------------- 1 | const https = require('https') 2 | const fs = require('fs') 3 | const path = require('path') 4 | const process = require('process') 5 | 6 | const mkdir = dirname => { 7 | const outputdir = path.resolve(process.env.PWD, dirname) 8 | fs.existsSync(outputdir) || fs.mkdirSync(outputdir) 9 | } 10 | 11 | const rmfile = ext => { 12 | const outputdir = path.resolve(__dirname, 'dist', ext) 13 | const fileList = fs.readdirSync(outputdir).filter(item => path.extname(item) === '.' + ext) 14 | fileList.forEach(file => { 15 | fs.unlinkSync(path.join(outputdir, file)) 16 | }) 17 | } 18 | 19 | const sleep = async(wait = 1000) => { 20 | return new Promise(res => { 21 | setTimeout(() => { 22 | res(1) 23 | }, wait) 24 | }) 25 | } 26 | 27 | function sendGet(hostname, path, headers) { 28 | return new Promise((resolve, reject) => { 29 | const req = https.request({ 30 | hostname, port: 443, path, method: 'GET' 31 | }, res => { 32 | let data = '' 33 | res.on('data', chunk => { data = data + chunk.toString() }) 34 | res.on('error', reject) 35 | res.on('end', () => { 36 | resolve({ 37 | headers: res.headers, 38 | data 39 | }) 40 | }) 41 | }) 42 | setHeaders(req, headers) 43 | req.on('error', reject) 44 | req.end() 45 | }) 46 | } 47 | 48 | function sendPost(hostname, path, data, headers) { 49 | return new Promise((resolve, reject) => { 50 | const req = https.request({ 51 | hostname, path, port: 443, method: 'POST' 52 | }, res => { 53 | let data = '' 54 | res.on('data', chunk => { data = data + chunk.toString() }) 55 | res.on('end', () => { resolve({ 56 | res, 57 | data 58 | }) }) 59 | res.on('error', reject) 60 | }) 61 | 62 | setHeaders(req, headers) 63 | req.on('error', reject) 64 | req.write(data) 65 | req.end() 66 | }) 67 | } 68 | 69 | function setHeaders(req, obj) { 70 | for (const key in obj) { 71 | if (obj.hasOwnProperty(key)) { 72 | req.setHeader(key, obj[key]) 73 | } 74 | } 75 | } 76 | 77 | function getCookieObj(cookie_arr) { 78 | cookie_arr = cookie_arr.map(item => item.split('; path=')[0]) 79 | cookie_arr = cookie_arr.map(item => item.split('=')) 80 | let result = {} 81 | cookie_arr.forEach(item => { 82 | result[item[0]] = item[1] 83 | }) 84 | return result 85 | } 86 | 87 | function getCookieArr(cookie_obj) { 88 | const result = [] 89 | for (const key in cookie_obj) { 90 | if (cookie_obj.hasOwnProperty(key)) { 91 | const value = cookie_obj[key] 92 | result.push(key + '=' + value) 93 | } 94 | } 95 | return result 96 | } 97 | 98 | module.exports = { 99 | setHeaders, getCookieObj, getCookieArr, sendPost, sendGet, sleep, rmfile, mkdir 100 | } 101 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at billyangg@qq.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | 78 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Foliyg%2Fjuejinxiaoce.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Foliyg%2Fjuejinxiaoce?ref=badge_shield) 2 | 3 | # 🔥 掘金小册 markdown 转换器 4 | 5 | ![20190121001820.png](https://i.loli.net/2019/01/21/5c449f4dbc3d5.png) 6 | 7 | [github 仓库欢迎 star](https://github.com/oliyg/juejinxiaoce) 8 | 9 | 采用 node https 模块，获取已购买小册 html 代码，并将 html 代码转换为 markdown 格式文件保存本地。 10 | 11 | **注意：目前本项目有两个版本，v2 不需要使用 chromium 作为无头浏览器；v1 则使用 chromi 作为无头浏览器模拟用户登录网站；** 12 | 13 | 根据需要选择不同版本 14 | 15 | - v2： 16 | - [latest](https://github.com/oliyg/juejinxiaoce/releases) 17 | - v1 不再维护： 18 | - [release v1](https://github.com/oliyg/juejinxiaoce/releases/tag/1.1.2) 19 | 20 | ## 使用方法 21 | 22 | **⚠️ 注意：掘金不支持境外网络访问，因此不要使用代理** 23 | 24 | ### 方法一：npx 直接执行 25 | 26 | 在本地某目录中执行 `npx @oliyg/juejinxiaoce` 按照提示输入用户名密码以及小册 ID 当提示 all done 完成 27 | 28 | ``` 29 | ➜ Desktop npx @oliyg/juejinxiaoce 30 | npx: 98 安装成功，用时 10.748 秒 31 | email: 输入你的用户名密码 32 | password: 输入你的用户名密码 33 | bookId: 小册 ID 34 | ===navagating to main page 35 | ===login... 36 | ===getting book section list 37 | ===getting book HTML content 38 | 面试常用技巧 39 | ===writing html... 40 | ===getting book HTML content 41 | ===write html file success 42 | ===writing markdown... 43 | ===write markdown file success 44 | 前方的路，让我们结伴同行 45 | ===writing html... 46 | ===write html file success 47 | ===writing markdown... 48 | ===write markdown file success 49 | 50 | ====== 51 | All Done...Enjoy. 52 | ====== 53 | ``` 54 | 55 | 在执行命令的这个目录中可以找到一个名为 md xxx 的文件夹，内包含 md 文档；在上面这个例子中，我们在 Desktop 桌面目录执行命令，因此在桌面目录中会生成这个文件夹： 56 | 57 | ```shell 58 | ➜ md 1548483715543 ls -al 59 | total 40 60 | drwxr-xr-x 4 oli staff 128 1 26 14:22 . 61 | drwx------+ 9 oli staff 288 1 26 14:21 .. 62 | -rw-r--r-- 1 oli staff 4915 1 26 14:21 面试常用技巧.md 63 | -rw-r--r-- 1 oli staff 8465 1 26 14:22 前方的路，让我们结伴同行.md 64 | ``` 65 | 66 | ### 方法二：npm i 命令 67 | 68 | 使用 `npm i -g` 安装，并使用 `juejinxiaoce` 命令执行： 69 | 70 | ``` 71 | ➜ Desktop npm i -g @oliyg/juejinxiaoce 72 | /Users/oli/.nvm/versions/node/v8.12.0/bin/juejinxiaoce -> /Users/oli/.nvm/versions/node/v8.12.0/lib/node_modules/@oliyg/juejinxiaoce/bin/juejinxiaoce 73 | + @oliyg/juejinxiaoce@2.2.1 74 | added 98 packages from 201 contributors in 5.89s 75 | ➜ Desktop juejinxiaoce 76 | email: 77 | password: 78 | bookId: 79 | ===navagating to main page 80 | ===login... 81 | ... 82 | ... 83 | ``` 84 | 85 | > 小册ID见 URL 链接： 86 | > 87 | > ![20190120235353.png](https://i.loli.net/2019/01/20/5c4499929e48e.png) 88 | 89 | 执行后等待出现消息 `all done. enjoy.` 完成转换，效果如下： 90 | 91 | ![20190121000703.png](https://i.loli.net/2019/01/21/5c449ca8d869e.png) 92 | 93 | ![20190121000715.png](https://i.loli.net/2019/01/21/5c449cb443d62.png) 94 | 95 | ## 更新日志 96 | 97 | - v2.2.4 修改文件名 98 | - v2.2.0 增加命令行模式 99 | - v2.0.0 使用 node 原生 https 模块，发送请求数据获取内容，不需要安装 chromium，没有软件权限问题 100 | - v1.1.2 使用谷歌 puppeteer 作为无头浏览器获取内容，需要安装 chromium，macOS 中可能有权限问题 101 | 102 | ## 常见问题 103 | 104 | - v1.1.2 105 | - 报错：spawn EACCES 106 | - 常见于 macOS，请保证 chromium 已被正常安装 107 | 108 | ## 开源贡献 109 | 110 | - 感谢 `@yangchendoit` 提交的 pr 完善手机号码登录功能 111 | 112 | ## 免责 113 | 114 | - 不提供用户名和密码，需使用用户自己的账号密码登录 115 | - 仅作为技术讨论，学习和研究使用 116 | 117 | ## 隐私 118 | 119 | - 该项目不会存储和发送任何用户隐私数据 120 | 121 | ## License 122 | 123 | The MIT License (MIT) 124 | Copyright (c) 2019 OliverYoung 125 | 126 | 127 | Permission is hereby granted, free of charge, to any person obtaining a copy 128 | of this software and associated documentation files (the "Software"), to deal 129 | in the Software without restriction, including without limitation the rights 130 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 131 | copies of the Software, and to permit persons to whom the Software is 132 | furnished to do so, subject to the following conditions: 133 | 134 | The above copyright notice and this permission notice shall be included in all 135 | copies or substantial portions of the Software. 136 | 137 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 138 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 139 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 140 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 141 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 142 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 143 | OR OTHER DEALINGS IN THE SOFTWARE. 144 | 145 | [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Foliyg%2Fjuejinxiaoce.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Foliyg%2Fjuejinxiaoce?ref=badge_large) 146 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | const fs = require('fs') 3 | const path = require('path') 4 | const os = require('os') 5 | const readline = require('readline') 6 | const Turndown = require('turndown') 7 | 8 | const { getCookieArr, getCookieObj, sendPost, sendGet, sleep, rmfile, mkdir } = require('./utils') 9 | const { USER_AGENT, URL_HOSTNAME, URL_LOGIN_EMAIL, URL_LOGIN_PHONENUMBER, URL_BOOK_HOSTNAME, URL_BOOK_LIST_SECTION, URL_BOOK_SECTION } = require('./constant') 10 | 11 | class Juejin { 12 | constructor() { 13 | this.loginType = '0' // 0为邮箱 1为手机 14 | this.account = '' 15 | this.password = '' 16 | this.bookID = '' 17 | this.cookie = '' 18 | this.src = 'web' 19 | this.userInfo = {} 20 | this.bookSectionList = [] 21 | this.count = 0 22 | this.pwd = process.env.PWD 23 | } 24 | 25 | copyToPWDDir(dirname) { 26 | mkdir(dirname) 27 | 28 | const output = path.resolve(__dirname, 'dist', 'md') 29 | 30 | const fileList = fs.readdirSync(output).filter(item => path.extname(item) === '.md') 31 | fileList.forEach(file => { 32 | fs.copyFileSync(path.join(output, file), path.resolve(process.env.PWD, dirname, file)) 33 | }) 34 | } 35 | 36 | async getMetaData() { 37 | const rl = readline.createInterface({ 38 | input: process.stdin, 39 | output: process.stdout 40 | }) 41 | 42 | const question = query => { 43 | return new Promise(resolve => { 44 | rl.question(query, resolve) 45 | }) 46 | } 47 | console.warn('loginType 0：邮箱 1：手机号码') 48 | const loginType = await question('loginType: ') 49 | const account = await question('account: ') 50 | const password = await question('password: ') 51 | const bookID = await question('bookId: ') 52 | 53 | rl.close() 54 | this.loginType = loginType 55 | this.account = account 56 | this.password = password 57 | this.bookID = bookID 58 | 59 | Promise.resolve() 60 | } 61 | 62 | async mainPage() { 63 | console.warn('===navagating to main page') 64 | const headers = { 65 | 'User-Agent': USER_AGENT, 66 | 'Connection': 'keep-alive' 67 | } 68 | const response = await sendGet(URL_HOSTNAME, '/', headers) 69 | this.cookie = JSON.stringify(getCookieObj(response.headers['set-cookie'])) 70 | } 71 | 72 | async login() { 73 | console.warn('===login...') 74 | const authObj = { 75 | password: this.password 76 | } 77 | let loginUrl; 78 | 79 | if(this.loginType === '0'){ 80 | Object.assign(authObj,{ 81 | email: this.account 82 | }) 83 | loginUrl = URL_LOGIN_EMAIL; 84 | }else{ 85 | Object.assign(authObj,{ 86 | phoneNumber: this.account 87 | }) 88 | loginUrl = URL_LOGIN_PHONENUMBER; 89 | } 90 | const auth = JSON.stringify(authObj) 91 | const headers = { 92 | 'User-Agent': USER_AGENT, 93 | 'Content-Type': 'application/json', 94 | 'Content-Length': Buffer.byteLength(auth), 95 | 'Cookie': getCookieArr(JSON.parse(this.cookie)) 96 | } 97 | const response = await sendPost(URL_HOSTNAME, loginUrl, auth, headers) 98 | this.cookie = JSON.stringify(Object.assign(JSON.parse(this.cookie), getCookieObj(response.res.headers['set-cookie']))) 99 | this.userInfo = JSON.parse(response.data) 100 | return response 101 | } 102 | 103 | async getTargetBookSectionList() { 104 | console.warn('===getting book section list') 105 | const headers = { 106 | 'User-Agent': USER_AGENT, 107 | 'Connection': 'keep-alive' 108 | } 109 | const response = await sendGet(URL_BOOK_HOSTNAME, `${URL_BOOK_LIST_SECTION}?uid=${this.userInfo.userId}&client_id=${this.userInfo.user.clientId}&token=${this.userInfo.user.token}&src=${this.src}&id=${this.bookID}`, headers) 110 | const data = response.data 111 | this.bookSectionList = JSON.parse(data).d 112 | return response 113 | } 114 | 115 | async getContentHTML(callback) { 116 | console.warn('===getting book HTML content') 117 | const headers = { 118 | 'User-Agent': USER_AGENT, 119 | 'Connection': 'keep-alive' 120 | } 121 | await sleep(3000) 122 | 123 | const url = `${URL_BOOK_SECTION}?uid=${this.userInfo.userId}&client_id=${this.userInfo.clientId}&token=${this.userInfo.token}&src=${this.src}§ionId=${this.bookSectionList[this.count].sectionId}` 124 | const response = await sendGet(URL_BOOK_HOSTNAME, url, headers) 125 | let data = JSON.parse(response.data) 126 | 127 | console.log(`${this.count + 1}.${data.d.title}`) 128 | data.d.isFinished || console.log('写作中...') 129 | callback(data.d) 130 | 131 | this.count ++ 132 | let maxCount = this.bookSectionList.length 133 | this.count < maxCount && await this.getContentHTML(callback) 134 | } 135 | 136 | saveHTML(d) { 137 | return new Promise((resolve, reject) => { 138 | console.log('===writing html...') 139 | const title = d.title.replace(/[/?*:|\\<>]/g, ' ') 140 | const output = path.resolve(__dirname, 'dist', 'html', title + '.html') 141 | fs.writeFile(output, d.html, { encoding: 'utf-8' }, err => { 142 | err && reject(err) 143 | console.log('===write html file success') 144 | resolve({title, output}) 145 | }) 146 | }) 147 | } 148 | 149 | toMarkdown(title, path) { 150 | return new Promise((resolve, reject) => { 151 | const turndownService = new Turndown({ 152 | headingStyle: 'atx', 153 | codeBlockStyle: 'fenced' 154 | }) 155 | try { 156 | const markdown = turndownService.turndown(fs.readFileSync(path, { encoding: 'utf-8' })) 157 | resolve({ title, markdown }) 158 | } catch (error) { 159 | reject(error) 160 | } 161 | }) 162 | } 163 | 164 | saveMD(title, data) { 165 | return new Promise((resolve, reject) => { 166 | console.log('===writing markdown...') 167 | const output = path.resolve(__dirname, 'dist', 'md', `${this.count}.${title}.md`) 168 | fs.writeFile(output, data, { encoding: 'utf-8' }, err => { 169 | err && reject(err) 170 | console.log('===write markdown file success') 171 | resolve() 172 | }) 173 | }) 174 | } 175 | 176 | } 177 | 178 | {(async () => { 179 | rmfile('html') 180 | rmfile('md') 181 | const juejin = new Juejin() 182 | try { 183 | await juejin.getMetaData() 184 | await juejin.mainPage() 185 | await sleep() 186 | await juejin.login() 187 | await sleep() 188 | await juejin.getTargetBookSectionList() 189 | 190 | const dirname = 'md ' + + new Date() 191 | 192 | await juejin.getContentHTML(async d => { 193 | const { title, output } = await juejin.saveHTML(d) 194 | const { title: mdTitle, markdown: markdownData } = await juejin.toMarkdown(title, output) 195 | await juejin.saveMD(mdTitle, markdownData) 196 | juejin.copyToPWDDir(dirname) 197 | }) 198 | 199 | setTimeout(() => { 200 | console.log(`${os.EOL}======${os.EOL}All Done...Enjoy.${os.EOL}======${os.EOL}`) 201 | }, 200) 202 | 203 | } catch (error) { 204 | console.log(error) 205 | } 206 | })()} 207 | --------------------------------------------------------------------------------