├── .gitignore ├── 89wxw ├── config.js ├── data │ └── 占位 ├── main.js └── readme.md ├── LICENSE ├── README.md ├── ali_sort ├── index.html ├── main.js └── static │ ├── index.css │ └── index.js ├── es ├── index.html └── readme.md ├── package-lock.json ├── package.json ├── pixiv_col ├── config │ └── index.js └── main.js ├── static └── img │ ├── 89wxw.png │ ├── ali_sort.png │ └── pixiv_col.png └── wx ├── README.md └── auto_artile.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | pixiv_col/imgs/ 3 | pixiv_col/config/cookie.js 4 | pixiv_col/progress.txt 5 | ali_sort/config.js -------------------------------------------------------------------------------- /89wxw/config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // 书籍章节列表url 3 | url: 'http://www.89wxw.cn/0/524/', 4 | // 书名 5 | name: '放开那个女巫', 6 | // 多久抓取一次, 单位毫秒, 建议大于30000 7 | time: 10000, 8 | } 9 | -------------------------------------------------------------------------------- /89wxw/data/占位: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kvker/js_spider/2cf43639962186756a791c51d6c14897f000a959/89wxw/data/占位 -------------------------------------------------------------------------------- /89wxw/main.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const superagent = require('superagent') 4 | const charset = require('superagent-charset') 5 | const cheerio = require('cheerio') 6 | const request = charset(superagent) 7 | const config = require('./config') 8 | const express = require('express') 9 | const cp = require('child_process') 10 | // 一小时重置一次,防爬虫或广告导致 11 | setTimeout(() => { 12 | cp.exec('pm2 restart 89wxw') 13 | }, 60 * 60 * 1000) 14 | 15 | const app = express() 16 | 17 | // 总进度 18 | let total_length = 1 19 | // 当前进度 20 | let current_progress 21 | // 本地保存的进度 22 | let save_progress = 0 23 | try { 24 | save_progress = Number(fs.readFileSync(`89wxw/progress`)) 25 | current_progress = save_progress 26 | } catch(error) { 27 | current_progress = 0 28 | } 29 | console.log({ current_progress }) 30 | 31 | // 服务器提供外网查看的接口 32 | app.get('/progress', (req, res) => { 33 | res.json({ 34 | current_progress, 35 | total_length, 36 | progress: (current_progress / total_length).toFixed(2), 37 | }) 38 | }) 39 | 40 | // 进度监控网站 41 | app.listen(4800, () => { 42 | console.log('spider run in: ' + 'http://localhost:4800') 43 | }) 44 | 45 | // 加载文章章节 46 | loadRoot(config.url, config.name) 47 | 48 | /** 49 | * 加载文章章节, 处理章节列表 50 | * @param {string} url 文章章节地址 51 | */ 52 | async function loadRoot(url) { 53 | if(url.trim().length && url.startsWith('http')) { 54 | try { 55 | let ret = await request.get(url).charset('gbk').buffer(true) 56 | // console.log(ret.text) 57 | const $ = cheerio.load(ret.text) 58 | // 解析目录 59 | let titles = $('#list dd > a') 60 | // 目录返回的页面标题和地址 61 | let pages = titles.map(function(idx) { 62 | return { 63 | label: $(this).text(), 64 | href: $(this).attr('href'), 65 | } 66 | }).get() 67 | // 总章节数 68 | total_length = pages.length 69 | let i = 0 70 | // 异步遍历获取章节内容 71 | for(const page of pages) { 72 | if(i < save_progress) { 73 | i++ 74 | continue 75 | } 76 | current_progress++ 77 | // 下载内容到本地 78 | await downloadContent(page.href, page.label) 79 | i++ 80 | } 81 | } catch(error) { 82 | console.log(error) 83 | } 84 | } else { 85 | throw new Error('非法链接') 86 | } 87 | } 88 | 89 | /** 90 | * 下载追加到本地 91 | * @param {string} path 章节内容地址 92 | * @param {string} title 章节标题 93 | */ 94 | function downloadContent(path, title) { 95 | return new Promise(async (resolve, reject) => { 96 | try { 97 | let ret = await request.get('http://www.89wxw.cn' + path).charset('gbk').buffer(true) 98 | const $ = cheerio.load(ret.text) 99 | let content = ` 100 | ${title} 101 | 102 | ${$('#content').text()} 103 | 104 | -------------------------------------------------------------------------------------------------------------- 105 | 106 | ` 107 | fs.appendFileSync(`89wxw/data/${config.name}.txt`, content) 108 | fs.writeFileSync(`89wxw/progress`, current_progress) 109 | setTimeout(() => { 110 | resolve() 111 | }, config.time) 112 | } catch(error) { 113 | let content = ` 114 | ${title} 115 | 116 | 章节挂了 117 | 118 | -------------------------------------------------------------------------------------------------------------- 119 | 120 | ` 121 | fs.appendFileSync(`89wxw/data/${config.name}.txt`, content) 122 | reject(error) 123 | } 124 | }) 125 | } -------------------------------------------------------------------------------- /89wxw/readme.md: -------------------------------------------------------------------------------- 1 | # 使用 2 | 3 | * 配置config.js 4 | 5 | ```js 6 | module.exports = { 7 | // 书籍章节列表url 8 | url: 'http://www.89wxw.cn/0/524/', 9 | // 书名 10 | name: '放开那个女巫', 11 | // 多久抓取一次, 单位毫秒, 建议大于30000 12 | time: 30000, 13 | } 14 | ``` 15 | 16 | * `npm run 89wxw` 项目根目录运行 17 | 18 | > pm2: `pm2 start npm -n 89wxw -- run 89wxw` 19 | 20 | * 然后就可以在data/书名.txt里面获取到啦, 哈哈哈哈 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 程序员有点田 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spider 2 | 3 | 日常脚本/爬虫. 4 | 5 | ## 安装 6 | 7 | `npm i` 安装依赖 8 | 9 | ## 运行 10 | 11 | ### 阿里试用排序 12 | 13 | ![示例](static/img/ali_sort.png) 14 | 15 | `touch ali_sort/config.js` 创建配置文件 16 | 17 | 写入如下内容, cookie请自行使用自己的. 18 | 19 | ```js 20 | module.exports = { 21 | host: 'localhost', 22 | port: 8000, 23 | // 登录try.taobao.com后开发控制台复制cookie, 下面示例cookie不保证可用性 24 | cookie: 'your_cookie' 25 | } 26 | ``` 27 | 28 | `npm run ali_sort` 29 | 30 | ### pixiv 个人收藏夹 31 | 32 | **请自行科学上网** 33 | 34 | ![示例](static/img/pixiv_col.png) 35 | 36 | `touch pixiv/config/cookie.js` 创建cookie文件 37 | 38 | `pixiv/config/cookie.js` 写入cookie 39 | 40 | `npm run pc` 运行 41 | 42 | ### 89wxw 43 | 44 | 89文学网爬虫 45 | 46 | ![示例](static/img/89wxw.png) 47 | 48 | `89wxw/config.js` 为配置文件 49 | 50 | `pm2 start npm -n 89wxw -- run 89wxw` pm2运行为了可以被踢重连 51 | -------------------------------------------------------------------------------- /ali_sort/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 阿里试用 - 一日内完结排序工具 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 |
序号概率名字缩略图
28 |
29 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /ali_sort/main.js: -------------------------------------------------------------------------------- 1 | const config = require('./config') 2 | const express = require('express') 3 | const path = require('path') 4 | const superagent = require('superagent') 5 | const app = express() 6 | const bodyParser = require("body-parser") 7 | app.use(bodyParser.urlencoded({ extended: false })) 8 | app.use(express.static(path.join(__dirname, './'))) 9 | 10 | function postPage(paylaod = {}) { 11 | return new Promise(resolve => { 12 | superagent 13 | .post(`https://try.taobao.com/api3/call?what=show&page=${paylaod.page}&pageSize&api=x%2Fsearch`) 14 | .set('cookie', config.cookie) 15 | .set('referer', 'https://try.taobao.com') 16 | .set('x-csrf-token', '7e885ad0b4833') 17 | .end((err, sres) => { 18 | // handle normal error 19 | if(err) { 20 | return next(err) 21 | } 22 | const result = JSON.parse(sres.text).result 23 | resolve(result) 24 | }) 25 | }) 26 | } 27 | 28 | app.get('/total', async (req, res) => { 29 | const result = await postPage() 30 | res.send({ pages: result.paging.pages }) 31 | }) 32 | 33 | app.post('/table', async (req, res) => { 34 | const result = await postPage({ 35 | page: req.body.page 36 | }) 37 | const items = result.items 38 | const currentTime = new Date().getTime() 39 | // end in one day 40 | items.filter(i => { 41 | return i.endTime - currentTime < 86400000 42 | }) 43 | 44 | res.send(items.map(item => ({ 45 | ...item, 46 | rate: item.totalNum / item.requestNum * 100 47 | }))) 48 | }) 49 | 50 | app.listen(config.port, config.host, function() { 51 | console.log('please open: http://' + config.host + ':' + config.port) 52 | }) 53 | -------------------------------------------------------------------------------- /ali_sort/static/index.css: -------------------------------------------------------------------------------- 1 | * { box-sizing: border-box !important; } 2 | 3 | /* html, body { height: 100%; } */ 4 | 5 | body { 6 | margin: 0; 7 | font-family: 'Microsoft YaHei'; 8 | } 9 | 10 | .btn-page{ 11 | display: flex; 12 | justify-content: center; 13 | position: fixed; 14 | bottom: 10px; 15 | left: 50%; 16 | transform: translate(-50%, 0); 17 | z-index: 1000; 18 | } 19 | 20 | .btn-page .btn-next { 21 | width: 92px; 22 | height: 36px; 23 | line-height: 36px; 24 | background: #4caf50; 25 | color: #fff; 26 | border-radius: 4px; 27 | text-align: center; 28 | cursor: pointer; 29 | } 30 | 31 | #currentPage { 32 | width: 120px; 33 | height: 36px; 34 | line-height: 36px; 35 | background: #F9FBE7; 36 | color: #666; 37 | margin-left: 10px; 38 | padding: 0 10px; 39 | } 40 | 41 | .main { 42 | } 43 | 44 | .warning { 45 | color: red; 46 | } 47 | 48 | button { 49 | width: 100px; 50 | height: 44px; 51 | margin-right: 44px; 52 | } 53 | 54 | table { 55 | width: 100%; 56 | margin: 0 0 60px; 57 | border-collapse: collapse; 58 | } 59 | 60 | thead { 61 | line-height: 44px; 62 | } 63 | 64 | tr { 65 | border-bottom: 1px solid #d8d8d8; 66 | min-height: 48px; 67 | color: #333; 68 | line-height: 44px; 69 | } 70 | 71 | #results { 72 | } 73 | 74 | #results tr:hover { 75 | background: #E6EE9C; 76 | cursor: pointer; 77 | } 78 | 79 | .img-box { 80 | display: flex; 81 | justify-content: center; 82 | } 83 | 84 | .img-box img { 85 | display: table-cell; 86 | width: 44px; 87 | height: 44px; 88 | } 89 | 90 | th:nth-child(1) { width: 10%; } 91 | th:nth-child(2) { width: 20%; } 92 | th:nth-child(3) { width: 50%; text-align: left; } 93 | th:nth-child(4) { width: 20%; } 94 | 95 | th, td { text-align: center; } 96 | 97 | td:nth-child(3) { text-align: left; } 98 | 99 | td { 100 | position: relative; 101 | } 102 | -------------------------------------------------------------------------------- /ali_sort/static/index.js: -------------------------------------------------------------------------------- 1 | let currentPage = 0 // 当前页面 2 | let allItems = [] // 全部数据 3 | let currentTime = 0 // 锁频率使用，标记上次时间 4 | const xhr = new XMLHttpRequest() 5 | const loopInterval = 2 // 锁频率步长，单位秒 6 | const results = document.querySelector('#results') 7 | const currentPageText = document.querySelector('#currentPage') 8 | const reFullTBody = arr => { 9 | let innerHtml = '' 10 | arr.forEach((item, i) => { 11 | item.rate = item.totalNum / item.requestNum * 100 12 | let tr = ` 13 | 14 | ${i + 1} 15 | ${item.rate.toFixed(3) + '%'} 16 | ${item.title} 17 |
18 | 19 | ` 20 | if (item.rate > 5) tr = tr.replace(' { 28 | // 锁频率步长内取消请求 29 | const newTime = new Date().getTime() 30 | const shoudBack = newTime - currentTime < loopInterval * 1000 31 | if(shoudBack) { 32 | alert(loopInterval + '秒内不要多次点击哦。') 33 | return 34 | } 35 | currentTime = newTime 36 | xhr.onreadystatechange = function() { 37 | if(this.readyState === 4 && this.status === 200) { 38 | const res = JSON.parse(this.response) 39 | if(res.length < 1) { 40 | alert('今天结束的已经筛选完了') 41 | return 42 | } 43 | allItems = [...allItems, ...res] 44 | allItems.sort((a, b) => b.rate - a.rate) 45 | reFullTBody(allItems) 46 | currentPage-- 47 | } 48 | } 49 | xhr.open('post', '/table') 50 | xhr.setRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); 51 | //发送请求 52 | xhr.send("page=" + currentPage) 53 | } 54 | 55 | xhr.onreadystatechange = function() { 56 | if(this.readyState === 4 && this.status === 200) { 57 | currentPage = JSON.parse(this.response).pages 58 | postPage() 59 | } 60 | } 61 | xhr.open('get', '/total') 62 | xhr.send() 63 | -------------------------------------------------------------------------------- /es/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | A站搜索演示 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /es/readme.md: -------------------------------------------------------------------------------- 1 | 简陋版爬虫 -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "accepts": { 6 | "version": "1.3.7", 7 | "resolved": "https://registry.npm.taobao.org/accepts/download/accepts-1.3.7.tgz", 8 | "integrity": "sha1-UxvHJlF6OytB+FACHGzBXqq1B80=", 9 | "requires": { 10 | "mime-types": "~2.1.24", 11 | "negotiator": "0.6.2" 12 | } 13 | }, 14 | "array-flatten": { 15 | "version": "1.1.1", 16 | "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", 17 | "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" 18 | }, 19 | "asynckit": { 20 | "version": "0.4.0", 21 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 22 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 23 | }, 24 | "body-parser": { 25 | "version": "1.19.0", 26 | "resolved": "https://registry.npm.taobao.org/body-parser/download/body-parser-1.19.0.tgz", 27 | "integrity": "sha1-lrJwnlfJxOCab9Zqj9l5hE9p8Io=", 28 | "requires": { 29 | "bytes": "3.1.0", 30 | "content-type": "~1.0.4", 31 | "debug": "2.6.9", 32 | "depd": "~1.1.2", 33 | "http-errors": "1.7.2", 34 | "iconv-lite": "0.4.24", 35 | "on-finished": "~2.3.0", 36 | "qs": "6.7.0", 37 | "raw-body": "2.4.0", 38 | "type-is": "~1.6.17" 39 | } 40 | }, 41 | "boolbase": { 42 | "version": "1.0.0", 43 | "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", 44 | "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" 45 | }, 46 | "bytes": { 47 | "version": "3.1.0", 48 | "resolved": "https://registry.npm.taobao.org/bytes/download/bytes-3.1.0.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fbytes%2Fdownload%2Fbytes-3.1.0.tgz", 49 | "integrity": "sha1-9s95M6Ng4FiPqf3oVlHNx/gF0fY=" 50 | }, 51 | "cheerio": { 52 | "version": "1.0.0-rc.12", 53 | "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", 54 | "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", 55 | "requires": { 56 | "cheerio-select": "^2.1.0", 57 | "dom-serializer": "^2.0.0", 58 | "domhandler": "^5.0.3", 59 | "domutils": "^3.0.1", 60 | "htmlparser2": "^8.0.1", 61 | "parse5": "^7.0.0", 62 | "parse5-htmlparser2-tree-adapter": "^7.0.0" 63 | } 64 | }, 65 | "cheerio-select": { 66 | "version": "2.1.0", 67 | "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", 68 | "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", 69 | "requires": { 70 | "boolbase": "^1.0.0", 71 | "css-select": "^5.1.0", 72 | "css-what": "^6.1.0", 73 | "domelementtype": "^2.3.0", 74 | "domhandler": "^5.0.3", 75 | "domutils": "^3.0.1" 76 | } 77 | }, 78 | "combined-stream": { 79 | "version": "1.0.8", 80 | "resolved": "https://registry.npm.taobao.org/combined-stream/download/combined-stream-1.0.8.tgz", 81 | "integrity": "sha1-w9RaizT9cwYxoRCoolIGgrMdWn8=", 82 | "requires": { 83 | "delayed-stream": "~1.0.0" 84 | } 85 | }, 86 | "component-emitter": { 87 | "version": "1.3.0", 88 | "resolved": "https://registry.npm.taobao.org/component-emitter/download/component-emitter-1.3.0.tgz", 89 | "integrity": "sha1-FuQHD7qK4ptnnyIVhT7hgasuq8A=" 90 | }, 91 | "content-disposition": { 92 | "version": "0.5.3", 93 | "resolved": "https://registry.npm.taobao.org/content-disposition/download/content-disposition-0.5.3.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcontent-disposition%2Fdownload%2Fcontent-disposition-0.5.3.tgz", 94 | "integrity": "sha1-4TDK9+cnkIfFYWwgB9BIVpiYT70=", 95 | "requires": { 96 | "safe-buffer": "5.1.2" 97 | }, 98 | "dependencies": { 99 | "safe-buffer": { 100 | "version": "5.1.2", 101 | "resolved": "https://registry.npm.taobao.org/safe-buffer/download/safe-buffer-5.1.2.tgz", 102 | "integrity": "sha1-mR7GnSluAxN0fVm9/St0XDX4go0=" 103 | } 104 | } 105 | }, 106 | "content-type": { 107 | "version": "1.0.4", 108 | "resolved": "https://registry.npm.taobao.org/content-type/download/content-type-1.0.4.tgz", 109 | "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=" 110 | }, 111 | "cookie": { 112 | "version": "0.4.0", 113 | "resolved": "https://registry.npm.taobao.org/cookie/download/cookie-0.4.0.tgz", 114 | "integrity": "sha1-vrQ35wIrO21JAZ0IhmUwPr6cFLo=" 115 | }, 116 | "cookie-signature": { 117 | "version": "1.0.6", 118 | "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", 119 | "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" 120 | }, 121 | "cookiejar": { 122 | "version": "2.1.4", 123 | "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", 124 | "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==" 125 | }, 126 | "css-select": { 127 | "version": "5.1.0", 128 | "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", 129 | "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", 130 | "requires": { 131 | "boolbase": "^1.0.0", 132 | "css-what": "^6.1.0", 133 | "domhandler": "^5.0.2", 134 | "domutils": "^3.0.1", 135 | "nth-check": "^2.0.1" 136 | } 137 | }, 138 | "css-what": { 139 | "version": "6.1.0", 140 | "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", 141 | "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==" 142 | }, 143 | "debug": { 144 | "version": "2.6.9", 145 | "resolved": "https://registry.npm.taobao.org/debug/download/debug-2.6.9.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-2.6.9.tgz", 146 | "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", 147 | "requires": { 148 | "ms": "2.0.0" 149 | } 150 | }, 151 | "delayed-stream": { 152 | "version": "1.0.0", 153 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 154 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 155 | }, 156 | "depd": { 157 | "version": "1.1.2", 158 | "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", 159 | "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" 160 | }, 161 | "destroy": { 162 | "version": "1.0.4", 163 | "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", 164 | "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" 165 | }, 166 | "dom-serializer": { 167 | "version": "2.0.0", 168 | "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", 169 | "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", 170 | "requires": { 171 | "domelementtype": "^2.3.0", 172 | "domhandler": "^5.0.2", 173 | "entities": "^4.2.0" 174 | } 175 | }, 176 | "domelementtype": { 177 | "version": "2.3.0", 178 | "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", 179 | "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" 180 | }, 181 | "domhandler": { 182 | "version": "5.0.3", 183 | "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", 184 | "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", 185 | "requires": { 186 | "domelementtype": "^2.3.0" 187 | } 188 | }, 189 | "domutils": { 190 | "version": "3.0.1", 191 | "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz", 192 | "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==", 193 | "requires": { 194 | "dom-serializer": "^2.0.0", 195 | "domelementtype": "^2.3.0", 196 | "domhandler": "^5.0.1" 197 | } 198 | }, 199 | "ee-first": { 200 | "version": "1.1.1", 201 | "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", 202 | "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" 203 | }, 204 | "encodeurl": { 205 | "version": "1.0.2", 206 | "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", 207 | "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" 208 | }, 209 | "entities": { 210 | "version": "4.4.0", 211 | "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", 212 | "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" 213 | }, 214 | "escape-html": { 215 | "version": "1.0.3", 216 | "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", 217 | "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" 218 | }, 219 | "etag": { 220 | "version": "1.8.1", 221 | "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", 222 | "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" 223 | }, 224 | "express": { 225 | "version": "4.17.1", 226 | "resolved": "https://registry.npm.taobao.org/express/download/express-4.17.1.tgz", 227 | "integrity": "sha1-RJH8OGBc9R+GKdOcK10Cb5ikwTQ=", 228 | "requires": { 229 | "accepts": "~1.3.7", 230 | "array-flatten": "1.1.1", 231 | "body-parser": "1.19.0", 232 | "content-disposition": "0.5.3", 233 | "content-type": "~1.0.4", 234 | "cookie": "0.4.0", 235 | "cookie-signature": "1.0.6", 236 | "debug": "2.6.9", 237 | "depd": "~1.1.2", 238 | "encodeurl": "~1.0.2", 239 | "escape-html": "~1.0.3", 240 | "etag": "~1.8.1", 241 | "finalhandler": "~1.1.2", 242 | "fresh": "0.5.2", 243 | "merge-descriptors": "1.0.1", 244 | "methods": "~1.1.2", 245 | "on-finished": "~2.3.0", 246 | "parseurl": "~1.3.3", 247 | "path-to-regexp": "0.1.7", 248 | "proxy-addr": "~2.0.5", 249 | "qs": "6.7.0", 250 | "range-parser": "~1.2.1", 251 | "safe-buffer": "5.1.2", 252 | "send": "0.17.1", 253 | "serve-static": "1.14.1", 254 | "setprototypeof": "1.1.1", 255 | "statuses": "~1.5.0", 256 | "type-is": "~1.6.18", 257 | "utils-merge": "1.0.1", 258 | "vary": "~1.1.2" 259 | }, 260 | "dependencies": { 261 | "safe-buffer": { 262 | "version": "5.1.2", 263 | "resolved": "https://registry.npm.taobao.org/safe-buffer/download/safe-buffer-5.1.2.tgz", 264 | "integrity": "sha1-mR7GnSluAxN0fVm9/St0XDX4go0=" 265 | } 266 | } 267 | }, 268 | "finalhandler": { 269 | "version": "1.1.2", 270 | "resolved": "https://registry.npm.taobao.org/finalhandler/download/finalhandler-1.1.2.tgz", 271 | "integrity": "sha1-t+fQAP/RGTjQ/bBTUG9uur6fWH0=", 272 | "requires": { 273 | "debug": "2.6.9", 274 | "encodeurl": "~1.0.2", 275 | "escape-html": "~1.0.3", 276 | "on-finished": "~2.3.0", 277 | "parseurl": "~1.3.3", 278 | "statuses": "~1.5.0", 279 | "unpipe": "~1.0.0" 280 | } 281 | }, 282 | "form-data": { 283 | "version": "2.5.1", 284 | "resolved": "https://registry.npm.taobao.org/form-data/download/form-data-2.5.1.tgz?cache=0&sync_timestamp=1573027118125&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fform-data%2Fdownload%2Fform-data-2.5.1.tgz", 285 | "integrity": "sha1-8svsV7XlniNxbhKP5E1OXdI4lfQ=", 286 | "requires": { 287 | "asynckit": "^0.4.0", 288 | "combined-stream": "^1.0.6", 289 | "mime-types": "^2.1.12" 290 | } 291 | }, 292 | "formidable": { 293 | "version": "1.2.2", 294 | "resolved": "https://registry.npm.taobao.org/formidable/download/formidable-1.2.2.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fformidable%2Fdownload%2Fformidable-1.2.2.tgz", 295 | "integrity": "sha1-v2muopcpgmdfAIZTQrmCmG9rjdk=" 296 | }, 297 | "forwarded": { 298 | "version": "0.1.2", 299 | "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", 300 | "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" 301 | }, 302 | "fresh": { 303 | "version": "0.5.2", 304 | "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", 305 | "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" 306 | }, 307 | "htmlparser2": { 308 | "version": "8.0.1", 309 | "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.1.tgz", 310 | "integrity": "sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==", 311 | "requires": { 312 | "domelementtype": "^2.3.0", 313 | "domhandler": "^5.0.2", 314 | "domutils": "^3.0.1", 315 | "entities": "^4.3.0" 316 | } 317 | }, 318 | "http-errors": { 319 | "version": "1.7.2", 320 | "resolved": "https://registry.npm.taobao.org/http-errors/download/http-errors-1.7.2.tgz?cache=0&sync_timestamp=1561418493658&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fhttp-errors%2Fdownload%2Fhttp-errors-1.7.2.tgz", 321 | "integrity": "sha1-T1ApzxMjnzEDblsuVSkrz7zIXI8=", 322 | "requires": { 323 | "depd": "~1.1.2", 324 | "inherits": "2.0.3", 325 | "setprototypeof": "1.1.1", 326 | "statuses": ">= 1.5.0 < 2", 327 | "toidentifier": "1.0.0" 328 | }, 329 | "dependencies": { 330 | "inherits": { 331 | "version": "2.0.3", 332 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 333 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" 334 | } 335 | } 336 | }, 337 | "iconv-lite": { 338 | "version": "0.4.24", 339 | "resolved": "https://registry.npm.taobao.org/iconv-lite/download/iconv-lite-0.4.24.tgz?cache=0&sync_timestamp=1579334008444&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ficonv-lite%2Fdownload%2Ficonv-lite-0.4.24.tgz", 340 | "integrity": "sha1-ICK0sl+93CHS9SSXSkdKr+czkIs=", 341 | "requires": { 342 | "safer-buffer": ">= 2.1.2 < 3" 343 | } 344 | }, 345 | "inherits": { 346 | "version": "2.0.4", 347 | "resolved": "https://registry.npm.taobao.org/inherits/download/inherits-2.0.4.tgz?cache=0&sync_timestamp=1560975547815&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Finherits%2Fdownload%2Finherits-2.0.4.tgz", 348 | "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=" 349 | }, 350 | "ipaddr.js": { 351 | "version": "1.9.1", 352 | "resolved": "https://registry.npm.taobao.org/ipaddr.js/download/ipaddr.js-1.9.1.tgz", 353 | "integrity": "sha1-v/OFQ+64mEglB5/zoqjmy9RngbM=" 354 | }, 355 | "media-typer": { 356 | "version": "0.3.0", 357 | "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", 358 | "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" 359 | }, 360 | "merge-descriptors": { 361 | "version": "1.0.1", 362 | "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", 363 | "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" 364 | }, 365 | "methods": { 366 | "version": "1.1.2", 367 | "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", 368 | "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" 369 | }, 370 | "mime": { 371 | "version": "1.6.0", 372 | "resolved": "https://registry.npm.taobao.org/mime/download/mime-1.6.0.tgz?cache=0&sync_timestamp=1560034758817&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmime%2Fdownload%2Fmime-1.6.0.tgz", 373 | "integrity": "sha1-Ms2eXGRVO9WNGaVor0Uqz/BJgbE=" 374 | }, 375 | "mime-db": { 376 | "version": "1.43.0", 377 | "resolved": "https://registry.npm.taobao.org/mime-db/download/mime-db-1.43.0.tgz?cache=0&sync_timestamp=1578281136306&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmime-db%2Fdownload%2Fmime-db-1.43.0.tgz", 378 | "integrity": "sha1-ChLgUCZQ5HPXNVNQUOfI9OtPrlg=" 379 | }, 380 | "mime-types": { 381 | "version": "2.1.26", 382 | "resolved": "https://registry.npm.taobao.org/mime-types/download/mime-types-2.1.26.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmime-types%2Fdownload%2Fmime-types-2.1.26.tgz", 383 | "integrity": "sha1-nJIfwJt+FJpl39wNpNIJlyALCgY=", 384 | "requires": { 385 | "mime-db": "1.43.0" 386 | } 387 | }, 388 | "ms": { 389 | "version": "2.0.0", 390 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 391 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 392 | }, 393 | "negotiator": { 394 | "version": "0.6.2", 395 | "resolved": "https://registry.npm.taobao.org/negotiator/download/negotiator-0.6.2.tgz", 396 | "integrity": "sha1-/qz3zPUlp3rpY0Q2pkiD/+yjRvs=" 397 | }, 398 | "nth-check": { 399 | "version": "2.1.1", 400 | "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", 401 | "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", 402 | "requires": { 403 | "boolbase": "^1.0.0" 404 | } 405 | }, 406 | "on-finished": { 407 | "version": "2.3.0", 408 | "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", 409 | "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", 410 | "requires": { 411 | "ee-first": "1.1.1" 412 | } 413 | }, 414 | "parse5": { 415 | "version": "7.1.2", 416 | "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", 417 | "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", 418 | "requires": { 419 | "entities": "^4.4.0" 420 | } 421 | }, 422 | "parse5-htmlparser2-tree-adapter": { 423 | "version": "7.0.0", 424 | "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", 425 | "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", 426 | "requires": { 427 | "domhandler": "^5.0.2", 428 | "parse5": "^7.0.0" 429 | } 430 | }, 431 | "parseurl": { 432 | "version": "1.3.3", 433 | "resolved": "https://registry.npm.taobao.org/parseurl/download/parseurl-1.3.3.tgz", 434 | "integrity": "sha1-naGee+6NEt/wUT7Vt2lXeTvC6NQ=" 435 | }, 436 | "path-to-regexp": { 437 | "version": "0.1.7", 438 | "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", 439 | "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" 440 | }, 441 | "proxy-addr": { 442 | "version": "2.0.6", 443 | "resolved": "https://registry.npm.taobao.org/proxy-addr/download/proxy-addr-2.0.6.tgz", 444 | "integrity": "sha1-/cIzZQVEfT8vLGOO0nLK9hS7sr8=", 445 | "requires": { 446 | "forwarded": "~0.1.2", 447 | "ipaddr.js": "1.9.1" 448 | } 449 | }, 450 | "qs": { 451 | "version": "6.7.0", 452 | "resolved": "https://registry.npm.taobao.org/qs/download/qs-6.7.0.tgz", 453 | "integrity": "sha1-QdwaAV49WB8WIXdr4xr7KHapsbw=" 454 | }, 455 | "range-parser": { 456 | "version": "1.2.1", 457 | "resolved": "https://registry.npm.taobao.org/range-parser/download/range-parser-1.2.1.tgz", 458 | "integrity": "sha1-PPNwI9GZ4cJNGlW4SADC8+ZGgDE=" 459 | }, 460 | "raw-body": { 461 | "version": "2.4.0", 462 | "resolved": "https://registry.npm.taobao.org/raw-body/download/raw-body-2.4.0.tgz", 463 | "integrity": "sha1-oc5vucm8NWylLoklarWQWeE9AzI=", 464 | "requires": { 465 | "bytes": "3.1.0", 466 | "http-errors": "1.7.2", 467 | "iconv-lite": "0.4.24", 468 | "unpipe": "1.0.0" 469 | } 470 | }, 471 | "readable-stream": { 472 | "version": "3.6.0", 473 | "resolved": "https://registry.npm.taobao.org/readable-stream/download/readable-stream-3.6.0.tgz?cache=0&sync_timestamp=1581624324274&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Freadable-stream%2Fdownload%2Freadable-stream-3.6.0.tgz", 474 | "integrity": "sha1-M3u9o63AcGvT4CRCaihtS0sskZg=", 475 | "requires": { 476 | "inherits": "^2.0.3", 477 | "string_decoder": "^1.1.1", 478 | "util-deprecate": "^1.0.1" 479 | } 480 | }, 481 | "safe-buffer": { 482 | "version": "5.2.0", 483 | "resolved": "https://registry.npm.taobao.org/safe-buffer/download/safe-buffer-5.2.0.tgz", 484 | "integrity": "sha1-t02uxJsRSPiMZLaNSbHoFcHy9Rk=" 485 | }, 486 | "safer-buffer": { 487 | "version": "2.1.2", 488 | "resolved": "https://registry.npm.taobao.org/safer-buffer/download/safer-buffer-2.1.2.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fsafer-buffer%2Fdownload%2Fsafer-buffer-2.1.2.tgz", 489 | "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" 490 | }, 491 | "send": { 492 | "version": "0.17.1", 493 | "resolved": "https://registry.npm.taobao.org/send/download/send-0.17.1.tgz", 494 | "integrity": "sha1-wdiwWfeQD3Rm3Uk4vcROEd2zdsg=", 495 | "requires": { 496 | "debug": "2.6.9", 497 | "depd": "~1.1.2", 498 | "destroy": "~1.0.4", 499 | "encodeurl": "~1.0.2", 500 | "escape-html": "~1.0.3", 501 | "etag": "~1.8.1", 502 | "fresh": "0.5.2", 503 | "http-errors": "~1.7.2", 504 | "mime": "1.6.0", 505 | "ms": "2.1.1", 506 | "on-finished": "~2.3.0", 507 | "range-parser": "~1.2.1", 508 | "statuses": "~1.5.0" 509 | }, 510 | "dependencies": { 511 | "ms": { 512 | "version": "2.1.1", 513 | "resolved": "https://registry.npm.taobao.org/ms/download/ms-2.1.1.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fms%2Fdownload%2Fms-2.1.1.tgz", 514 | "integrity": "sha1-MKWGTrPrsKZvLr5tcnrwagnYbgo=" 515 | } 516 | } 517 | }, 518 | "serve-static": { 519 | "version": "1.14.1", 520 | "resolved": "https://registry.npm.taobao.org/serve-static/download/serve-static-1.14.1.tgz", 521 | "integrity": "sha1-Zm5jbcTwEPfvKZcKiKZ0MgiYsvk=", 522 | "requires": { 523 | "encodeurl": "~1.0.2", 524 | "escape-html": "~1.0.3", 525 | "parseurl": "~1.3.3", 526 | "send": "0.17.1" 527 | } 528 | }, 529 | "setprototypeof": { 530 | "version": "1.1.1", 531 | "resolved": "https://registry.npm.taobao.org/setprototypeof/download/setprototypeof-1.1.1.tgz?cache=0&sync_timestamp=1563425414995&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fsetprototypeof%2Fdownload%2Fsetprototypeof-1.1.1.tgz", 532 | "integrity": "sha1-fpWsskqpL1iF4KvvW6ExMw1K5oM=" 533 | }, 534 | "statuses": { 535 | "version": "1.5.0", 536 | "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", 537 | "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" 538 | }, 539 | "string_decoder": { 540 | "version": "1.3.0", 541 | "resolved": "https://registry.npm.taobao.org/string_decoder/download/string_decoder-1.3.0.tgz", 542 | "integrity": "sha1-QvEUWUpGzxqOMLCoT1bHjD7awh4=", 543 | "requires": { 544 | "safe-buffer": "~5.2.0" 545 | } 546 | }, 547 | "superagent": { 548 | "version": "4.0.0-beta.5", 549 | "resolved": "https://registry.npm.taobao.org/superagent/download/superagent-4.0.0-beta.5.tgz", 550 | "integrity": "sha1-RBqJIUETUV6H0vVF/I3Qiqowyyk=", 551 | "requires": { 552 | "component-emitter": "^1.2.0", 553 | "cookiejar": "^2.1.2", 554 | "debug": "^4.0.0", 555 | "form-data": "^2.3.2", 556 | "formidable": "^1.2.0", 557 | "methods": "^1.1.1", 558 | "mime": "^2.0.3", 559 | "qs": "^6.5.1", 560 | "readable-stream": "^3.0.3" 561 | }, 562 | "dependencies": { 563 | "debug": { 564 | "version": "4.1.1", 565 | "resolved": "https://registry.npm.taobao.org/debug/download/debug-4.1.1.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-4.1.1.tgz", 566 | "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", 567 | "requires": { 568 | "ms": "^2.1.1" 569 | } 570 | }, 571 | "mime": { 572 | "version": "2.4.4", 573 | "resolved": "https://registry.npm.taobao.org/mime/download/mime-2.4.4.tgz?cache=0&sync_timestamp=1560034758817&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmime%2Fdownload%2Fmime-2.4.4.tgz", 574 | "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" 575 | }, 576 | "ms": { 577 | "version": "2.1.2", 578 | "resolved": "https://registry.npm.taobao.org/ms/download/ms-2.1.2.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fms%2Fdownload%2Fms-2.1.2.tgz", 579 | "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" 580 | } 581 | } 582 | }, 583 | "superagent-charset": { 584 | "version": "1.2.0", 585 | "resolved": "https://registry.npm.taobao.org/superagent-charset/download/superagent-charset-1.2.0.tgz", 586 | "integrity": "sha1-nuulqSdfbxzT1rE4YflEY/PDo0g=", 587 | "requires": { 588 | "iconv-lite": "^0.4.16" 589 | } 590 | }, 591 | "toidentifier": { 592 | "version": "1.0.0", 593 | "resolved": "https://registry.npm.taobao.org/toidentifier/download/toidentifier-1.0.0.tgz", 594 | "integrity": "sha1-fhvjRw8ed5SLxD2Uo8j013UrpVM=" 595 | }, 596 | "type-is": { 597 | "version": "1.6.18", 598 | "resolved": "https://registry.npm.taobao.org/type-is/download/type-is-1.6.18.tgz", 599 | "integrity": "sha1-TlUs0F3wlGfcvE73Od6J8s83wTE=", 600 | "requires": { 601 | "media-typer": "0.3.0", 602 | "mime-types": "~2.1.24" 603 | } 604 | }, 605 | "unpipe": { 606 | "version": "1.0.0", 607 | "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", 608 | "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" 609 | }, 610 | "util-deprecate": { 611 | "version": "1.0.2", 612 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 613 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 614 | }, 615 | "utils-merge": { 616 | "version": "1.0.1", 617 | "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", 618 | "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" 619 | }, 620 | "vary": { 621 | "version": "1.1.2", 622 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 623 | "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" 624 | } 625 | } 626 | } 627 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "ali_sort": "node ali_sort/main.js", 4 | "pc": "node pixiv_col/main.js", 5 | "89wxw": "node 89wxw/main.js" 6 | }, 7 | "dependencies": { 8 | "cheerio": "^1.0.0-rc.12", 9 | "express": "^4.17.1", 10 | "superagent": "^4.0.0-beta.5", 11 | "superagent-charset": "^1.2.0" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /pixiv_col/config/index.js: -------------------------------------------------------------------------------- 1 | const cookie = require('./cookie') 2 | 3 | module.exports = { 4 | host: 'localhost', 5 | port: 8000, 6 | // 多久一张 7 | timeout: 3000, 8 | // 自行填充 9 | cookie, 10 | // 下面不用改 11 | referer: 'https://www.pixiv.net/member_illust.php?mode=medium&illust_id=70108489', 12 | } 13 | -------------------------------------------------------------------------------- /pixiv_col/main.js: -------------------------------------------------------------------------------- 1 | const config = require('./config/index') 2 | const request = require('superagent') 3 | const fs = require('fs') 4 | const cheerio = require('cheerio') 5 | const exec = require('child_process').exec 6 | let page = 26 // 起始页面，如果中间断了，可以从这里连上去 7 | let pageCurrentLength = 0 8 | let pageMaxLength = 1 9 | let total = 0 10 | 11 | // 清空上次记录 12 | exec(`> pixiv_col/progress.txt`) 13 | exec(`echo ${new Date().toLocaleString()} > pixiv_col/progress.txt`) 14 | 15 | /** 16 | * cookie配置在：./config/index.js -> cookie后面的引号内 17 | * 获取方式，登录Pixiv后，打开控制台刷新查看NetWork->Doc里面的.php后缀请求，找到Request相关，里面有cookie，复制来就行 18 | * 19 | * 请自行配置你的cookie！！！ 20 | * 请自行配置你的cookie！！！ 21 | * 请自行配置你的cookie！！！ 22 | * 请自行配置你的cookie！！！ 23 | * 请自行配置你的cookie！！！ 24 | * 请自行配置你的cookie！！！ 25 | * 请自行配置你的cookie！！！ 26 | */ 27 | 28 | /** 29 | * 获取每一页的内容 30 | */ 31 | function getPageImgs() { 32 | // 创建关于页的文件夹 33 | exec(`mkdir pixiv_col/imgs/${page}/`) 34 | 35 | pageCurrentLength = 0 36 | request 37 | .get(`https://www.pixiv.net/bookmark.php?rest=show&p=${page}`) 38 | .set('cookie', config.cookie) 39 | .set('referer', config.referer) 40 | .end((err, res) => { 41 | if(err) { 42 | console.log(err) 43 | } 44 | const result = res.text 45 | // 加载为伪JQ 46 | $ = cheerio.load(result) 47 | let thumbnails = $('._layout-thumbnail>img') 48 | let titles = $('a>h1.title') 49 | // 获取总页数 50 | if(!total) total = +$('.bookmark-tag-all')[0].firstChild.data.match(/\d+/)[0] 51 | 52 | pageMaxLength = thumbnails.length 53 | Array.from(thumbnails).forEach((thumbnail, index) => { 54 | let imgUrl = thumbnail.attribs['data-src'] 55 | // 作者删除资源导致爆掉问题处理 56 | try { 57 | let imgFix = imgUrl.match(/\.\w+$/)[0] 58 | let imgTitle = titles[index].attribs.title 59 | let imgName = imgTitle + imgFix 60 | let originImgUrl = imgUrl.replace(/\/c\/150x150\/img-master(.*?)_master1200/, '/img-original$1') 61 | 62 | checkLinkStatus({ imgTitle, imgName, originImgUrl }) 63 | } catch(error) { 64 | pageCurrentLength++ 65 | } 66 | }) 67 | }) 68 | } 69 | 70 | function checkLinkStatus({ imgTitle, imgName, originImgUrl }) { 71 | request.head(originImgUrl) 72 | .set('cookie', config.cookie) 73 | .set('referer', config.referer) 74 | .end((err, res) => { 75 | // 那啥，缩略图跟实际大小图，后缀不一定一样，你可以自己浏览几张看看就知道了 76 | if(err) { 77 | let originImgUrlFix = originImgUrl.match(/\.\w+$/)[0] 78 | let png = '.png' 79 | let jpg = '.jpg' 80 | if(originImgUrlFix === png) { 81 | imgName = imgTitle + jpg 82 | originImgUrl = originImgUrl.replace(/\.\w+$/, jpg) 83 | } else { 84 | imgName = imgTitle + png 85 | originImgUrl = originImgUrl.replace(/\.\w+$/, png) 86 | } 87 | } 88 | // 每三秒处理一张，别太急，容易出问题。。。比如，封号 89 | downloadFile({ imgTitle, imgName, originImgUrl }) 90 | }) 91 | } 92 | 93 | function downloadFile({ imgName, originImgUrl }) { 94 | // 写入文件夹 95 | let filePath = `./pixiv_col/imgs/${page}/${imgName.replace(/\//g, '')}` 96 | let stream = fs.createWriteStream(filePath) 97 | let req = request.get(originImgUrl) 98 | .set('cookie', config.cookie) 99 | .set('referer', config.referer) 100 | 101 | req.pipe(stream) 102 | 103 | req.on('end', () => { 104 | let downloadsLength = pageCurrentLength + 20 * (page - 1) 105 | // 保存每一个完成的进度日志 106 | exec(`echo ${downloadsLength}/${total} --- ${imgName} >> pixiv_col/progress.txt`) 107 | console.log(`${downloadsLength}/${total} --- ${imgName}`) 108 | 109 | // 超过当前页数的数量，就进行下一页 110 | if(++pageCurrentLength >= pageMaxLength) { 111 | // 保存每一页完成的进度日志 112 | exec(`echo ---- page${page} have done!!! ---- >> pixiv_col/progress.txt`) 113 | console.log(`---- page${page} have done!!! ----`) 114 | getPageImgs(++page) 115 | } 116 | }) 117 | } 118 | 119 | getPageImgs() -------------------------------------------------------------------------------- /static/img/89wxw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kvker/js_spider/2cf43639962186756a791c51d6c14897f000a959/static/img/89wxw.png -------------------------------------------------------------------------------- /static/img/ali_sort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kvker/js_spider/2cf43639962186756a791c51d6c14897f000a959/static/img/ali_sort.png -------------------------------------------------------------------------------- /static/img/pixiv_col.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kvker/js_spider/2cf43639962186756a791c51d6c14897f000a959/static/img/pixiv_col.png -------------------------------------------------------------------------------- /wx/README.md: -------------------------------------------------------------------------------- 1 | # 油猴_微信公众号后台爬取其他公众号文章列表 2 | 3 | #### 介绍 4 | 油猴脚本获取某公众号列表href, 也可以自定义条件等等, 源码简单一批 5 | 6 | ## 使用 7 | 1. 公众号后台登录 8 | 2. 新建群发 9 | 3. 自建图文 10 | 4. 顶部点击"超链接" 11 | 5. 选择其他公众号 12 | 6. 公众号列表点击需要的公众号 13 | 7. 文章列表出现后, 如果已经安装此油猴脚本, 右上角有个开始按钮 14 | 8. 点击开始, 然后就不要动了, 脚本会自动翻页保存 15 | 9. 完成后, 会在开始按钮下面显示一个文本域, 里面全选复制就能拿到全部的文章链接了, 数据模型是Array. 16 | -------------------------------------------------------------------------------- /wx/auto_artile.js: -------------------------------------------------------------------------------- 1 | // 油猴脚本获取某公众号列表href, 也可以自定义条件等等, 源码简单一批 2 | 3 | (function() { 4 | 'use strict'; 5 | 6 | // Your code here... 7 | 8 | // 链接列表 9 | let list = [] 10 | 11 | // 点开超链接后再点击... 12 | let start_btn = ` 13 | 14 | ` 15 | let end_ele = ` 16 | 17 | ` 18 | document.body.insertAdjacentHTML('afterend', start_btn) 19 | 20 | window.patchListPluginStart = function start() { 21 | console.log('start') 22 | patch() 23 | } 24 | 25 | window.patchListPluginEnd = function end() { 26 | document.body.insertAdjacentHTML('afterend', end_ele) 27 | document.querySelector('#result_text_area').value = JSON.stringify(list) 28 | console.log('end list length: ' + list.length) 29 | } 30 | 31 | function patch() { 32 | // 下一页按钮 33 | let next_ctrl = getNextPageCtrl() 34 | // 获取a标签 35 | let as = getAs() 36 | // 获取标题, 可选 37 | let titles = getTitles() 38 | as.forEach(function(a, idx) { 39 | // 除链接外其他处理 start 40 | if(titles[idx].innerText.includes('B站')) { 41 | list.push(a.href) 42 | } 43 | // end 44 | }) 45 | console.log('list length: ' + list.length) 46 | 47 | // 有页码则翻页 48 | if(document.querySelector('.weui-desktop-pagination__num')) { 49 | let current_page = $('.weui-desktop-pagination__num')[0].innerText 50 | let max_page = $('.weui-desktop-pagination__num')[1].innerText 51 | // 当前页码等于最大页码则结束 52 | if(current_page === max_page) { 53 | patchListPluginEnd() 54 | return 55 | } else { 56 | // 点击下一页事件 57 | if(next_ctrl) { 58 | next_ctrl.click() 59 | // 延时2s后开始采集并插入列表 60 | setTimeout(function() { 61 | patch() 62 | }, 2000) 63 | } 64 | } 65 | } else { // 没页码直接结束 66 | patchListPluginEnd() 67 | return 68 | } 69 | } 70 | 71 | // 获取下一页按钮 72 | function getNextPageCtrl() { 73 | let as = document.querySelectorAll('.weui-desktop-btn.weui-desktop-btn_default.weui-desktop-btn_mini') 74 | // 第一页只有一个下一页 75 | return as[1] || as[0] 76 | } 77 | 78 | // 获取当前列表的a标签 79 | function getAs() { 80 | return document.querySelectorAll('.weui-desktop-vm_default a') 81 | } 82 | 83 | // 获取标题 84 | function getTitles(){ 85 | return document.querySelectorAll('.inner_link_article_title') 86 | } 87 | })() 88 | --------------------------------------------------------------------------------