├── assets
├── mimetype
├── container.xml
├── toc.ejs
├── content.ejs
└── Style.css
├── .gitignore
├── package.json
├── README.md
├── uuid.js
└── jianhelper.js
/assets/mimetype:
--------------------------------------------------------------------------------
1 | application/epub+zip
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | out
3 | out.epub
--------------------------------------------------------------------------------
/assets/container.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "jianhelper",
3 | "version": "1.0.0",
4 | "description": "A downloader for articles in jianshu.com.",
5 | "main": "jianhelper.js",
6 | "author": "wizardforcel",
7 | "license": "MIT",
8 | "dependencies": {
9 | "cheerio": "^0.19.0",
10 | "ejs": "^2.4.1",
11 | "jszip": "^2.5.0",
12 | "sync-request": "^2.1.0"
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 简书助手
2 |
3 | 爬取简书的文章，并生成EPUB格式。
4 |
5 | ## 用法
6 |
7 | 首先到[官网](https://nodejs.org/en/download/)下载并安装node.js。
8 |
9 | ```
10 | git clone https://github.com/wizardforcel/jianhelper.git
11 | cd jianhelper
12 | npm install
13 | node jianhelper url [start [end]]
14 | ```
15 |
16 | url：支持三种类型
17 |
18 | + http://www.jianshu.com/users/{id} 用户
19 | + http://www.jianshu.com/notebooks/{id} 文集
20 | + http://www.jianshu.com/collection/{id} 专题
21 |
22 | start：起始页，默认为第一页
23 |
24 | end：终止页，默认为最后一页
25 |
26 | ## TODO
27 |
28 | + 添加标题、封面
29 |
30 | ## 协议
31 |
32 | MIT-LICENSE
33 |
--------------------------------------------------------------------------------
/assets/toc.ejs:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 | Unknown
12 |
13 |
14 | <% for(var i = 0; i < toc.length; i++) {%>
15 |
16 |
17 | <%- toc[i].title %>
18 |
19 |
20 |
21 | <% } %>
22 |
23 |
--------------------------------------------------------------------------------
/assets/content.ejs:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | urn:uuid:<%- uuid %>
5 |
6 | zh-CN
7 | <%- date %>
8 |
9 |
10 | (.+?)<\/h1>/.exec(co)[1];
69 | toc.push({file: fname, title: title});
70 | console.log('article: ' + fname + ', title: ' + title);
71 | } catch(ex) { console.log(ex.toString()); }
72 | }
73 | }
74 |
75 | var uuidGenerator = require('./uuid.js');
76 | var uuid = uuidGenerator.uuid();
77 |
78 | console.log('Generate content.opf...')
79 | var contentOpf = getContentOpf(toc, uuid);
80 | fs.writeFileSync('./out/OEBPS/content.opf', contentOpf, {encoding: 'utf-8'})
81 |
82 | console.log('Generate toc.ncx...')
83 | var tocNcx = getTocNcx(toc, uuid);
84 | fs.writeFileSync('./out/OEBPS/toc.ncx', tocNcx, {encoding: 'utf-8'})
85 |
86 | console.log('Gnerate epub...');
87 | generate();
88 |
89 | console.log('Done..');
90 |
91 | function showUsage() {
92 | var usage = "用法：node jianhelper url [start [end]]\n\n" +
93 | " url：支持三种类型\n\n" +
94 | " http://www.jianshu.com/users/{id} 用户\n" +
95 | " http://www.jianshu.com/notebooks/{id} 文集\n" +
96 | " http://www.jianshu.com/collection/{id} 专题\n\n" +
97 | " start：起始页，默认为第一页\n\n" +
98 | " end：终止页，默认为最后一页";
99 | console.log(usage);
100 | }
101 |
102 | function checkUrl(url)
103 | {
104 | var regexes = {
105 | user: /^https?:\/\/www\.jianshu\.com\/users\/(\w{6,12})\/?$/,
106 | notebook: /^https?:\/\/www\.jianshu\.com\/notebooks\/(\d+)\/?$/,
107 | collection: /^https?:\/\/www\.jianshu\.com\/collection\/(\w{6,12})\/?$/
108 | };
109 |
110 | var type = types.unknown;
111 | var id;
112 |
113 | for(var k in regexes)
114 | {
115 | var rms;
116 | if(rms = regexes[k].exec(url))
117 | {
118 | type = types[k];
119 | id = rms[1];
120 | break;
121 | }
122 | }
123 |
124 | return {type: type, id: id};
125 | }
126 |
127 | function getRealUrl(type, id, url)
128 | {
129 | if(type == types.user)
130 | {
131 | return 'http://www.jianshu.com/users/' + id +
132 | '/latest_articles?page={page}';
133 | }
134 | else if(type == types.notebook)
135 | {
136 | return 'http://www.jianshu.com/notebooks/' + id +
137 | '/latest?page={page}';
138 | }
139 | else if(type == types.collection)
140 | {
141 | var content = request('GET', url).getBody().toString();;
142 | var realId = /href="\/collections\/(\d+)\//.exec(content)[1];
143 | return 'http://www.jianshu.com/collections/' + realId +
144 | '/notes?order_by=added_at&page={page}';
145 | }
146 | }
147 |
148 | function getList(html) {
149 |
150 | var $ = cheerio.load(html);
151 |
152 | var $list = $(selectors.article);
153 | var res = [];
154 | for(var i = 0; i < $list.length; i++)
155 | {
156 | var url = $list.eq(i).attr('href');
157 | if(!url.startsWith('http'))
158 | url = 'http://www.jianshu.com' + url;
159 | res.push(url);
160 | }
161 | return res;
162 | }
163 |
164 |
165 | function getContent(html) {
166 | var $ = cheerio.load(html);
167 | dealWithImg($);
168 |
169 | var header = '\r\n\r\n\r\n' +
170 | '' +
171 | '\r\n\r\n';
172 |
173 | var title = '' + $(selectors.title).text() + '
';
174 | var author = '
作者：' + $(selectors.author).text() + '
';
175 | var content = $(selectors.content).html();
176 |
177 | var footer = '\r\n\r\n';
178 |
179 | return header + title + '\n' + author + '\n' + content + footer;
180 | }
181 |
182 | function initPath()
183 | {
184 | if(fs.existsSync('./out') &&
185 | fs.statSync('./out').isDirectory())
186 | rRmDir('./out');
187 | fs.mkdirSync('./out');
188 | fs.mkdirSync('./out/OEBPS');
189 | fs.mkdirSync('./out/OEBPS/Text');
190 | fs.mkdirSync('./out/OEBPS/Images');
191 | //try {fs.mkdirSync('./out/OEBPS/Styles');} catch(ex) {}
192 | //try {fs.mkdirSync('./out/META-INF');} catch(ex) {}
193 | //fs.writeFileSync('./out/META-INF/container.xml', fs.readFileSync('./assets/container.xml'));
194 | //fs.writeFileSync('./out/mimetype', fs.readFileSync('./assets/mimetype'));
195 | //fs.writeFileSync('./out/OEBPS/Styles/Style.css', fs.readFileSync('./assets/Style.css'));
196 | }
197 |
198 | function dealWithImg($)
199 | {
200 | var imgs = $(selectors.content + ' img');
201 | for(var i = 0; i < imgs.length; i++)
202 | {
203 | try {
204 | var img = imgs.eq(i);
205 | var url = img.attr('src');
206 | var co = request('GET', url).getBody();
207 | var fname = /[\w\-]+\.(?:jpg|png|gif)/.exec(url)[0];
208 | console.log('img: ' + fname);
209 | fs.writeFileSync('./out/OEBPS/Images/' + fname, co);
210 | img.attr('src', '../Images/' + fname);
211 | } catch(ex) { console.log(ex.toString()); }
212 | }
213 | }
214 |
215 | function getContentOpf(toc, uuid)
216 | {
217 | var date = new Date();
218 | var dateStr
219 | = date.getFullYear() + "-" + (date.getMonth() + 1) + "-" + date.getDate();
220 |
221 | return ejs.render(fs.readFileSync('assets/content.ejs', 'utf-8'), {
222 | date: dateStr,
223 | toc: toc,
224 | uuid: uuid
225 | });
226 | }
227 |
228 | function getTocNcx(toc, uuid)
229 | {
230 | return ejs.render(fs.readFileSync('assets/toc.ejs', 'utf-8'), {
231 | toc: toc,
232 | uuid: uuid
233 | });
234 | }
235 |
236 | function generate()
237 | {
238 | var zip = new jszip();
239 | zip.file('mimetype', fs.readFileSync('./assets/mimetype'));
240 | zip.file('META-INF/container.xml', fs.readFileSync('./assets/container.xml'));
241 | zip.file('OEBPS/Styles/Style.css', fs.readFileSync('./assets/Style.css'));
242 | zip.file('OEBPS/content.opf', fs.readFileSync('./out/OEBPS/content.opf'));
243 | zip.file('OEBPS/toc.ncx', fs.readFileSync('./out/OEBPS/toc.ncx'));
244 |
245 | var articles = fs.readdirSync('./out/OEBPS/Text');
246 | for(var i = 0; i < articles.length; i++)
247 | {
248 | var fname = articles[i];
249 | zip.file('OEBPS/Text/' + fname, fs.readFileSync('./out/OEBPS/Text/' + fname));
250 | }
251 |
252 | var images = fs.readdirSync('./out/OEBPS/Images');
253 | for(var i = 0; i < images.length; i++)
254 | {
255 | var fname = images[i];
256 | zip.file('OEBPS/Images/' + fname, fs.readFileSync('./out/OEBPS/Images/' + fname));
257 | }
258 |
259 | fs.writeFileSync('out.epub', zip.generate({type: 'nodebuffer', 'compression':'DEFLATE'}));
260 | }
261 |
262 | function rRmDir(dir)
263 | {
264 | var flist = fs.readdirSync(dir);
265 | for(var i = 0; i < flist.length; i++)
266 | {
267 | var item = dir + '/' + flist[i];
268 | if(fs.statSync(item).isDirectory())
269 | rRmDir(item);
270 | else
271 | fs.unlinkSync(item);
272 | }
273 | fs.rmdirSync(dir);
274 | }
--------------------------------------------------------------------------------