22 |
23 | Options:
24 | --help Show help [boolean]
25 | --version Show version number [boolean]
26 | --input-file, -i medium export zip file [required]
27 | --output-dir, -o target folder for the converted files [required]
28 | --concurrency, -c number of posts to process in parallel [default: 1]
29 | --download-images, -d should images be downloaded [default: true]
30 | --import-drafts, -r should drafts be imported [default: true]
31 |
32 | Examples:
33 | medium-importer -i medium-export.zip -o output/
34 | ```
35 |
36 | # Importer output
37 |
38 | The output directory is populated with the imported data, following a structure that makes it straightforward to later use with an SSG. For certain SSG's (like Hugo) it's possible to point the utility directly at your project.
39 |
40 | The output directories can be configured when working with the library directly.
41 |
42 | ### `content/posts`
43 |
44 | Markdown pages representing imported Medium posts.
45 |
46 | ```yaml
47 | ---
48 | template: post
49 | title: My Blog Post
50 | date: 2018-10-08T15:01:02.452Z
51 | subtitle: >-
52 | This post is a posty post, and this is its subtitle.
53 | excerpt: >-
54 | An excerpt of the post
55 | thumb_img_path: >-
56 | images/My-Blog-Post/image.jpeg
57 | content_img_path: >-
58 | images/My-Blog-Post/splash.jpeg
59 | ---
60 |
61 | # My Post
62 |
63 | Markdown content goes **HERE**
64 | ```
65 |
66 | ### `static/images`
67 |
68 | * Per-post images organized in directories with the post's slug.
69 | * Global images extracted from the Medium export: `avatar.png` if available.
70 |
71 |
72 | ### `data/data.json`
73 |
74 | Extraced information in `JSON` format:
75 |
76 | ```json
77 | {
78 | "author": {
79 | "name": "My Name",
80 | "email": "email@example.com",
81 | "avatar": "images/avatar.png"
82 | },
83 | "social": {
84 | "twitter": {
85 | "username": "@myuser",
86 | "url": "https://twitter.com/@myuser"
87 | }
88 | }
89 | ```
90 |
91 |
--------------------------------------------------------------------------------
/src/converter.js:
--------------------------------------------------------------------------------
1 |
2 |
3 | const sanitizeHtml = require('sanitize-html');
4 | const TurndownService = require('turndown');
5 |
6 | function processHtml(html, slug, imageMap) {
7 | return sanitizeHtml(html, {
8 | allowedTags: sanitizeHtml.defaults.allowedTags.concat(['figcaption', 'img', 'iframe', 'script', 'hr', 'blockquote', 'pre']),
9 | allowedAttributes: {
10 | iframe: ['*'],
11 | img: ['*'],
12 | a: ['*'],
13 | div: ['class'],
14 | h3: ['class'],
15 | script: ['*'],
16 | blockquote: ['class']
17 | },
18 | transformTags: {
19 | 'img': (tagName, attribs) => {
20 | const newAttribs = Object.assign(attribs, {
21 | src: imageMap.has(attribs.src)
22 | ? `/images/${slug}/${imageMap.get(attribs.src)}`
23 | : attribs.src
24 | });
25 | return {
26 | tagName: tagName,
27 | attribs: newAttribs
28 | };
29 | },
30 | 'blockquote': (tagName, attribs) => {
31 | let result = {
32 | tagName: tagName,
33 | attribs: attribs,
34 | };
35 | if (attribs.class === 'twitter-tweet') {
36 | // placeholder needed to get rendered properly
37 | result.text = 'tweet';
38 | }
39 | return result;
40 | },
41 | },
42 | exclusiveFilter: function(frame) {
43 | return (frame.tag === 'h3' &&
44 | frame.attribs['class'].includes('graf--title')) ||
45 | (frame.tag === 'h4' &&
46 | frame.attribs['class'].includes('graf--subtitle'));
47 | }
48 | });
49 | }
50 |
51 | function convertToMarkdown(html) {
52 |
53 | // workaround issue with multiline emphasis
54 | let bodyHtml = html.replace(/
<\/em>/gmi, '
');
55 |
56 | // workaround issue with pre blocks
57 | bodyHtml = bodyHtml.replace(//gmi, '');
58 | bodyHtml = bodyHtml.replace(/<\/pre>/gmi, '
');
59 |
60 | // unicode spaces are causing issues with whitespace detection
61 | bodyHtml = bodyHtml.replace(/[\u2000-\u200D]+/gmi, ' ');
62 |
63 | const turndownService = new TurndownService({
64 | emDelimiter: '*'
65 | });
66 | turndownService.keep(['iframe', 'figcaption', 'script']);
67 | turndownService.addRule('twitter-tweet', {
68 | filter: (node) => {
69 | return node.nodeName === 'BLOCKQUOTE' &&
70 | node.getAttribute('class') === 'twitter-tweet';
71 | },
72 | replacement: (innerHTML, node) => node.outerHTML
73 | });
74 | turndownService.addRule('pre', {
75 | filter: ['pre'],
76 | replacement: (content) => {
77 | return '\n ' + content.split('\n').join('\n ');
78 | }
79 | });
80 |
81 | const bodyMarkdown = turndownService.turndown(bodyHtml);
82 |
83 | return bodyMarkdown;
84 | }
85 |
86 | function convert(html, slug, imageMap) {
87 | const processedHtml = processHtml(html, slug, imageMap);
88 |
89 | const markdown = convertToMarkdown(processedHtml);
90 |
91 | return markdown;
92 | }
93 |
94 | module.exports = convert;
--------------------------------------------------------------------------------
/src/importer.js:
--------------------------------------------------------------------------------
1 | const _ = require('lodash');
2 | const fs = require('fs');
3 | const path = require('path');
4 | const pAll = require('p-all');
5 | const unzipper = require('unzipper');
6 | const matter = require('gray-matter');
7 | const chalk = require('chalk');
8 |
9 | const Errors = require('./errors');
10 | const Post = require('./model/post');
11 | const Profile = require('./model/profile');
12 | const downloadImages = require('./image-downloader');
13 | const convertHtml = require('./converter');
14 |
15 | function unzipInput(inputFilePath, outputDir) {
16 | return fs.createReadStream(inputFilePath)
17 | .pipe(unzipper.Extract({ path: outputDir }))
18 | .promise()
19 | .catch((err) => {
20 | console.error(err);
21 | throw Errors.InvalidZip;
22 | });
23 | }
24 |
25 | function processPost(postFilePath, outputDir, assetsDir, shouldDownloadImages, shouldImportDrafts) {
26 | console.log(chalk.bold('processing post ' + postFilePath));
27 |
28 | const post = new Post(fs.readFileSync(postFilePath));
29 |
30 | const fileName = path.parse(postFilePath).name;
31 | if (!shouldImportDrafts && fileName.startsWith('draft_')) {
32 | console.log('skipping draft: ' + fileName);
33 | return Promise.resolve();
34 | }
35 |
36 | const slug = fileName
37 | .substring(0, fileName.lastIndexOf('-'))
38 | .substring(fileName.indexOf('_') + 1)
39 | .replace(/-+$/, '')
40 | .substring(0, 200);
41 | const imagesDir = path.join(assetsDir, slug);
42 |
43 | if (!fs.existsSync(imagesDir)) {
44 | fs.mkdirSync(imagesDir);
45 | }
46 |
47 | const imageUrls = post.imageUrls;
48 |
49 | const downloadPromise = shouldDownloadImages
50 | ? downloadImages(imageUrls, imagesDir)
51 | : Promise.resolve([]);
52 |
53 | return downloadPromise.then((results) => {
54 |
55 | const images = results.filter(Boolean);
56 | const imageMap = new Map(images.map(item => [item.url, item.filename]));
57 |
58 | const thumbPath = _.isEmpty(images)
59 | ? (_.isEmpty(imageUrls) ? null : imageUrls[0])
60 | : `images/${slug}/${images[0].filename}`;
61 |
62 | const bodyMarkdown = convertHtml(post.bodyHtml, slug, imageMap);
63 |
64 | const data = {
65 | template: 'post',
66 | title: post.title,
67 | date: post.publishDate || new Date()
68 | };
69 |
70 | if (post.subtitle) {
71 | data.subtitle = post.subtitle;
72 | }
73 | if (post.excerpt) {
74 | data.excerpt = post.excerpt;
75 | }
76 | if (thumbPath) {
77 | data.thumb_img_path = thumbPath;
78 | }
79 | if (post.splashImageUrl) {
80 | data.content_img_path = imageMap.has(post.splashImageUrl)
81 | ? `images/${slug}/${imageMap.get(post.splashImageUrl)}`
82 | : post.splashImageUrl;
83 | }
84 |
85 | const outputFilename = path.join(outputDir, `${slug}.md`);
86 | const yamlContent = matter.stringify(bodyMarkdown, data);
87 | fs.writeFileSync(outputFilename, yamlContent);
88 |
89 | console.log('done with ' + outputFilename);
90 | });
91 | }
92 |
93 | function processPosts(inputDir, outputDir, assetsDir, concurrency, shouldDownloadImages, shouldImportDrafts) {
94 | if (!fs.existsSync(inputDir)) {
95 | throw Errors.EmptyInput;
96 | }
97 | if (!fs.existsSync(outputDir)) {
98 | fs.mkdirSync(outputDir);
99 | }
100 | if (!fs.existsSync(assetsDir)) {
101 | fs.mkdirSync(assetsDir);
102 | }
103 |
104 | const postFiles = fs.readdirSync(inputDir);
105 |
106 | if (_.isEmpty(postFiles)) {
107 | throw Errors.EmptyInput;
108 | }
109 |
110 | return pAll(postFiles.map((filename) => {
111 | return () => processPost(path.join(inputDir, filename), outputDir, assetsDir, shouldDownloadImages, shouldImportDrafts);
112 | }), { concurrency: concurrency });
113 | }
114 |
115 | function processProfile(inputFile, dataFile, assetsDir) {
116 | console.log(chalk.bold('processing profile ' + inputFile));
117 |
118 | if (!fs.existsSync(inputFile)) {
119 | console.log('no profile file');
120 | return Promise.resolve();
121 | }
122 |
123 | const profile = new Profile(fs.readFileSync(inputFile));
124 |
125 | let data = {
126 | author: {},
127 | social: {}
128 | };
129 |
130 | if (profile.name) {
131 | data.author.name = profile.name;
132 | }
133 | if (profile.email) {
134 | data.author.email = profile.email;
135 | }
136 | if (profile.twitterUsername) {
137 | data.social.twitter = {
138 | username: profile.twitterUsername,
139 | url: `https://twitter.com/${profile.twitterUsername}`
140 | };
141 | }
142 |
143 | const avatarUrl = profile.avatarUrl;
144 | if (avatarUrl) {
145 | return downloadImages([avatarUrl], assetsDir).then((results) => {
146 | const filename = _.get(results, '[0].filename');
147 | if (filename) {
148 | fs.renameSync(
149 | path.join(assetsDir, filename),
150 | path.join(assetsDir, 'avatar.png')
151 | );
152 | data.author.avatar = 'images/avatar.png';
153 | fs.writeFileSync(dataFile, JSON.stringify(data));
154 | }
155 | });
156 | }
157 |
158 | fs.writeFileSync(dataFile, JSON.stringify(data));
159 |
160 | return Promise.resolve();
161 | }
162 |
163 | function doImport(inputFilePath, postsDir, imagesDir, dataFile, originalDir, shouldDownloadImages=true, shouldImportDrafts=false, concurrency=1) {
164 | if (!fs.existsSync(inputFilePath)) {
165 | return Promise.reject(Errors.InputFileNotFound);
166 | }
167 |
168 | return unzipInput(inputFilePath, originalDir).then(() => {
169 | return processPosts(
170 | path.join(originalDir, 'posts'),
171 | postsDir,
172 | imagesDir,
173 | concurrency,
174 | shouldDownloadImages,
175 | shouldImportDrafts
176 | );
177 | }).then(() => {
178 | if (dataFile) {
179 | return processProfile(
180 | path.join(originalDir, 'profile/profile.html'),
181 | dataFile,
182 | imagesDir
183 | );
184 | }
185 | });
186 | }
187 |
188 | module.exports = {
189 | doImport
190 | };
--------------------------------------------------------------------------------