├── .travis.yml
├── tsconfig.json
├── typings.json
├── .npmignore
├── .gitignore
├── LICENSE
├── .vscode
└── tasks.json
├── package.json
├── README.md
├── index.ts
└── test
└── index.ts
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js:
3 | - "6"
4 | before_script: "npm run-script build"
5 | script: "npm run-script test-travis"
6 | after_script: "cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js"
7 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es6",
4 | "module": "commonjs",
5 | "sourceMap": true,
6 | "declaration": true
7 | },
8 | "exclude": [
9 | "node_modules",
10 | "typings/browser",
11 | "typings/browser.d.ts"
12 | ]
13 | }
14 |
--------------------------------------------------------------------------------
/typings.json:
--------------------------------------------------------------------------------
1 | {
2 | "ambientDependencies": {
3 | "cheerio": "registry:dt/cheerio#0.17.0+20160407085313",
4 | "mocha": "registry:dt/mocha#2.2.5+20160317120654",
5 | "node": "registry:dt/node#4.0.0+20160501135006",
6 | "request": "registry:dt/request#0.0.0+20160316155526"
7 | }
8 | }
9 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 |
6 | # Runtime data
7 | pids
8 | *.pid
9 | *.seed
10 |
11 | # Directory for instrumented libs generated by jscoverage/JSCover
12 | lib-cov
13 |
14 | # Coverage directory used by tools like istanbul
15 | coverage
16 |
17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
18 | .grunt
19 |
20 | # node-waf configuration
21 | .lock-wscript
22 |
23 | # Compiled binary addons (http://nodejs.org/api/addons.html)
24 | build/Release
25 |
26 | # Dependency directory
27 | node_modules
28 |
29 | # Optional npm cache directory
30 | .npm
31 |
32 | # Optional REPL history
33 | .node_repl_history
34 |
35 | *.map
36 | typings
37 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 |
6 | # Runtime data
7 | pids
8 | *.pid
9 | *.seed
10 |
11 | # Directory for instrumented libs generated by jscoverage/JSCover
12 | lib-cov
13 |
14 | # Coverage directory used by tools like istanbul
15 | coverage
16 |
17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
18 | .grunt
19 |
20 | # node-waf configuration
21 | .lock-wscript
22 |
23 | # Compiled binary addons (http://nodejs.org/api/addons.html)
24 | build/Release
25 |
26 | # Dependency directory
27 | node_modules
28 |
29 | # Optional npm cache directory
30 | .npm
31 |
32 | # Optional REPL history
33 | .node_repl_history
34 |
35 | *.js
36 | *.map
37 | *.d.ts
38 | typings
39 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2016 Emma Kuo
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/.vscode/tasks.json:
--------------------------------------------------------------------------------
1 | // Available variables which can be used inside of strings.
2 | // ${workspaceRoot}: the root folder of the team
3 | // ${file}: the current opened file
4 | // ${fileBasename}: the current opened file's basename
5 | // ${fileDirname}: the current opened file's dirname
6 | // ${fileExtname}: the current opened file's extension
7 | // ${cwd}: the current working directory of the spawned process
8 |
9 | // A task runner that calls the Typescript compiler (tsc) and
10 | // compiles based on a tsconfig.json file that is present in
11 | // the root of the folder open in VSCode
12 |
13 | {
14 | "version": "0.1.0",
15 |
16 | // The command is tsc. Assumes that tsc has been installed using npm install -g typescript
17 | "command": "tsc",
18 |
19 | // The command is a shell script
20 | "isShellCommand": true,
21 |
22 | // Show the output window only if unrecognized errors occur.
23 | "showOutput": "silent",
24 |
25 | // Tell the tsc compiler to use the tsconfig.json from the open folder.
26 | "args": ["-p", "."],
27 |
28 | // use the standard tsc problem matcher to find compile problems
29 | // in the output.
30 | "problemMatcher": "$tsc"
31 | }
32 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mf-obj",
3 | "version": "2.1.0",
4 | "description": "Microformat objects",
5 | "main": "index.js",
6 | "typings": "index.d.ts",
7 | "engines": {
8 | "node": ">=4.0.0"
9 | },
10 | "scripts": {
11 | "prebuild": "typings install",
12 | "build": "tsc || true",
13 | "test": "mocha test",
14 | "test-travis": "./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha -- test"
15 | },
16 | "repository": {
17 | "type": "git",
18 | "url": "https://github.com/notenoughneon/mf-obj.git"
19 | },
20 | "keywords": [
21 | "microformat",
22 | "indieweb"
23 | ],
24 | "author": "Emma Kuo",
25 | "license": "MIT",
26 | "bugs": {
27 | "url": "https://github.com/notenoughneon/mf-obj/issues"
28 | },
29 | "homepage": "https://github.com/notenoughneon/mf-obj#readme",
30 | "dependencies": {
31 | "cheerio": "^0.20.0",
32 | "debug": "^2.2.0",
33 | "microformat-node": "^2.0.0",
34 | "request": "^2.72.0"
35 | },
36 | "devDependencies": {
37 | "coveralls": "^2.11.9",
38 | "istanbul": "^0.4.3",
39 | "mocha": "^2.4.5",
40 | "typescript": "^1.8.10",
41 | "typings": "^0.8.1"
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # mf-obj
2 | [](https://www.npmjs.com/package/mf-obj)
3 | [](https://travis-ci.org/notenoughneon/mf-obj)
4 | [](https://coveralls.io/github/notenoughneon/mf-obj?branch=master)
5 |
6 | Microformat objects are a set of utility classes for working with indieweb [posts](http://indiewebcamp.com/posts).
7 | * Read different kinds of posts:
8 | * notes
9 | * articles
10 | * replies
11 | * likes
12 | * reposts
13 | * Parse comments and reply contexts as nested objects
14 | * Resolve author with the [authorship algorithm](http://indiewebcamp.com/authorship)
15 | * Get a list of [webmention](http://indiewebcamp.com/Webmention) targets
16 | * Serialize and deserialize from JSON
17 |
18 | ## Installation
19 |
20 | Microformat objects makes use of ES6 features and requires Node >= 4.0.0.
21 |
22 | ```
23 | npm install mf-obj --save
24 | ```
25 |
26 | ## Examples
27 |
28 | ### Get entry from url
29 | ```javascript
30 | mfo.getEntry('http://somesite/2016/5/1/1')
31 | .then(entry => {
32 | if (entry.isReply() {
33 | console.log('I\'m a reply to "' + entry.replyTo.name + '"');
34 | }
35 | });
36 | ```
37 |
38 | ## API
39 |
40 | 1. [Utility functions](#utility-functions)
41 | * [getEntry(url, strategies?)](#getentry)
42 | * [getCard(url)](#getcard)
43 | * [getEvent(url)](#getevent)
44 | * [getFeed(url)](#getfeed)
45 | 2. [Entry](#entry)
46 | * [name](#name)
47 | * [published](#published)
48 | * [content](#content)
49 | * [summary](#summary)
50 | * [url](#url)
51 | * [author](#author)
52 | * [category](#category)
53 | * [syndication](#syndication)
54 | * [syndicateTo](#syndicateto)
55 | * [photo](#photo)
56 | * [audio](#audio)
57 | * [video](#video)
58 | * [replyTo](#replyto)
59 | * [likeOf](#likeof)
60 | * [repostOf](#repostof)
61 | * [embed](#embed)
62 | * [getDomain()](#getdomain)
63 | * [getPath()](#getpath)
64 | * [getReferences()](#getreferences)
65 | * [getMentions()](#getmentions)
66 | * [getChildren(sortFunc?)](#getchildren)
67 | * [addChild(entry)](#addchild)
68 | * [deleteChild(url)](#deletechild)
69 | * [isReply()](#isreply)
70 | * [isLike()](#islike)
71 | * [isRepost()](#isrepost)
72 | * [isArticle()](#isarticle)
73 | * [serialize()](#serialize)
74 | * [deserialize(json)](#deserialize)
75 | 3. [Card](#card)
76 | * [name](#name-1)
77 | * [photo](#photo)
78 | * [url](#url-1)
79 | * [uid](#uid)
80 | 4. [Event](#event)
81 | * [name](#name-2)
82 | * [url](#url-1)
83 | * [start](#start)
84 | * [end](#end)
85 | * [location](#location)
86 | 5. [Feed](#feed)
87 | * [name](#name-3)
88 | * [url](#url-2)
89 | * [author](#author-2)
90 | * [prev](#prev)
91 | * [next](#next)
92 | * [getChildren(sortFunc?)](#getchildren-2)
93 | * [addChild(entry)](#addchild-2)
94 | * [deleteChild(url)](#deletechild-2)
95 |
96 |
97 | ### Utility functions
98 |
99 | #### getEntry()
100 |
101 | ```javascript
102 | mfo.getEntry(url)
103 | .then(entry => {
104 | //...
105 | });
106 | ```
107 |
108 | ```javascript
109 | mfo.getEntry(url, ['entry','event','oembed'])
110 | .then(entry => {
111 | //...
112 | });
113 | ```
114 |
115 | Fetches the page at `url` and returns a *Promise* for an [Entry](#entry). This will perform the authorship algorithm and fetch the author h-card from a separate url if necessary.
116 |
117 | The second parameter `strategies` is an optional array of strategies to attempt to marshal to an Entry. Strategies are tried in order and if all fail, an exception is thrown. This can be used for displaying comments or reply contexts of URLs that don't contain h-entries. The default value for this parameter is `['entry']`.
118 |
119 | * `entry` - Default h-entry strategy.
120 | * `event` - Marshall an h-event to an Entry. Useful for creating RSVP reply-contexts to an h-event.
121 | * `oembed` - Marshall oembed data to an Entry. Useful for creating reply-contexts or reposts of silo content.
122 | * `opengraph` - Marshall opengraph data to an Entry. Useful for creating reply-contexts or reposts of silo content.
123 | * `html` - Most basic strategy. Marshalls html `` to name and `<body>` to content.
124 |
125 | #### getCard()
126 |
127 | ```javascript
128 | mfo.getCard(url)
129 | .then(card => {
130 | //...
131 | });
132 | ```
133 | Fetches the page at url and returns a *Promise* for a [Card](#card). This will return null if an h-card could not be found according to the authorship algorithm.
134 |
135 | #### getEvent()
136 |
137 | ```javascript
138 | mfo.getEvent(url)
139 | .then(event => {
140 | //...
141 | });
142 | ```
143 | Fetches the page at url and returns a *Promise* for an [Event](#event).
144 |
145 | #### getFeed()
146 |
147 | ```javascript
148 | mfo.getFeed(url)
149 | .then(feed => {
150 | //...
151 | });
152 | ```
153 | Fetches the page at url and returns a *Promise* for a [Feed](#feed).
154 |
155 | ### Entry
156 |
157 | Represents an h-entry or h-cite. Properties of this object correspond to output from the mf2 parser, but have been converted from arrays of string to other data types for convenience.
158 |
159 | ```javascript
160 | var entry = new mfo.Entry();
161 | var entry2 = new mfo.Entry('http://somesite/2016/5/2/1');
162 | ```
163 | The constructor takes an optional argument to set the url.
164 |
165 | #### name
166 |
167 | string || null
168 |
169 | #### published
170 |
171 | Date || null
172 |
173 | #### content
174 |
175 | {html: string, value: string} || null
176 |
177 | #### summary
178 |
179 | string || null
180 |
181 | #### url
182 |
183 | string || null
184 |
185 | #### author
186 |
187 | Card || null
188 |
189 | See [Card](#card).
190 |
191 | #### category
192 |
193 | string[]
194 |
195 | #### syndication
196 |
197 | string[]
198 |
199 | #### syndicateTo
200 |
201 | Parsed from syndicate-to.
202 |
203 | string[]
204 |
205 | #### photo
206 |
207 | string[]
208 |
209 | #### audio
210 |
211 | string[]
212 |
213 | #### video
214 |
215 | string[]
216 |
217 | #### replyTo
218 |
219 | Parsed from in-reply-to.
220 |
221 | Entry[] || null
222 |
223 | #### likeOf
224 |
225 | Parsed from like-of.
226 |
227 | Entry[] || null
228 |
229 | #### repostOf
230 |
231 | Parsed from repost-of.
232 |
233 | Entry[] || null
234 |
235 | #### embed
236 |
237 | {html: string, value: string} || null
238 |
239 | Experimental property for storing oembed content. Parsed from e-x-embed.
240 |
241 | #### getDomain()
242 |
243 | Returns the domain component of the url.
244 |
245 | #### getPath()
246 |
247 | Returns the path component of the url.
248 |
249 | #### getReferences()
250 |
251 | Returns an array of urls from the reply-to, like-of, or repost-of properties.
252 |
253 | #### getMentions()
254 |
255 | Returns an array of urls found in links in the e-content, in addition to getReferences(). Intended for sending webmentions.
256 |
257 | #### getChildren()
258 |
259 | Returns an array of Entries. Use this instead of directly accessing the children property. Takes an optional argument to sort the results.
260 |
261 | ```javascript
262 | var unsorted = entry.getChildren();
263 | var sorted = entry.getChildren(mfo.Entry.byDate);
264 | ```
265 |
266 | #### addChild()
267 |
268 | Adds an Entry to the list of children. If there is an existing child with the same url, it will be overwritten.
269 |
270 | ```javascript
271 | function receiveWebmention(sourceUrl, targetUrl) {
272 | // ...
273 | var sourceEntry = mfo.getEntryFromUrl(sourceUrl);
274 | targetEntry.addChild(sourceEntry);
275 | // ...
276 | }
277 | ```
278 |
279 | #### deleteChild()
280 |
281 | Remove an entry from the list of children by url.
282 |
283 | ```javascript
284 | function receiveWebmention(sourceUrl, targetUrl) {
285 | // ...
286 | if (got404) {
287 | targetEntry.deleteChild(sourceUrl);
288 | }
289 | // ...
290 | }
291 | ```
292 |
293 | #### isReply()
294 |
295 | Tests if reply-to is non-empty.
296 |
297 | #### isLike()
298 |
299 | Tests if like-of is non-empty.
300 |
301 | #### isRepost()
302 |
303 | Tests if repost-of is non-empty.
304 |
305 | #### isArticle()
306 |
307 | Tests if name and content.value properties exist and differ, in addition to other heuristics.
308 |
309 | #### serialize()
310 |
311 | Serialize object to JSON. Nested Entry objects in replyTo, likeOf, repostOf, and children are serialized as an url string.
312 |
313 | Example output:
314 | ```json
315 | {
316 | "name":"Hello World!",
317 | "published":"2015-08-28T08:00:00.000Z",
318 | "content":{
319 | "value":"Hello World!",
320 | "html":"Hello <b>World!</b>"
321 | },
322 | "summary":"Summary",
323 | "url":"http://testsite/2015/8/28/2",
324 | "author":{
325 | "name":"Test User",
326 | "photo":null,
327 | "url":"http://testsite",
328 | "uid":null
329 | },
330 | "category":["indieweb"],
331 | "syndication":[],
332 | "syndicateTo":[],
333 | "photo":[],
334 | "audio":[],
335 | "video":[],
336 | "replyTo":["http://testsite/2015/8/28/2"],
337 | "likeOf":[],
338 | "repostOf":[],
339 | "embed":null,
340 | "children":["http://testsite/2015/8/28/3"]
341 | }
342 | ```
343 |
344 | #### deserialize
345 |
346 | Static method to deserialize json. Nested objects from replyTo, likeOf, repostOf, and children are deserialized as stub Entry objects with only url set.
347 |
348 | ```javascript
349 | var entry = mfo.Entry.deserialize(json);
350 | ```
351 |
352 | ### Card
353 |
354 | Represents an h-card. Properties of this object correspond to output from the mf2 parser, but have been converted from arrays of string to string for convenience.
355 |
356 | ```javascript
357 | var author = new mfo.Card();
358 | var author = new mfo.Card('http://somesite');
359 | ```
360 | The constructor takes an optional argument to set the url.
361 |
362 | #### name
363 |
364 | string || null
365 |
366 | #### photo
367 |
368 | string || null
369 |
370 | #### url
371 |
372 | string || null
373 |
374 | #### uid
375 |
376 | string || null
377 |
378 | ### Event
379 |
380 | Represents an h-event. Properties of this object correspond to output from the mf2 parser, but have been converted from arrays of string to other datatypes for convenience.
381 |
382 | ```javascript
383 | var event = new mfo.Event();
384 | var event = new mfo.Event('http://somesite/event');
385 | ```
386 | The constructor takes an optional argument to set the url.
387 |
388 | #### name
389 |
390 | string || null
391 |
392 | #### url
393 |
394 | string || null
395 |
396 | #### start
397 |
398 | Date || null
399 |
400 | #### stop
401 |
402 | Date || null
403 |
404 | #### Location
405 |
406 | Card || null
407 |
408 | ### Feed
409 |
410 | Represents an h-feed. Properties of this object correspond to output from the mf2 parser, but have been converted from arrays of string to other datatypes for convenience.
411 |
412 | ```javascript
413 | var event = new mfo.Feed();
414 | var event = new mfo.Feed('http://somesite');
415 | ```
416 | The constructor takes an optional argument to set the url.
417 |
418 | #### name
419 |
420 | string || null
421 |
422 | #### url
423 |
424 | string || null
425 |
426 | #### author
427 |
428 | Card || null
429 |
430 | See [Card](#card).
431 |
432 | #### prev
433 |
434 | Parsed from rel="prev" or rel="previous".
435 |
436 | string || null
437 |
438 | #### next
439 |
440 | Parsed from rel="next".
441 |
442 | string || null
443 |
444 | #### getChildren()
445 |
446 | Returns an array of Entries. Use this instead of directly accessing the children property. Takes an optional argument to sort the results.
447 |
448 | #### addChild()
449 |
450 | Adds an Entry to the list of children. If there is an existing child with the same url, it will be overwritten.
451 |
452 | #### deleteChild()
453 |
454 | Remove an entry from the list of children by url.
455 |
--------------------------------------------------------------------------------
/index.ts:
--------------------------------------------------------------------------------
1 | var parser = require('microformat-node');
2 | import Request = require('request');
3 | import cheerio = require('cheerio');
4 | import url = require('url');
5 | var debug = require('debug')('mf-obj');
6 |
7 | export var request = function(url: string): Promise<any> {
8 | return new Promise((resolve, reject) => {
9 | Request.get({url, headers: {'User-Agent': 'mf-obj'}}, (err, result) => err !== null ? reject(err) : resolve(result));
10 | });
11 | }
12 |
13 | async function getOembed(html: string) {
14 | var $ = cheerio.load(html);
15 | var link = $('link[rel=\'alternate\'][type=\'application/json+oembed\'],' +
16 | 'link[rel=\'alternate\'][type=\'text/json+oembed\']').attr('href');
17 | if (link == null)
18 | throw new Error('No oembed link found');
19 | debug('Fetching ' + link);
20 | var res = await request(link);
21 | if (res.statusCode !== 200)
22 | throw new Error('Server returned status ' + res.statusCode);
23 | var embed = JSON.parse(res.body);
24 | return embed;
25 | }
26 |
27 | function getOpengraph(html: string) {
28 | var $ = cheerio.load(html);
29 | var res = {
30 | title: $('meta[property=\'og:title\']').attr('content'),
31 | image: $('meta[property=\'og:image\']').attr('content'),
32 | url: $('meta[property=\'og:url\']').attr('content'),
33 | description: $('meta[property=\'og:description\']').attr('content')
34 | };
35 | if (res.title == null || res.url == null)
36 | throw new Error('No opengraph data found');
37 | return res;
38 | }
39 |
40 | export function escapeHtml(str) {
41 | return str.replace(/&/g, '&').
42 | replace(/</g, '<').
43 | replace(/>/g, '>');
44 | }
45 |
46 | function getLinks(html) {
47 | var $ = cheerio.load(html);
48 | return $('a').toArray().map(a => a.attribs['href']);
49 | }
50 |
51 | export type EntryStrategy = 'entry' | 'event' | 'oembed' | 'opengraph' | 'html';
52 |
53 | var entryStrategies = {
54 | 'entry' : async function(html, url) {
55 | var entry = await getEntryFromHtml(html, url);
56 | if (entry.author !== null && entry.author.url !== null && entry.author.name === null) {
57 | try {
58 | var author = await getCard(entry.author.url);
59 | if (author !== null)
60 | entry.author = author;
61 | } catch (err) {
62 | debug('Failed to fetch author page: ' + err.message);
63 | }
64 | }
65 | return entry;
66 | },
67 | 'event' : async function(html, url) {
68 | var event = await getEventFromHtml(html, url);
69 | var entry = new Entry(url);
70 | entry.name = event.name;
71 | entry.content = {html: escapeHtml(event.name), value: event.name};
72 | return entry;
73 | },
74 | 'oembed': async function(html, url) {
75 | let entry = new Entry(url);
76 | var oembed = await getOembed(html);
77 | if (oembed.title != null)
78 | entry.name = oembed.title;
79 | if (oembed.html != null) {
80 | let $ = cheerio.load(oembed.html);
81 | entry.content = {html: oembed.html, value: $(':root').text()};
82 | }
83 | if (oembed.author_url != null && oembed.author_name != null) {
84 | entry.author = new Card(oembed.author_url);
85 | entry.author.name = oembed.author_name;
86 | }
87 | return entry;
88 | },
89 | 'opengraph': async function(html, url) {
90 | let entry = new Entry(url);
91 | let og = getOpengraph(html);
92 | if (og.description != null) {
93 | entry.name = og.title;
94 | entry.content = {html: escapeHtml(og.description), value: og.description};
95 | } else {
96 | entry.content = {html: escapeHtml(og.title), value: og.title};
97 | }
98 | return entry;
99 | },
100 | 'html': async function(html, url) {
101 | let entry = new Entry(url);
102 | let $ = cheerio.load(html);
103 | entry.name = $('title').text();
104 | entry.content = {html: html, value: $('body').text()};
105 | return entry;
106 | }
107 | }
108 |
109 | export async function getEntry(url: string, strategies?: EntryStrategy[]): Promise<Entry> {
110 | if (strategies == null)
111 | strategies = ['entry'];
112 | var errs = [];
113 | debug('Fetching ' + url);
114 | var res = await request(url);
115 | if (res.statusCode != 200)
116 | throw new Error('Server returned status ' + res.statusCode);
117 | for (let s of strategies) {
118 | try {
119 | return await entryStrategies[s](res.body, url);
120 | } catch (err) {
121 | errs.push(err);
122 | }
123 | }
124 | throw new Error('All strategies failed: ' + errs.reduce((p,c) => p + ',' + c.message));
125 | }
126 |
127 | export async function getEvent(url: string): Promise<Event> {
128 | debug('Fetching ' + url);
129 | var res = await request(url);
130 | if (res.statusCode != 200)
131 | throw new Error('Server returned status ' + res.statusCode);
132 | return getEventFromHtml(res.body, url);
133 | }
134 |
135 | export async function getCard(url: string): Promise<Card> {
136 | debug('Fetching ' + url);
137 | var res = await request(url);
138 | if (res.statusCode != 200)
139 | throw new Error('Server returned status ' + res.statusCode);
140 | var mf = await parser.getAsync({html: res.body, baseUrl: url});
141 | var cards = mf.items.
142 | filter(i => i.type.some(t => t == 'h-card')).
143 | map(h => buildCard(h));
144 | // 1. uid and url match author-page url
145 | var match = cards.filter(c =>
146 | c.url != null &&
147 | c.uid != null &&
148 | urlsEqual(c.url, url) &&
149 | urlsEqual(c.uid, url)
150 | );
151 | if (match.length > 0) return match[0];
152 | // 2. url matches rel=me
153 | if (mf.rels.me != null) {
154 | var match = cards.filter(c =>
155 | mf.rels.me.some(r =>
156 | c.url != null &&
157 | urlsEqual(c.url, r)
158 | )
159 | );
160 | if (match.length > 0) return match[0];
161 | }
162 | // 3. url matches author-page url
163 | var match = cards.filter(c =>
164 | c.url != null &&
165 | urlsEqual(c.url, url)
166 | );
167 | if (match.length > 0) return match[0];
168 | return null;
169 | }
170 |
171 | export async function getFeed(url: string): Promise<Feed> {
172 | debug('Fetching ' + url);
173 | var res = await request(url);
174 | if (res.statusCode != 200)
175 | throw new Error('Server returned status ' + res.statusCode);
176 | return getFeedFromHtml(res.body, url);
177 | }
178 |
179 | export async function getEntryFromHtml(html: string, url: string): Promise<Entry> {
180 | var mf = await parser.getAsync({html: html, baseUrl: url});
181 | var entries = mf.items.filter(i => i.type.some(t => t == 'h-entry'));
182 | if (entries.length == 0)
183 | throw new Error('No h-entry found');
184 | else if (entries.length > 1)
185 | throw new Error('Multiple h-entries found');
186 | var relAuthor = mf.rels.author != null && mf.rels.author.length > 0 ? new Card(mf.rels.author[0]) : null;
187 | let entry = buildEntry(entries[0], relAuthor);
188 | return entry;
189 | }
190 |
191 | async function getEventFromHtml(html: string, url: string): Promise<Event> {
192 | var mf = await parser.getAsync({html: html, baseUrl: url});
193 | var events = mf.items.filter(i => i.type.some(t => t === 'h-event'));
194 | if (events.length == 0)
195 | throw new Error('No h-event found');
196 | else if (events.length > 1)
197 | throw new Error('Multiple h-events found');
198 | var event = buildEvent(events[0]);
199 | if (event.url == null)
200 | event.url = url;
201 | return event;
202 | }
203 |
204 | var feedStrategies = {
205 | 'hfeed': async function(html, url) {
206 | var mf = await parser.getAsync({html: html, baseUrl: url});
207 | var feeds = mf.items.filter(i => i.type.some(t => t === 'h-feed'));
208 | if (feeds.length == 0)
209 | throw new Error('No h-feed found');
210 | else if (feeds.length > 1)
211 | throw new Error('Multiple h-feeds found');
212 | var feed = await buildFeed(feeds[0]);
213 | if (feed.url == null)
214 | feed.url = url;
215 | if (mf.rels.prev != null && mf.rels.prev.length > 0)
216 | feed.prev = mf.rels.prev[0];
217 | else if (mf.rels.previous != null && mf.rels.previous.length > 0)
218 | feed.prev = mf.rels.previous[0];
219 | if (mf.rels.next != null && mf.rels.next.length > 0)
220 | feed.next = mf.rels.next[0];
221 | return feed;
222 | },
223 | 'implied': async function(html, url) {
224 | var mf = await parser.getAsync({html: html, baseUrl: url});
225 | var entries = mf.items.filter(i => i.type.some(t => t === 'h-entry'));
226 | if (entries.length == 0)
227 | throw new Error('No h-entries found');
228 | var feed = new Feed(url);
229 | var $ = cheerio.load(html);
230 | feed.name = $('title').text();
231 | feed.author = await getCard(url);
232 | for (let entry of entries) {
233 | feed.addChild(buildEntry(entry, feed.author));
234 | }
235 | if (mf.rels.prev != null && mf.rels.prev.length > 0)
236 | feed.prev = mf.rels.prev[0];
237 | else if (mf.rels.previous != null && mf.rels.previous.length > 0)
238 | feed.prev = mf.rels.previous[0];
239 | if (mf.rels.next != null && mf.rels.next.length > 0)
240 | feed.next = mf.rels.next[0];
241 | return feed;
242 | }
243 | };
244 |
245 | async function getFeedFromHtml(html: string, url: string): Promise<Feed> {
246 | var strategies = ['hfeed', 'implied'];
247 | var errs = [];
248 | for (let s of strategies) {
249 | try {
250 | return await feedStrategies[s](html, url);
251 | } catch (err) {
252 | errs.push(err);
253 | }
254 | }
255 | throw new Error('All strategies failed: ' + errs.reduce((p,c) => p + ',' + c.message));
256 | }
257 |
258 | function prop(mf, name, f?) {
259 | if (mf.properties[name] != null) {
260 | if (f != null)
261 | return mf.properties[name].filter(e => e !== '').map(f);
262 | return mf.properties[name].filter(e => e !== '');
263 | }
264 | return [];
265 | }
266 |
267 | function firstProp(mf, name, f?) {
268 | if (mf.properties[name] != null) {
269 | if (f != null)
270 | return f(mf.properties[name][0]);
271 | return mf.properties[name][0];
272 | }
273 | return null;
274 | }
275 |
276 | function buildCard(mf) {
277 | if (typeof(mf) === 'string')
278 | return new Card(mf);
279 | var card = new Card();
280 | if (!mf.type.some(t => t === 'h-card'))
281 | throw new Error('Attempt to parse ' + mf.type + ' as Card');
282 | card.name = firstProp(mf, 'name');
283 | card.photo = firstProp(mf, 'photo');
284 | card.url = firstProp(mf, 'url');
285 | card.uid = firstProp(mf, 'uid');
286 | return card;
287 | }
288 |
289 | function buildEvent(mf) {
290 | if (typeof(mf) === 'string')
291 | return new Event(mf);
292 | var event = new Event();
293 | if (!mf.type.some(t => t === 'h-event'))
294 | throw new Error('Attempt to parse ' + mf.type + ' as Event');
295 | event.name = firstProp(mf, 'name');
296 | event.url = firstProp(mf, 'url');
297 | event.start = firstProp(mf, 'start', s => new Date(s));
298 | event.end = firstProp(mf, 'end', e => new Date(e));
299 | event.location = firstProp(mf, 'location', l => buildCard(l));
300 | return event;
301 | }
302 |
303 | async function buildFeed(mf) {
304 | if (typeof(mf) === 'string')
305 | return new Feed(mf);
306 | var feed = new Feed();
307 | if (!mf.type.some(t => t === 'h-feed'))
308 | throw new Error('Attempt to parse ' + mf.type + ' as Feed');
309 | feed.name = firstProp(mf, 'name');
310 | feed.url = firstProp(mf, 'url');
311 | feed.author = firstProp(mf, 'author', a => buildCard(a));
312 | if (feed.author !== null && feed.author.url !== null && feed.author.name === null) {
313 | try {
314 | var author = await getCard(feed.author.url);
315 | if (author !== null)
316 | feed.author = author;
317 | } catch (err) {
318 | debug('Failed to fetch author page: ' + err.message);
319 | }
320 | }
321 | (mf.children || [])
322 | .filter(i => i.type.some(t => t === 'h-cite' || t === 'h-entry'))
323 | .map(e => buildEntry(e, feed.author))
324 | .filter(e => e.url != null)
325 | .map(e => feed.addChild(e));
326 | return feed;
327 | }
328 |
329 | function buildEntry(mf, defaultAuthor?: Card) {
330 | if (typeof(mf) === 'string')
331 | return new Entry(mf);
332 | var entry = new Entry();
333 | if (!mf.type.some(t => t === 'h-entry' || t === 'h-cite'))
334 | throw new Error('Attempt to parse ' + mf.type + ' as Entry');
335 | entry.name = firstProp(mf, 'name');
336 | entry.published = firstProp(mf, 'published', p => new Date(p));
337 | entry.content = firstProp(mf, 'content');
338 | entry.summary = firstProp(mf, 'summary');
339 | entry.url = firstProp(mf, 'url');
340 | entry.author = firstProp(mf, 'author', a => buildCard(a));
341 | if (entry.author === null && defaultAuthor)
342 | entry.author = defaultAuthor;
343 | entry.category = prop(mf, 'category');
344 | entry.syndication = prop(mf, 'syndication');
345 | entry.syndicateTo = prop(mf, 'syndicate-to');
346 | entry.photo = prop(mf, 'photo');
347 | entry.audio = prop(mf, 'audio');
348 | entry.video = prop(mf, 'video');
349 | entry.replyTo = prop(mf, 'in-reply-to', r => buildEntry(r));
350 | entry.likeOf = prop(mf, 'like-of', r => buildEntry(r));
351 | entry.repostOf = prop(mf, 'repost-of', r => buildEntry(r));
352 | entry.embed = firstProp(mf, 'x-embed');
353 | (mf.children || [])
354 | .concat(mf.properties['comment'] || [])
355 | .filter(i => i.type.some(t => t === 'h-cite' || t === 'h-entry'))
356 | .map(e => buildEntry(e))
357 | .filter(e => e.url != null)
358 | .map(e => entry.addChild(e));
359 | return entry;
360 | }
361 |
362 | function urlsEqual(u1, u2) {
363 | var p1 = url.parse(u1);
364 | var p2 = url.parse(u2);
365 | return p1.protocol === p2.protocol &&
366 | p1.host === p2.host &&
367 | p1.path === p2.path;
368 | }
369 |
370 | export class Entry {
371 | name: string = null;
372 | published: Date = null;
373 | content: {value: string, html: string} = null;
374 | summary: string = null;
375 | url: string = null;
376 | author: Card = null;
377 | category: string[] = [];
378 | syndication: string[] = [];
379 | syndicateTo: string[] = [];
380 | photo: string[] = [];
381 | audio: string[] = [];
382 | video: string[] = [];
383 | replyTo: Entry[] = [];
384 | likeOf: Entry[] = [];
385 | repostOf: Entry[] = [];
386 | embed: {value: string, html: string} = null;
387 | private children: Map<string, Entry> = new Map();
388 |
389 | constructor(url?: string) {
390 | if (typeof(url) === 'string') {
391 | this.url = url;
392 | }
393 | }
394 |
395 | private _getTime() {
396 | if (this.published != null)
397 | return this.published.getTime();
398 | return -1;
399 | }
400 |
401 | private _getType(): number {
402 | if (this.isLike() || this.isRepost())
403 | return 1;
404 | return 0;
405 | }
406 |
407 | static byDate = (a: Entry, b: Entry) => a._getTime() - b._getTime();
408 | static byDateDesc = (a: Entry, b: Entry) => b._getTime() - a._getTime();
409 | static byType = (a: Entry, b: Entry) => a._getType() - b._getType();
410 | static byTypeDesc = (a: Entry, b: Entry) => b._getType() - a._getType();
411 |
412 | getDomain(): string {
413 | var p = url.parse(this.url);
414 | return p.protocol + '//' + p.host;
415 | }
416 |
417 | getPath(): string {
418 | return url.parse(this.url).path;
419 | }
420 |
421 | getReferences(): string[] {
422 | return this.replyTo.concat(this.likeOf).concat(this.repostOf).map(r => r.url);
423 | }
424 |
425 | getMentions(): string[] {
426 | var allLinks = this.getReferences();
427 | if (this.content != null)
428 | allLinks = allLinks.concat(getLinks(this.content.html));
429 | return allLinks;
430 | }
431 |
432 | getChildren(sortFunc?: (a: Entry, b: Entry) => number) {
433 | var values = Array.from(this.children.values());
434 | if (sortFunc != null)
435 | values.sort(sortFunc);
436 | return values;
437 | }
438 |
439 | addChild(entry: Entry) {
440 | if (entry.url == null)
441 | throw new Error('Url must be set');
442 | this.children.set(entry.url, entry);
443 | }
444 |
445 | deleteChild(url: string) {
446 | return this.children.delete(url);
447 | }
448 |
449 | isReply(): boolean {
450 | return this.replyTo.length > 0;
451 | }
452 |
453 | isRepost(): boolean {
454 | return this.repostOf.length > 0;
455 | }
456 |
457 | isLike(): boolean {
458 | return this.likeOf.length > 0;
459 | }
460 |
461 | isArticle(): boolean {
462 | return !this.isReply() &&
463 | !this.isRepost() &&
464 | !this.isLike() &&
465 | this.name != null &&
466 | this.content != null &&
467 | this.content.value != '' &&
468 | this.name !== this.content.value;
469 | }
470 |
471 | serialize(): string {
472 | return JSON.stringify(this, (key,val) => {
473 | if (key === 'replyTo' || key === 'repostOf' || key === 'likeOf')
474 | return val.map(e => e.url);
475 | if (key === 'children')
476 | return Array.from(val.values()).map(r => r.url);
477 | return val;
478 | });
479 | }
480 |
481 | static deserialize(json: string): Entry {
482 | return JSON.parse(json, (key,val) => {
483 | if (val != null && key === 'author') {
484 | var author = new Card();
485 | author.name = val.name;
486 | author.photo = val.photo;
487 | author.uid = val.uid;
488 | author.url = val.url;
489 | return author;
490 | }
491 | if (key === 'replyTo' || key === 'repostOf' || key === 'likeOf')
492 | return val.map(e => new Entry(e));
493 | if (key === 'children')
494 | return new Map(val.map(url => [url, new Entry(url)]));
495 | if (key === '') {
496 | var entry = new Entry();
497 | entry.name = val.name;
498 | entry.published = val.published ? new Date(val.published) : null;
499 | entry.content = val.content;
500 | entry.summary = val.summary;
501 | entry.url = val.url;
502 | entry.author = val.author;
503 | entry.category = val.category;
504 | entry.syndication = val.syndication;
505 | entry.syndicateTo = val.syndicateTo;
506 | entry.replyTo = val.replyTo;
507 | entry.likeOf = val.likeOf;
508 | entry.repostOf = val.repostOf;
509 | entry.embed = val.embed;
510 | entry.children = val.children;
511 | return entry;
512 | }
513 | return val;
514 | });
515 | }
516 | }
517 |
518 | export class Card {
519 | name: string = null;
520 | photo: string = null;
521 | url: string = null;
522 | uid: string = null;
523 |
524 | constructor(urlOrName?: string) {
525 | if (typeof(urlOrName) === 'string') {
526 | if (urlOrName.startsWith('http://') || urlOrName.startsWith('https://'))
527 | this.url = urlOrName;
528 | else
529 | this.name = urlOrName;
530 | }
531 | }
532 | }
533 |
534 | export class Event {
535 | name: string = null;
536 | url: string = null;
537 | start: Date = null;
538 | end: Date = null;
539 | location: Card = null;
540 |
541 | constructor(url?: string) {
542 | if (typeof(url) === 'string') {
543 | this.url = url;
544 | }
545 | }
546 | }
547 |
548 | export class Feed {
549 | name: string = null;
550 | url: string = null;
551 | author: Card = null;
552 | prev: string = null;
553 | next: string = null;
554 | private children: Map<string, Entry> = new Map();
555 |
556 | constructor(url?: string) {
557 | if (typeof(url) === 'string') {
558 | this.url = url;
559 | }
560 | }
561 |
562 | getChildren(sortFunc?: (a: Entry, b: Entry) => number) {
563 | var values = Array.from(this.children.values());
564 | if (sortFunc != null)
565 | values.sort(sortFunc);
566 | return values;
567 | }
568 |
569 | addChild(entry: Entry) {
570 | if (entry.url == null)
571 | throw new Error('Url must be set');
572 | this.children.set(entry.url, entry);
573 | }
574 |
575 | deleteChild(url: string) {
576 | return this.children.delete(url);
577 | }
578 | }
--------------------------------------------------------------------------------
/test/index.ts:
--------------------------------------------------------------------------------
1 | import assert = require('assert');
2 | import mfo = require('../index');
3 |
4 | describe('event', function() {
5 | var orig_request;
6 | var pages;
7 |
8 | before(function() {
9 | orig_request = mfo.request;
10 | mfo.request = url => Promise.resolve(pages[url] ? {statusCode: 200, body: pages[url]} : {statusCode: 404, body: ''});
11 | });
12 |
13 | after(function() {
14 | mfo.request = orig_request;
15 | });
16 |
17 | it('can be constructed with no args', function() {
18 | var event = new mfo.Event();
19 | assert.equal(event.url, null);
20 | assert.equal(event.start, null);
21 | assert.equal(event.location, null);
22 | });
23 |
24 | it('can be constructed from url', function() {
25 | var url = 'http://2016.indieweb.org';
26 | var event = new mfo.Event(url);
27 | assert.equal(event.url, url);
28 | });
29 |
30 | it('can load an event', function(done) {
31 | pages = {
32 | 'http://2016.indieweb.org': '<div class="h-event">\
33 | <h1 class="p-name">Indieweb Summit</h1>\
34 | <time class="dt-start" datetime="2016-06-03">June 3</time>\
35 | <time class="dt-end" datetime="2016-06-05">5</time>\
36 | <span class="h-card p-location">\
37 | <span class="p-name">Vadio</span>, \
38 | <span class="p-street-address">919 SW Taylor St, Ste 300</span>, \
39 | <span class="p-locality">Portland</span>, <span class="p-region">Oregon</span>\
40 | </span>\
41 | </div>'};
42 | mfo.getEvent('http://2016.indieweb.org')
43 | .then(event => {
44 | assert.equal(event.url, 'http://2016.indieweb.org');
45 | assert.equal(event.name, 'Indieweb Summit');
46 | assert.deepEqual(event.start, new Date('2016-06-03'));
47 | assert.deepEqual(event.end, new Date('2016-06-05'));
48 | assert.equal(event.location.name, 'Vadio');
49 | })
50 | .then(done)
51 | .catch(done);
52 | });
53 |
54 | it('getEventFromUrl works', function(done) {
55 | pages = {
56 | 'http://2016.indieweb.org': '<div class="h-event">\
57 | <h1 class="p-name">Indieweb Summit</h1>\
58 | <time class="dt-start" datetime="2016-06-03">June 3</time>\
59 | <time class="dt-end" datetime="2016-06-05">5</time>\
60 | <span class="h-card p-location">\
61 | <span class="p-name">Vadio</span>, \
62 | <span class="p-street-address">919 SW Taylor St, Ste 300</span>, \
63 | <span class="p-locality">Portland</span>, <span class="p-region">Oregon</span>\
64 | </span>\
65 | </div>',
66 | };
67 | mfo.getEvent('http://2016.indieweb.org')
68 | .then(e => {
69 | assert(e.name === 'Indieweb Summit');
70 | })
71 | .then(done)
72 | .catch(done);
73 | });
74 | });
75 |
76 | describe('feed', function() {
77 | var orig_request;
78 | var pages;
79 |
80 | before(function() {
81 | orig_request = mfo.request;
82 | mfo.request = url => Promise.resolve(pages[url] ? {statusCode: 200, body: pages[url]} : {statusCode: 404, body: ''});
83 | });
84 |
85 | after(function() {
86 | mfo.request = orig_request;
87 | });
88 |
89 | it('can be constructed with no args', function() {
90 | var feed = new mfo.Feed();
91 | assert.equal(feed.url, null);
92 | assert.equal(feed.name, null);
93 | assert.equal(feed.author, null);
94 | assert.deepEqual(feed.getChildren(), []);
95 | });
96 |
97 | it('can be constructed from url', function() {
98 | var url = 'http://sometsite';
99 | var feed = new mfo.Feed(url);
100 | assert.equal(feed.url, url);
101 | });
102 |
103 | it('getFeed (h-feed)', function(done) {
104 | pages = {
105 | 'http://somesite': '<div class="h-feed">\
106 | <a class="u-url" href="http://somesite"></a>\
107 | <div class="p-name">Notes</div>\
108 | <div class="h-entry">\
109 | <a class="u-url" href="/3"></a>\
110 | <div class="p-name e-content">Hello 3</div>\
111 | </div>\
112 | <div class="h-entry">\
113 | <a class="u-url" href="/2"></a>\
114 | <div class="p-name e-content">Hello 2</div>\
115 | </div>\
116 | <div class="h-entry">\
117 | <a class="u-url" href="/1"></a>\
118 | <div class="p-name e-content">Hello 1</div>\
119 | </div>\
120 | <a rel="prev" href="prev"></a>\
121 | <a rel="next" href="next"></a>\
122 | </div>'};
123 | mfo.getFeed('http://somesite')
124 | .then(feed => {
125 | assert.equal(feed.url, 'http://somesite');
126 | assert.equal(feed.name, 'Notes');
127 | var children = feed.getChildren();
128 | assert.equal(children.length, 3);
129 | assert.equal(children[0].url, 'http://somesite/3');
130 | assert.equal(children[0].name, 'Hello 3');
131 | assert.equal(children[2].url, 'http://somesite/1');
132 | assert.equal(children[2].name, 'Hello 1');
133 | assert.equal(feed.prev, 'http://somesite/prev');
134 | assert.equal(feed.next, 'http://somesite/next');
135 | })
136 | .then(done)
137 | .catch(done);
138 | });
139 |
140 | it('getFeed (implied)', function(done) {
141 | pages = {
142 | 'http://somesite': '<html>\
143 | <head><title>Notes\
144 | \
145 | \
146 |
\
147 |
\
148 |
Hello 3
\
149 |
\
150 |
\
151 |
\
152 |
Hello 2
\
153 |
\
154 |
\
155 |
\
156 |
Hello 1
\
157 |
\
158 |
\
159 |
\
160 |