10 |
One two three.
11 |
Four five six.
12 |
Seven eight nine.
13 |
Ten eleven twelve.
14 | `;
15 |
16 | const wordCount = GenericWordCountExtractor.extract({ content });
17 |
18 | assert.equal(wordCount, 12);
19 | });
20 | });
21 | });
22 |
--------------------------------------------------------------------------------
/src/extractors/get-extractor.js:
--------------------------------------------------------------------------------
1 | import URL from 'url';
2 |
3 | import Extractors from './all';
4 | import GenericExtractor from './generic';
5 | import detectByHtml from './detect-by-html';
6 | import { apiExtractors } from './add-extractor';
7 |
8 | export default function getExtractor(url, parsedUrl, $) {
9 | parsedUrl = parsedUrl || URL.parse(url);
10 | const { hostname } = parsedUrl;
11 | const baseDomain = hostname
12 | .split('.')
13 | .slice(-2)
14 | .join('.');
15 |
16 | return (
17 | apiExtractors[hostname] ||
18 | apiExtractors[baseDomain] ||
19 | Extractors[hostname] ||
20 | Extractors[baseDomain] ||
21 | detectByHtml($) ||
22 | GenericExtractor
23 | );
24 | }
25 |
--------------------------------------------------------------------------------
/src/extractors/index.js:
--------------------------------------------------------------------------------
1 | const Extractor = {};
2 |
3 | export default Extractor;
4 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/clean.js:
--------------------------------------------------------------------------------
1 | import { TAGS_TO_REMOVE } from './constants';
2 |
3 | function isComment(index, node) {
4 | return node.type === 'comment';
5 | }
6 |
7 | function cleanComments($) {
8 | $.root()
9 | .find('*')
10 | .contents()
11 | .filter(isComment)
12 | .remove();
13 |
14 | return $;
15 | }
16 |
17 | export default function clean($) {
18 | $(TAGS_TO_REMOVE).remove();
19 |
20 | $ = cleanComments($);
21 | return $;
22 | }
23 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/clean.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import clean from './clean';
5 |
6 | describe('clean($)', () => {
7 | it('removes script elements', () => {
8 | const html = "
";
9 | const $ = cheerio.load(html);
10 |
11 | assert.equal(clean($).html(), '
');
12 | });
13 |
14 | it('removes style elements', () => {
15 | const html = '
';
16 | const $ = cheerio.load(html);
17 |
18 | assert.equal(clean($).html(), '
');
19 | });
20 |
21 | it('removes comments', () => {
22 | const html = '
HI
';
23 | const $ = cheerio.load(html);
24 |
25 | assert.equal(clean($).html(), '
HI
');
26 | });
27 | });
28 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/constants.js:
--------------------------------------------------------------------------------
1 | export const IS_LINK = new RegExp('https?://', 'i');
2 | const IMAGE_RE = '.(png|gif|jpe?g)';
3 | export const IS_IMAGE = new RegExp(`${IMAGE_RE}`, 'i');
4 | export const IS_SRCSET = new RegExp(
5 | `${IMAGE_RE}(\\?\\S+)?(\\s*[\\d.]+[wx])`,
6 | 'i'
7 | );
8 |
9 | export const TAGS_TO_REMOVE = ['script', 'style', 'form'].join(',');
10 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/index.js:
--------------------------------------------------------------------------------
1 | export { default as normalizeMetaTags } from './normalize-meta-tags';
2 | export {
3 | default as convertLazyLoadedImages,
4 | } from './convert-lazy-loaded-images';
5 | export { default as clean } from './clean';
6 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/normalize-meta-tags.js:
--------------------------------------------------------------------------------
1 | function convertMetaProp($, from, to) {
2 | $(`meta[${from}]`).each((_, node) => {
3 | const $node = $(node);
4 |
5 | const value = $node.attr(from);
6 | $node.attr(to, value);
7 | $node.removeAttr(from);
8 | });
9 |
10 | return $;
11 | }
12 |
13 | // For ease of use in extracting from meta tags,
14 | // replace the "content" attribute on meta tags with the
15 | // "value" attribute.
16 | //
17 | // In addition, normalize 'property' attributes to 'name' for ease of
18 | // querying later. See, e.g., og or twitter meta tags.
19 |
20 | export default function normalizeMetaTags($) {
21 | $ = convertMetaProp($, 'content', 'value');
22 | $ = convertMetaProp($, 'property', 'name');
23 | return $;
24 | }
25 |
--------------------------------------------------------------------------------
/src/resource/utils/dom/normalize-meta-tags.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import normalizeMetaTags from './normalize-meta-tags';
5 |
6 | describe('normalizeMetaTags($)', () => {
7 | it('replaces "content" attributes with "value"', () => {
8 | // browser cheerio/jquery will remove/replace html, so result is different
9 | const test = cheerio.browser
10 | ? '
'
11 | : '
';
12 |
13 | const $ = cheerio.load('
');
14 | const result = normalizeMetaTags($).html();
15 |
16 | assert.equal(result, test);
17 | });
18 |
19 | it('replaces "property" attributes with "name"', () => {
20 | const test = cheerio.browser
21 | ? '
'
22 | : '
';
23 |
24 | const $ = cheerio.load('
');
25 | const result = normalizeMetaTags($).html();
26 |
27 | assert.equal(result, test);
28 | });
29 | });
30 |
--------------------------------------------------------------------------------
/src/resource/utils/index.js:
--------------------------------------------------------------------------------
1 | export { default as fetchResource } from './fetch-resource';
2 |
--------------------------------------------------------------------------------
/src/shims/iconv-lite.js:
--------------------------------------------------------------------------------
1 | // this is a shim for the browser build;
2 | // iconv-lite doubles build size, and we
3 | // don't need it for already rendered text
4 | const iconv = {
5 | encodingExists: () => false,
6 | decode: s => s,
7 | };
8 |
9 | export default iconv;
10 |
--------------------------------------------------------------------------------
/src/utils/dom/brs-to-ps.js:
--------------------------------------------------------------------------------
1 | import { paragraphize } from './index';
2 |
3 | // ## NOTES:
4 | // Another good candidate for refactoring/optimizing.
5 | // Very imperative code, I don't love it. - AP
6 |
7 | // Given cheerio object, convert consecutive
tags into
8 | //
tags instead.
9 | //
10 | // :param $: A cheerio object
11 |
12 | export default function brsToPs($) {
13 | let collapsing = false;
14 | $('br').each((index, element) => {
15 | const $element = $(element);
16 | const nextElement = $element.next().get(0);
17 |
18 | if (nextElement && nextElement.tagName.toLowerCase() === 'br') {
19 | collapsing = true;
20 | $element.remove();
21 | } else if (collapsing) {
22 | collapsing = false;
23 | paragraphize(element, $, true);
24 | }
25 | });
26 |
27 | return $;
28 | }
29 |
--------------------------------------------------------------------------------
/src/utils/dom/clean-attributes.js:
--------------------------------------------------------------------------------
1 | import { getAttrs, setAttrs } from 'utils/dom';
2 |
3 | import { WHITELIST_ATTRS_RE, KEEP_CLASS } from './constants';
4 |
5 | function removeAllButWhitelist($article, $) {
6 | $article.find('*').each((index, node) => {
7 | const attrs = getAttrs(node);
8 |
9 | setAttrs(
10 | node,
11 | Reflect.ownKeys(attrs).reduce((acc, attr) => {
12 | if (WHITELIST_ATTRS_RE.test(attr)) {
13 | return { ...acc, [attr]: attrs[attr] };
14 | }
15 |
16 | return acc;
17 | }, {})
18 | );
19 | });
20 |
21 | // Remove the mercury-parser-keep class from result
22 | $(`.${KEEP_CLASS}`, $article).removeClass(KEEP_CLASS);
23 |
24 | return $article;
25 | }
26 |
27 | // Remove attributes like style or align
28 | export default function cleanAttributes($article, $) {
29 | // Grabbing the parent because at this point
30 | // $article will be wrapped in a div which will
31 | // have a score set on it.
32 | return removeAllButWhitelist(
33 | $article.parent().length ? $article.parent() : $article,
34 | $
35 | );
36 | }
37 |
--------------------------------------------------------------------------------
/src/utils/dom/clean-attributes.test.js:
--------------------------------------------------------------------------------
1 | import cheerio from 'cheerio';
2 |
3 | import { assertClean } from 'test-helpers';
4 |
5 | import { cleanAttributes } from './index';
6 |
7 | describe('cleanAttributes($)', () => {
8 | it('removes style attributes from nodes', () => {
9 | const $ = cheerio.load(`
10 |
11 |
What do you think?
12 |
13 | `);
14 |
15 | const result = cleanAttributes($('*').first(), $);
16 | assertClean(
17 | $.html(result),
18 | `
19 |
20 |
What do you think?
21 |
22 | `
23 | );
24 | });
25 |
26 | it('removes align attributes from nodes', () => {
27 | const $ = cheerio.load(`
28 |
29 |
What do you think?
30 |
31 | `);
32 |
33 | const result = cleanAttributes($('*').first(), $);
34 | assertClean(
35 | $.html(result),
36 | `
37 |
38 |
What do you think?
39 |
40 | `
41 | );
42 | });
43 | });
44 |
--------------------------------------------------------------------------------
/src/utils/dom/clean-h-ones.js:
--------------------------------------------------------------------------------
1 | import { convertNodeTo } from 'utils/dom';
2 |
3 | // H1 tags are typically the article title, which should be extracted
4 | // by the title extractor instead. If there's less than 3 of them (<3),
5 | // strip them. Otherwise, turn 'em into H2s.
6 | export default function cleanHOnes(article, $) {
7 | const $hOnes = $('h1', article);
8 |
9 | if ($hOnes.length < 3) {
10 | $hOnes.each((index, node) => $(node).remove());
11 | } else {
12 | $hOnes.each((index, node) => {
13 | convertNodeTo($(node), $, 'h2');
14 | });
15 | }
16 |
17 | return $;
18 | }
19 |
--------------------------------------------------------------------------------
/src/utils/dom/clean-headers.js:
--------------------------------------------------------------------------------
1 | import { getWeight } from 'extractors/generic/content/scoring';
2 |
3 | import { HEADER_TAG_LIST } from './constants';
4 | import { normalizeSpaces } from '../text';
5 |
6 | export default function cleanHeaders($article, $, title = '') {
7 | $(HEADER_TAG_LIST, $article).each((index, header) => {
8 | const $header = $(header);
9 | // Remove any headers that appear before all other p tags in the
10 | // document. This probably means that it was part of the title, a
11 | // subtitle or something else extraneous like a datestamp or byline,
12 | // all of which should be handled by other metadata handling.
13 | if ($($header, $article).prevAll('p').length === 0) {
14 | return $header.remove();
15 | }
16 |
17 | // Remove any headers that match the title exactly.
18 | if (normalizeSpaces($(header).text()) === title) {
19 | return $header.remove();
20 | }
21 |
22 | // If this header has a negative weight, it's probably junk.
23 | // Get rid of it.
24 | if (getWeight($(header)) < 0) {
25 | return $header.remove();
26 | }
27 |
28 | return $header;
29 | });
30 |
31 | return $;
32 | }
33 |
--------------------------------------------------------------------------------
/src/utils/dom/clean-images.js:
--------------------------------------------------------------------------------
1 | import { SPACER_RE } from './constants';
2 |
3 | function cleanForHeight($img, $) {
4 | const height = parseInt($img.attr('height'), 10);
5 | const width = parseInt($img.attr('width'), 10) || 20;
6 |
7 | // Remove images that explicitly have very small heights or
8 | // widths, because they are most likely shims or icons,
9 | // which aren't very useful for reading.
10 | if ((height || 20) < 10 || width < 10) {
11 | $img.remove();
12 | } else if (height) {
13 | // Don't ever specify a height on images, so that we can
14 | // scale with respect to width without screwing up the
15 | // aspect ratio.
16 | $img.removeAttr('height');
17 | }
18 |
19 | return $;
20 | }
21 |
22 | // Cleans out images where the source string matches transparent/spacer/etc
23 | // TODO This seems very aggressive - AP
24 | function removeSpacers($img, $) {
25 | if (SPACER_RE.test($img.attr('src'))) {
26 | $img.remove();
27 | }
28 |
29 | return $;
30 | }
31 |
32 | export default function cleanImages($article, $) {
33 | $article.find('img').each((index, img) => {
34 | const $img = $(img);
35 |
36 | cleanForHeight($img, $);
37 | removeSpacers($img, $);
38 | });
39 |
40 | return $;
41 | }
42 |
--------------------------------------------------------------------------------
/src/utils/dom/convert-node-to.js:
--------------------------------------------------------------------------------
1 | import { getAttrs } from 'utils/dom';
2 |
3 | export default function convertNodeTo($node, $, tag = 'p') {
4 | const node = $node.get(0);
5 | if (!node) {
6 | return $;
7 | }
8 | const attrs = getAttrs(node) || {};
9 |
10 | const attribString = Reflect.ownKeys(attrs)
11 | .map(key => `${key}=${attrs[key]}`)
12 | .join(' ');
13 | let html;
14 |
15 | if ($.browser) {
16 | // In the browser, the contents of noscript tags aren't rendered, therefore
17 | // transforms on the noscript tag (commonly used for lazy-loading) don't work
18 | // as expected. This test case handles that
19 | html =
20 | node.tagName.toLowerCase() === 'noscript' ? $node.text() : $node.html();
21 | } else {
22 | html = $node.contents();
23 | }
24 | $node.replaceWith(`<${tag} ${attribString}>${html}`);
25 | return $;
26 | }
27 |
--------------------------------------------------------------------------------
/src/utils/dom/extract-from-meta.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import { extractFromMeta } from './index';
5 |
6 | describe('extractFromMeta($, metaNames, cachedNames, cleanTags)', () => {
7 | it('extracts an arbitrary meta tag by name', () => {
8 | const $ = cheerio.load(`
9 |
10 |
11 |
12 | `);
13 | const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);
14 |
15 | assert.equal(result, 'bar');
16 | });
17 |
18 | it('returns nothing if a meta name is duplicated', () => {
19 | const $ = cheerio.load(`
20 |
21 |
22 |
23 |
24 | `);
25 | const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);
26 |
27 | assert.equal(result, null);
28 | });
29 |
30 | it('ignores duplicate meta names with empty values', () => {
31 | const $ = cheerio.load(`
32 |
33 |
34 |
35 |
36 | `);
37 | const result = extractFromMeta($, ['foo', 'baz'], ['foo', 'bat']);
38 |
39 | assert.equal(result, 'bar');
40 | });
41 | });
42 |
--------------------------------------------------------------------------------
/src/utils/dom/get-attrs.js:
--------------------------------------------------------------------------------
1 | export default function getAttrs(node) {
2 | const { attribs, attributes } = node;
3 |
4 | if (!attribs && attributes) {
5 | const attrs = Reflect.ownKeys(attributes).reduce((acc, index) => {
6 | const attr = attributes[index];
7 |
8 | if (!attr.name || !attr.value) return acc;
9 |
10 | acc[attr.name] = attr.value;
11 | return acc;
12 | }, {});
13 | return attrs;
14 | }
15 |
16 | return attribs;
17 | }
18 |
--------------------------------------------------------------------------------
/src/utils/dom/get-attrs.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import getAttrs from './get-attrs';
4 |
5 | describe('getAttrs(node)', () => {
6 | it('returns attrs for a raw jquery node', () => {
7 | const domNode = {
8 | attributes: {
9 | 0: {
10 | name: 'class',
11 | value: 'foo bar',
12 | },
13 | },
14 | };
15 |
16 | const attrs = {
17 | class: 'foo bar',
18 | };
19 |
20 | assert.deepEqual(getAttrs(domNode), attrs);
21 | });
22 |
23 | it('returns attrs for a raw cheerio node', () => {
24 | const cheerioNode = {
25 | attribs: {
26 | class: 'foo bar',
27 | id: 'baz bat',
28 | },
29 | };
30 |
31 | assert.deepEqual(getAttrs(cheerioNode), cheerioNode.attribs);
32 | });
33 | });
34 |
--------------------------------------------------------------------------------
/src/utils/dom/is-wordpress.js:
--------------------------------------------------------------------------------
1 | import { IS_WP_SELECTOR } from './constants';
2 |
3 | export default function isWordpress($) {
4 | return $(IS_WP_SELECTOR).length > 0;
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/dom/is-wordpress.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import isWordpress from './is-wordpress';
5 |
6 | describe('isWordpress($)', () => {
7 | it('returns false if a site is not generated by wordpress', () => {
8 | const html = `
9 |
10 |
11 |
12 |
13 |
14 | `;
15 | let $ = cheerio.load(html);
16 |
17 | assert.equal(isWordpress($), false);
18 |
19 | const html2 = `
20 |
21 |
22 |
23 |
24 |
25 | `;
26 | $ = cheerio.load(html2);
27 |
28 | assert.equal(isWordpress($), false);
29 | });
30 |
31 | it('returns true if a site is generated by wordpress', () => {
32 | const html = `
33 |
34 |
35 |
36 |
37 |
38 | `;
39 | const $ = cheerio.load(html);
40 |
41 | assert.equal(isWordpress($), true);
42 | });
43 | });
44 |
--------------------------------------------------------------------------------
/src/utils/dom/link-density.js:
--------------------------------------------------------------------------------
1 | export function textLength(text) {
2 | return text.trim().replace(/\s+/g, ' ').length;
3 | }
4 |
5 | // Determines what percentage of the text
6 | // in a node is link text
7 | // Takes a node, returns a float
8 | export function linkDensity($node) {
9 | const totalTextLength = textLength($node.text());
10 |
11 | const linkText = $node.find('a').text();
12 | const linkLength = textLength(linkText);
13 |
14 | if (totalTextLength > 0) {
15 | return linkLength / totalTextLength;
16 | }
17 | if (totalTextLength === 0 && linkLength > 0) {
18 | return 1;
19 | }
20 |
21 | return 0;
22 | }
23 |
--------------------------------------------------------------------------------
/src/utils/dom/link-density.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import { linkDensity } from './index';
5 |
6 | describe('linkDensity($)', () => {
7 | it('returns 0.5 if half of the text is a link', () => {
8 | const $ = cheerio.load(`
9 |
10 | `);
11 |
12 | const density = linkDensity($('div').first(), $);
13 |
14 | assert.equal(density, 0.5);
15 | });
16 |
17 | it('returns 1 if all of the text is a link', () => {
18 | const $ = cheerio.load(`
19 |
20 | `);
21 |
22 | const density = linkDensity($('div').first(), $);
23 |
24 | assert.equal(density, 1);
25 | });
26 |
27 | it("returns 0 if there's no text", () => {
28 | const $ = cheerio.load(`
29 |
30 | `);
31 |
32 | const density = linkDensity($('div').first());
33 |
34 | assert.equal(density, 0);
35 | });
36 | });
37 |
--------------------------------------------------------------------------------
/src/utils/dom/mark-to-keep.js:
--------------------------------------------------------------------------------
1 | import URL from 'url';
2 |
3 | import { KEEP_SELECTORS, KEEP_CLASS } from './constants';
4 |
5 | export default function markToKeep(article, $, url, tags = []) {
6 | if (tags.length === 0) {
7 | tags = KEEP_SELECTORS;
8 | }
9 |
10 | if (url) {
11 | const { protocol, hostname } = URL.parse(url);
12 | tags = [...tags, `iframe[src^="${protocol}//${hostname}"]`];
13 | }
14 |
15 | $(tags.join(','), article).addClass(KEEP_CLASS);
16 |
17 | return $;
18 | }
19 |
--------------------------------------------------------------------------------
/src/utils/dom/node-is-sufficient.js:
--------------------------------------------------------------------------------
1 | // Given a node, determine if it's article-like enough to return
2 | // param: node (a cheerio node)
3 | // return: boolean
4 |
5 | export default function nodeIsSufficient($node) {
6 | return $node.text().trim().length >= 100;
7 | }
8 |
--------------------------------------------------------------------------------
/src/utils/dom/node-is-sufficient.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import nodeIsSufficient from './node-is-sufficient';
5 |
6 | describe('Utils', () => {
7 | describe('nodeIsSufficient(node)', () => {
8 | it('returns false if node text length < 100 chars', () => {
9 | const $ = cheerio.load(`
10 |
11 |
This is too short
12 |
13 | `);
14 |
15 | assert.equal(nodeIsSufficient($.root()), false);
16 | });
17 |
18 | it('returns true if node text length > 100 chars', () => {
19 | const $ = cheerio.load(`
20 |
21 |
22 | Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean m
23 |
24 |
25 | `);
26 |
27 | assert.equal(nodeIsSufficient($.root()), true);
28 | });
29 | });
30 | });
31 |
--------------------------------------------------------------------------------
/src/utils/dom/paragraphize.js:
--------------------------------------------------------------------------------
1 | import { BLOCK_LEVEL_TAGS_RE } from './constants';
2 |
3 | // Given a node, turn it into a P if it is not already a P, and
4 | // make sure it conforms to the constraints of a P tag (I.E. does
5 | // not contain any other block tags.)
6 | //
7 | // If the node is a
, it treats the following inline siblings
8 | // as if they were its children.
9 | //
10 | // :param node: The node to paragraphize; this is a raw node
11 | // :param $: The cheerio object to handle dom manipulation
12 | // :param br: Whether or not the passed node is a br
13 |
14 | export default function paragraphize(node, $, br = false) {
15 | const $node = $(node);
16 |
17 | if (br) {
18 | let sibling = node.nextSibling;
19 | const p = $('
');
20 |
21 | // while the next node is text or not a block level element
22 | // append it to a new p node
23 | while (
24 | sibling &&
25 | !(sibling.tagName && BLOCK_LEVEL_TAGS_RE.test(sibling.tagName))
26 | ) {
27 | const { nextSibling } = sibling;
28 | $(sibling).appendTo(p);
29 | sibling = nextSibling;
30 | }
31 |
32 | $node.replaceWith(p);
33 | $node.remove();
34 | return $;
35 | }
36 |
37 | return $;
38 | }
39 |
--------------------------------------------------------------------------------
/src/utils/dom/remove-empty.js:
--------------------------------------------------------------------------------
1 | export default function removeEmpty($article, $) {
2 | $article.find('p').each((index, p) => {
3 | const $p = $(p);
4 | if ($p.find('iframe, img').length === 0 && $p.text().trim() === '')
5 | $p.remove();
6 | });
7 |
8 | return $;
9 | }
10 |
--------------------------------------------------------------------------------
/src/utils/dom/rewrite-top-level.js:
--------------------------------------------------------------------------------
1 | import { convertNodeTo } from 'utils/dom';
2 |
3 | // Rewrite the tag name to div if it's a top level node like body or
4 | // html to avoid later complications with multiple body tags.
5 | export default function rewriteTopLevel(article, $) {
6 | // I'm not using context here because
7 | // it's problematic when converting the
8 | // top-level/root node - AP
9 | $ = convertNodeTo($('html'), $, 'div');
10 | $ = convertNodeTo($('body'), $, 'div');
11 |
12 | return $;
13 | }
14 |
--------------------------------------------------------------------------------
/src/utils/dom/rewrite-top-level.test.js:
--------------------------------------------------------------------------------
1 | import cheerio from 'cheerio';
2 | import assert from 'assert';
3 |
4 | import { assertClean } from 'test-helpers';
5 |
6 | import rewriteTopLevel from './rewrite-top-level';
7 |
8 | describe('rewriteTopLevel(node, $)', () => {
9 | it('turns html and body tags into divs', () => {
10 | const $ = cheerio.load(`
11 |
12 | `);
13 | const result = rewriteTopLevel($('html').first(), $);
14 |
15 | assert.equal(result('html').length, 0);
16 | assert.equal(result('body').length, 0);
17 |
18 | if (!cheerio.browser) {
19 | assertClean(
20 | result.html(),
21 | `
22 |
23 | `
24 | );
25 | }
26 | });
27 | });
28 |
--------------------------------------------------------------------------------
/src/utils/dom/set-attr.js:
--------------------------------------------------------------------------------
1 | export default function setAttr(node, attr, val) {
2 | if (node.attribs) {
3 | node.attribs[attr] = val;
4 | } else if (node.attributes) {
5 | node.setAttribute(attr, val);
6 | }
7 |
8 | return node;
9 | }
10 |
--------------------------------------------------------------------------------
/src/utils/dom/set-attr.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import { MockDomNode } from 'test-helpers';
4 | import setAttr from './set-attr';
5 |
6 | describe('setAttr(node, attr, val)', () => {
7 | it('sets attrs for a raw jquery node', () => {
8 | const domNode = new MockDomNode();
9 |
10 | const node = setAttr(domNode, 'class', 'foo');
11 |
12 | assert.equal(node.attributes[0].value, 'foo');
13 | });
14 |
15 | it('sets attrs for a raw cheerio node', () => {
16 | const cheerioNode = {
17 | attribs: {
18 | class: 'foo bar',
19 | id: 'baz bat',
20 | },
21 | };
22 |
23 | const node = setAttr(cheerioNode, 'class', 'foo');
24 |
25 | assert.equal(node.attribs.class, 'foo');
26 | });
27 | });
28 |
--------------------------------------------------------------------------------
/src/utils/dom/set-attrs.js:
--------------------------------------------------------------------------------
1 | export default function setAttrs(node, attrs) {
2 | if (node.attribs) {
3 | node.attribs = attrs;
4 | } else if (node.attributes) {
5 | while (node.attributes.length > 0) {
6 | node.removeAttribute(node.attributes[0].name);
7 | }
8 |
9 | Reflect.ownKeys(attrs).forEach(key => {
10 | node.setAttribute(key, attrs[key]);
11 | });
12 | }
13 |
14 | return node;
15 | }
16 |
--------------------------------------------------------------------------------
/src/utils/dom/set-attrs.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import { MockDomNode } from 'test-helpers';
4 | import setAttrs from './set-attrs';
5 |
6 | describe('setAttrs(node, attrs)', () => {
7 | it('sets attrs for a raw jquery node', () => {
8 | const attrs = {
9 | class: 'baz',
10 | };
11 |
12 | const postAttrs = [
13 | {
14 | name: 'class',
15 | value: 'baz',
16 | },
17 | ];
18 |
19 | const domNode = new MockDomNode();
20 | const node = setAttrs(domNode, attrs);
21 |
22 | assert.deepEqual(node.attributes, postAttrs);
23 | });
24 |
25 | it('sets attrs for a raw cheerio node', () => {
26 | const cheerioNode = {
27 | attribs: {
28 | class: 'foo bar',
29 | id: 'baz bat',
30 | },
31 | };
32 |
33 | const attrs = {
34 | class: 'baz',
35 | id: 'bar',
36 | };
37 |
38 | const node = setAttrs(cheerioNode, attrs);
39 |
40 | assert.deepEqual(node.attribs, attrs);
41 | });
42 | });
43 |
--------------------------------------------------------------------------------
/src/utils/dom/strip-junk-tags.js:
--------------------------------------------------------------------------------
1 | import { STRIP_OUTPUT_TAGS, KEEP_CLASS } from './constants';
2 |
3 | export default function stripJunkTags(article, $, tags = []) {
4 | if (tags.length === 0) {
5 | tags = STRIP_OUTPUT_TAGS;
6 | }
7 |
8 | // Remove matching elements, but ignore
9 | // any element with a class of mercury-parser-keep
10 | $(tags.join(','), article)
11 | .not(`.${KEEP_CLASS}`)
12 | .remove();
13 |
14 | return $;
15 | }
16 |
--------------------------------------------------------------------------------
/src/utils/dom/strip-tags.js:
--------------------------------------------------------------------------------
1 | // strips all tags from a string of text
2 | export default function stripTags(text, $) {
3 | // Wrapping text in html element prevents errors when text
4 | // has no html
5 | const cleanText = $(`
${text}`).text();
6 | return cleanText === '' ? text : cleanText;
7 | }
8 |
--------------------------------------------------------------------------------
/src/utils/dom/strip-tags.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import stripTags from './strip-tags';
5 |
6 | describe('stripTags(title, $)', () => {
7 | it('strips tags from a string of text', () => {
8 | const $ = cheerio.load('
');
9 |
10 | const result = stripTags('What a
Wonderful Day', $);
11 |
12 | assert.equal(result, 'What a Wonderful Day');
13 | });
14 |
15 | it('returns the original text if no tags found', () => {
16 | const $ = cheerio.load('
');
17 |
18 | const result = stripTags('What a Wonderful Day', $);
19 |
20 | assert.equal(result, 'What a Wonderful Day');
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/src/utils/dom/strip-unlikely-candidates.js:
--------------------------------------------------------------------------------
1 | import { CANDIDATES_WHITELIST, CANDIDATES_BLACKLIST } from './constants';
2 |
3 | export default function stripUnlikelyCandidates($) {
4 | // Loop through the provided document and remove any non-link nodes
5 | // that are unlikely candidates for article content.
6 | //
7 | // Links are ignored because there are very often links to content
8 | // that are identified as non-body-content, but may be inside
9 | // article-like content.
10 | //
11 | // :param $: a cheerio object to strip nodes from
12 | // :return $: the cleaned cheerio object
13 | $('*')
14 | .not('a')
15 | .each((index, node) => {
16 | const $node = $(node);
17 | const classes = $node.attr('class');
18 | const id = $node.attr('id');
19 | if (!id && !classes) return;
20 |
21 | const classAndId = `${classes || ''} ${id || ''}`;
22 | if (CANDIDATES_WHITELIST.test(classAndId)) {
23 | return;
24 | }
25 | if (CANDIDATES_BLACKLIST.test(classAndId)) {
26 | $node.remove();
27 | }
28 | });
29 |
30 | return $;
31 | }
32 |
--------------------------------------------------------------------------------
/src/utils/dom/within-comment.js:
--------------------------------------------------------------------------------
1 | import { getAttrs } from 'utils/dom';
2 |
3 | export default function withinComment($node) {
4 | const parents = $node.parents().toArray();
5 | const commentParent = parents.find(parent => {
6 | const attrs = getAttrs(parent);
7 | const { class: nodeClass, id } = attrs;
8 | const classAndId = `${nodeClass} ${id}`;
9 | return classAndId.includes('comment');
10 | });
11 |
12 | return commentParent !== undefined;
13 | }
14 |
--------------------------------------------------------------------------------
/src/utils/dom/within-comment.test.js:
--------------------------------------------------------------------------------
1 | import cheerio from 'cheerio';
2 | import assert from 'assert';
3 |
4 | import withinComment from './within-comment';
5 |
6 | describe('withinComment(node)', () => {
7 | it('returns false if its parent is not a comment', () => {
8 | const $ = cheerio.load(`
9 |
14 | `);
15 | assert.equal(withinComment($('.author').first()), false);
16 | });
17 |
18 | it('returns true if its parent has a class of comment', () => {
19 | const $ = cheerio.load(`
20 |
25 | `);
26 | assert.equal(withinComment($('.author').first()), true);
27 | });
28 |
29 | it('returns true if its parent has an id of comment', () => {
30 | const $ = cheerio.load(`
31 |
36 | `);
37 | assert.equal(withinComment($('.author').first()), true);
38 | });
39 | });
40 |
--------------------------------------------------------------------------------
/src/utils/index.js:
--------------------------------------------------------------------------------
1 | export { default as range } from './range';
2 | export { default as validateUrl } from './validate-url';
3 |
--------------------------------------------------------------------------------
/src/utils/merge-supported-domains.js:
--------------------------------------------------------------------------------
1 | const merge = (extractor, domains) =>
2 | domains.reduce((acc, domain) => {
3 | acc[domain] = extractor;
4 | return acc;
5 | }, {});
6 |
7 | export default function mergeSupportedDomains(extractor) {
8 | return extractor.supportedDomains
9 | ? merge(extractor, [extractor.domain, ...extractor.supportedDomains])
10 | : merge(extractor, [extractor.domain]);
11 | }
12 |
--------------------------------------------------------------------------------
/src/utils/merge-supported-domains.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import mergeSupportedDomains from './merge-supported-domains';
3 |
4 | describe('mergeSupportedDomains(extractor, domains)', () => {
5 | it('returns an object w/domains as keys and extractor as value', () => {
6 | const extractor = {
7 | domain: 'foo.com',
8 | supportedDomains: ['example.com'],
9 | };
10 |
11 | const expected = {
12 | 'foo.com': extractor,
13 | 'example.com': extractor,
14 | };
15 |
16 | const result = mergeSupportedDomains(extractor);
17 | assert.deepEqual(result, expected);
18 | });
19 |
20 | it('returns an object w/single domain if no supportedDomains', () => {
21 | const extractor = {
22 | domain: 'foo.com',
23 | };
24 |
25 | const expected = {
26 | 'foo.com': extractor,
27 | };
28 |
29 | const result = mergeSupportedDomains(extractor);
30 | assert.deepEqual(result, expected);
31 | });
32 | });
33 |
--------------------------------------------------------------------------------
/src/utils/range.js:
--------------------------------------------------------------------------------
1 | export default function* range(start = 1, end = 1) {
2 | while (start <= end) {
3 | yield (start += 1);
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/text/article-base-url.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import articleBaseUrl from './article-base-url';
4 |
5 | describe('articleBaseUrl(url, parsedUrl)', () => {
6 | it('returns the base url of a paginated url', () => {
7 | const url = 'http://example.com/foo/bar/wow-cool/page=10';
8 | const cleaned = 'http://example.com/foo/bar/wow-cool';
9 |
10 | assert.equal(articleBaseUrl(url), cleaned);
11 | });
12 |
13 | it('returns same url if url has no pagination info', () => {
14 | const url = 'http://example.com/foo/bar/wow-cool/';
15 | const cleaned = 'http://example.com/foo/bar/wow-cool';
16 |
17 | assert.equal(articleBaseUrl(url), cleaned);
18 | });
19 | });
20 |
--------------------------------------------------------------------------------
/src/utils/text/constants.js:
--------------------------------------------------------------------------------
1 | // An expression that looks to try to find the page digit within a URL, if
2 | // it exists.
3 | // Matches:
4 | // page=1
5 | // pg=1
6 | // p=1
7 | // paging=12
8 | // pag=7
9 | // pagination/1
10 | // paging/88
11 | // pa/83
12 | // p/11
13 | //
14 | // Does not match:
15 | // pg=102
16 | // page:2
17 | export const PAGE_IN_HREF_RE = new RegExp(
18 | '(page|paging|(p(a|g|ag)?(e|enum|ewanted|ing|ination)))?(=|/)([0-9]{1,3})',
19 | 'i'
20 | );
21 |
22 | export const HAS_ALPHA_RE = /[a-z]/i;
23 |
24 | export const IS_ALPHA_RE = /^[a-z]+$/i;
25 | export const IS_DIGIT_RE = /^[0-9]+$/i;
26 |
27 | export const ENCODING_RE = /charset=([\w-]+)\b/;
28 | export const DEFAULT_ENCODING = 'utf-8';
29 |
--------------------------------------------------------------------------------
/src/utils/text/excerpt-content.js:
--------------------------------------------------------------------------------
1 | export default function excerptContent(content, words = 10) {
2 | return content
3 | .trim()
4 | .split(/\s+/)
5 | .slice(0, words)
6 | .join(' ');
7 | }
8 |
--------------------------------------------------------------------------------
/src/utils/text/excerpt.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import excerptContent from './excerpt-content';
3 |
4 | describe('excerptContent(content, words)', () => {
5 | it('extracts the requested number of words from content', () => {
6 | const content = ' One two three four five six, seven eight, nine, ten.';
7 |
8 | const three = excerptContent(content, 3);
9 | assert.equal(three, 'One two three');
10 |
11 | const ten = excerptContent(content, 10);
12 | assert.equal(ten, content.trim().replace(/\s+/, ' '));
13 | });
14 | });
15 |
--------------------------------------------------------------------------------
/src/utils/text/extract-from-url.js:
--------------------------------------------------------------------------------
1 | // Given a node type to search for, and a list of regular expressions,
2 | // look to see if this extraction can be found in the URL. Expects
3 | // that each expression in r_list will return group(1) as the proper
4 | // string to be cleaned.
5 | // Only used for date_published currently.
6 | export default function extractFromUrl(url, regexList) {
7 | const matchRe = regexList.find(re => re.test(url));
8 | if (matchRe) {
9 | return matchRe.exec(url)[1];
10 | }
11 |
12 | return null;
13 | }
14 |
--------------------------------------------------------------------------------
/src/utils/text/extract-from-url.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import extractFromUrl from './extract-from-url';
4 |
5 | describe('extractFromUrl(url)', () => {
6 | it('extracts datePublished from url', () => {
7 | const url = 'https://example.com/2012/08/01/this-is-good';
8 | const regexList = [new RegExp('/(20\\d{2}/\\d{2}/\\d{2})/')];
9 | const result = extractFromUrl(url, regexList);
10 |
11 | assert.equal(result, '2012/08/01');
12 | });
13 |
14 | it('returns null if nothing found', () => {
15 | const url = 'https://example.com/this-is-good';
16 | const regexList = [new RegExp('/(20\\d{2}/\\d{2}/\\d{2})/')];
17 | const result = extractFromUrl(url, regexList);
18 |
19 | assert.equal(result, null);
20 | });
21 | });
22 |
--------------------------------------------------------------------------------
/src/utils/text/get-encoding.js:
--------------------------------------------------------------------------------
1 | import iconv from 'iconv-lite';
2 | import { DEFAULT_ENCODING, ENCODING_RE } from './constants';
3 |
4 | // check a string for encoding; this is
5 | // used in our fetchResource function to
6 | // ensure correctly encoded responses
7 | export default function getEncoding(str) {
8 | let encoding = DEFAULT_ENCODING;
9 | const matches = ENCODING_RE.exec(str);
10 | if (matches !== null) {
11 | [, str] = matches;
12 | }
13 | if (iconv.encodingExists(str)) {
14 | encoding = str;
15 | }
16 | return encoding;
17 | }
18 |
--------------------------------------------------------------------------------
/src/utils/text/get-encoding.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import getEncoding from './get-encoding';
5 |
6 | // Tests are bypassed in the browser because it has an encoding
7 | // A shim is used /src/shims/iconv-lite.js to decrease load size
8 |
9 | describe('getEncoding(str)', () => {
10 | if (cheerio.browser) return;
11 |
12 | it('returns the encoding as a string', () => {
13 | const contentType = 'text/html; charset=iso-8859-15';
14 | assert.equal(getEncoding(contentType), 'iso-8859-15');
15 | });
16 |
17 | it('returns utf-8 as a default if no encoding found', () => {
18 | const contentType = 'text/html';
19 | assert.equal(getEncoding(contentType), 'utf-8');
20 | });
21 |
22 | it('returns utf-8 if there is an invalid encoding', () => {
23 | const contentType = 'text/html; charset=fake-charset';
24 | assert.equal(getEncoding(contentType), 'utf-8');
25 | });
26 | });
27 |
--------------------------------------------------------------------------------
/src/utils/text/has-sentence-end.js:
--------------------------------------------------------------------------------
1 | // Given a string, return True if it appears to have an ending sentence
2 | // within it, false otherwise.
3 | const SENTENCE_END_RE = new RegExp('.( |$)');
4 | export default function hasSentenceEnd(text) {
5 | return SENTENCE_END_RE.test(text);
6 | }
7 |
--------------------------------------------------------------------------------
/src/utils/text/index.js:
--------------------------------------------------------------------------------
1 | export { default as normalizeSpaces } from './normalize-spaces';
2 | export { default as extractFromUrl } from './extract-from-url';
3 | export { default as pageNumFromUrl } from './page-num-from-url';
4 | export { default as removeAnchor } from './remove-anchor';
5 | export { default as articleBaseUrl } from './article-base-url';
6 | export { default as hasSentenceEnd } from './has-sentence-end';
7 | export { default as excerptContent } from './excerpt-content';
8 | export { default as getEncoding } from './get-encoding';
9 |
--------------------------------------------------------------------------------
/src/utils/text/normalize-spaces.js:
--------------------------------------------------------------------------------
1 | const NORMALIZE_RE = /\s{2,}(?![^<>]*<\/(pre|code|textarea)>)/g;
2 |
3 | export default function normalizeSpaces(text) {
4 | return text.replace(NORMALIZE_RE, ' ').trim();
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/text/normalize-spaces.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import cheerio from 'cheerio';
3 |
4 | import { normalizeSpaces } from './index';
5 |
6 | describe('normalizeSpaces(text)', () => {
7 | it('normalizes spaces from text', () => {
8 | const $ = cheerio.load(`
9 |
10 |
What do you think?
11 |
12 | `);
13 |
14 | const result = normalizeSpaces(
15 | $('*')
16 | .first()
17 | .text()
18 | );
19 | assert.equal(result, 'What do you think?');
20 | });
21 |
22 | it('preserves spaces in preformatted text blocks', () => {
23 | const $ = cheerio.load(`
24 |
25 |
What do you think?
26 |
What happens to spaces?
27 |
28 | `);
29 |
30 | const result = normalizeSpaces($.html());
31 | assert.equal(
32 | result,
33 | '
What do you think?
What happens to spaces?
'
34 | );
35 | });
36 | });
37 |
--------------------------------------------------------------------------------
/src/utils/text/page-num-from-url.js:
--------------------------------------------------------------------------------
1 | import { PAGE_IN_HREF_RE } from './constants';
2 |
3 | export default function pageNumFromUrl(url) {
4 | const matches = url.match(PAGE_IN_HREF_RE);
5 | if (!matches) return null;
6 |
7 | const pageNum = parseInt(matches[6], 10);
8 |
9 | // Return pageNum < 100, otherwise
10 | // return null
11 | return pageNum < 100 ? pageNum : null;
12 | }
13 |
--------------------------------------------------------------------------------
/src/utils/text/page-num-from-url.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import pageNumFromUrl from './page-num-from-url';
4 |
5 | describe('pageNumFromUrl(url)', () => {
6 | it('returns null if there is no page num in the url', () => {
7 | assert.equal(pageNumFromUrl('http://example.com'), null);
8 | assert.equal(pageNumFromUrl('http://example.com/?pg=102'), null);
9 | assert.equal(pageNumFromUrl('http://example.com/?page:102'), null);
10 | });
11 |
12 | it('returns a page num if one matches the url', () => {
13 | assert.equal(pageNumFromUrl('http://example.com/foo?page=1'), 1);
14 | assert.equal(pageNumFromUrl('http://example.com/foo?pg=1'), 1);
15 | assert.equal(pageNumFromUrl('http://example.com/foo?p=1'), 1);
16 | assert.equal(pageNumFromUrl('http://example.com/foo?paging=1'), 1);
17 | assert.equal(pageNumFromUrl('http://example.com/foo?pag=1'), 1);
18 | assert.equal(pageNumFromUrl('http://example.com/foo?pagination/1'), 1);
19 | assert.equal(pageNumFromUrl('http://example.com/foo?paging/99'), 99);
20 | assert.equal(pageNumFromUrl('http://example.com/foo?pa/99'), 99);
21 | assert.equal(pageNumFromUrl('http://example.com/foo?p/99'), 99);
22 | });
23 | });
24 |
--------------------------------------------------------------------------------
/src/utils/text/remove-anchor.js:
--------------------------------------------------------------------------------
1 | export default function removeAnchor(url) {
2 | return url.split('#')[0].replace(/\/$/, '');
3 | }
4 |
--------------------------------------------------------------------------------
/src/utils/text/remove-anchor.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 |
3 | import removeAnchor from './remove-anchor';
4 |
5 | describe('removeAnchor(url)', () => {
6 | it('returns a url w/out #anchor', () => {
7 | const url = 'http://example.com/foo/bar/wow-cool/page=10/#wow';
8 | const cleaned = 'http://example.com/foo/bar/wow-cool/page=10';
9 |
10 | assert.equal(removeAnchor(url), cleaned);
11 | });
12 |
13 | it('returns same url if url has no anchor found', () => {
14 | const url = 'http://example.com/foo/bar/wow-cool';
15 | const cleaned = 'http://example.com/foo/bar/wow-cool';
16 |
17 | assert.equal(removeAnchor(url), cleaned);
18 | });
19 | });
20 |
--------------------------------------------------------------------------------
/src/utils/validate-url.js:
--------------------------------------------------------------------------------
1 | // extremely simple url validation as a first step
2 | export default function validateUrl({ hostname }) {
3 | // If this isn't a valid url, return an error message
4 | return !!hostname;
5 | }
6 |
--------------------------------------------------------------------------------
/src/utils/validate-url.test.js:
--------------------------------------------------------------------------------
1 | import assert from 'assert';
2 | import URL from 'url';
3 |
4 | import validateUrl from './validate-url';
5 |
6 | describe('validateUrl(parsedUrl)', () => {
7 | it('returns false if url is not valid', () => {
8 | const url = URL.parse('example.com');
9 | const valid = validateUrl(url);
10 |
11 | assert.equal(valid, false);
12 | });
13 |
14 | it('returns true if url is valid', () => {
15 | const url = URL.parse('http://example.com');
16 | const valid = validateUrl(url);
17 |
18 | assert.equal(valid, true);
19 | });
20 | });
21 |
--------------------------------------------------------------------------------