├── .travis.yml ├── .gitignore ├── test ├── test-pages │ ├── ietf-1 │ │ └── expected-metadata.json │ ├── replace-brs │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 001 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── daringfireball-1 │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── bbc-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── mozilla-2 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── liberation-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── buzzfeed-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── keep-images │ │ └── expected-metadata.json │ ├── medium-2 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── webmd-2 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── basic-tags-cleaning │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-extra-brs │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── remove-script-tags │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── ars-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── remove-extra-paragraphs │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── comment-inside-script-parsing │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── 002 │ │ └── expected-metadata.json │ ├── lemonde-1 │ │ └── expected-metadata.json │ ├── medium-1 │ │ └── expected-metadata.json │ ├── tmz-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── wapo-2 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── heise │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── salon-1 │ │ └── expected-metadata.json │ ├── wapo-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── herald-sun-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── mozilla-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── simplyfound-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── clean-links │ │ └── expected-metadata.json │ ├── ehow-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── lifehacker-working │ │ └── expected-metadata.json │ ├── webmd-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── lifehacker-post-comment-load │ │ └── expected-metadata.json │ ├── iab-1 │ │ ├── expected-metadata.json │ │ └── expected.html │ ├── svg-parsing │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── lwn-1 │ │ └── expected-metadata.json │ ├── base-url │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── embedded-videos │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── style-tags-removal │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── replace-font-tags │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── social-buttons │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── normalize-spaces │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ ├── reordering-paragraphs │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html │ └── missing-paragraphs │ │ ├── expected-metadata.json │ │ ├── expected.html │ │ └── source.html ├── utils.js ├── generate-testcase.js └── test-readability.js ├── package.json ├── index.js ├── benchmarks ├── benchmark-reporter.js └── benchmarks.js ├── README.md └── .eslintrc /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - '5.3' 5 | script: 6 | - npm run lint 7 | - npm run test 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | npm-debug.log 3 | node_modules 4 | .metadata 5 | *.pyc 6 | *~ 7 | .*.sw? 8 | .sw? 9 | *.jar 10 | *.xpi 11 | -------------------------------------------------------------------------------- /test/test-pages/ietf-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "draft-dejong-remotestorage-04 - remoteStorage", 3 | "byline": "AUTHORING", 4 | "readerable": true 5 | } 6 | -------------------------------------------------------------------------------- /test/test-pages/replace-brs/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Replace brs test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/001/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Get your Frontend JavaScript Code Covered | Code", 3 | "byline": "Nicolas Perriault —", 4 | "excerpt": "Nicolas Perriault's homepage.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/daringfireball-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Daring Fireball: Colophon", 3 | "byline": null, 4 | "excerpt": "Daring Fireball is written and produced by John Gruber.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/bbc-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Obama admits US gun laws are his 'biggest frustration' - BBC News", 3 | "byline": null, 4 | "dir": "ltr", 5 | "excerpt": "President Barack Obama tells the BBC his failure to pass", 6 | "readerable": true 7 | } 8 | -------------------------------------------------------------------------------- /test/test-pages/mozilla-2/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Welcome to Firefox Developer Edition", 3 | "byline": null, 4 | "dir": "ltr", 5 | "excerpt": "Get to know the features that make it the most complete browser for building the Web.", 6 | "readerable": false 7 | } 8 | -------------------------------------------------------------------------------- /test/test-pages/liberation-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Un troisième Français mort dans le séisme au Népal", 3 | "byline": "AFP", 4 | "excerpt": "Laurent Fabius a accueilli jeudi matin à Roissy un premier avion spécial ramenant des rescapés.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/buzzfeed-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Student Dies After Diet Pills She Bought Online \"Burned Her Up From Within\"", 3 | "byline": "Mark Di Stefano", 4 | "excerpt": "An inquest into Eloise Parry's death has been adjourned until July...", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/keep-images/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Inside the Deep Web Drug Lab", 3 | "byline": "Joseph Cox", 4 | "excerpt": "Welcome to DoctorX’s Barcelona lab, where the drugs you bought online are tested for safety and purity. No questions ask…", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/medium-2/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "On Behalf of “Literally”", 3 | "byline": "Courtney Kirchoff", 4 | "excerpt": "In defense of the word “literally” and why you or someone you know should stop misusing the word, lest they drive us fig…", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/webmd-2/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Superbugs: What They Are and How You Get Them", 3 | "byline": "By Kelli Miller\n WebMD Health News", 4 | "excerpt": "Drug-resistant bacteria, dubbed", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/basic-tags-cleaning/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Basic tag cleaning test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/remove-extra-brs/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Remove trailing brs test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/remove-script-tags/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Remove script tags test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/ars-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Just-released Minecraft exploit makes it easy to crash game servers", 3 | "byline": "by Dan Goodin - Apr 16, 2015 8:02 pm UTC", 4 | "excerpt": "Two-year-old bug exposes thousands of servers to crippling attack.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/remove-extra-paragraphs/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Replace font tags test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/comment-inside-script-parsing/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Test script parsing", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/002/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "This API is so Fetching!", 3 | "byline": "Nikhil Marathe", 4 | "excerpt": "For more than a decade the Web has used XMLHttpRequest (XHR) to achieve asynchronous requests in JavaScript. While very useful, XHR is not a very ...", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/lemonde-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Le projet de loi sur le renseignement massivement approuvé à l'Assemblée", 3 | "byline": null, 4 | "excerpt": "Largement approuvé par les députés, le texte sera désormais examiné par le Sénat, puis le Conseil constitutionnel.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/medium-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "The Open Journalism Project: Better Student Journalism", 3 | "byline": "Pippin Lee", 4 | "excerpt": "We pushed out the first version of the Open Journalism site in January. Here’s what we’ve learned about student journali…", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/tmz-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Lupita Nyong'o's $150K Pearl Oscar Dress -- STOLEN!!!", 3 | "byline": null, 4 | "excerpt": "Lupita Nyong'o's now-famous Oscar dress -- adorned in pearls -- was stolen right out of her hotel room ... TMZ has learned. Law enforcement sources tell…", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/wapo-2/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Where do strained U.S.-Israeli relations go after Netanyahu’s victory?", 3 | "byline": "By Steven Mufson", 4 | "excerpt": "Few foreign leaders have so brazenly stood up to President Obama and the relationship could face its next test this month.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/heise/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "1Password für Mac generiert Einmal-Passwörter", 3 | "byline": null, 4 | "excerpt": "Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/salon-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "The sharing economy is a lie: Uber, Ayn Rand and the truth about tech and libertarians", 3 | "byline": "Joanna Rothkopf", 4 | "excerpt": "Disruptive companies talk a good game about sharing. Uber's really just an under-regulated company making riches", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/wapo-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Attack stokes instability fears in North Africa", 3 | "byline": "By Erin Cunningham", 4 | "excerpt": "The assault on Tunisia’s most renowned museum, in which gunmen killed at least 19 people, could heighten tensions in a nation that has become deeply divided between pro- and anti-Islamist factions.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/herald-sun-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Angry media won’t buckle over new surveillance laws", 3 | "byline": "JOE HILDEBRAND", 4 | "excerpt": "A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/mozilla-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Firefox — Customize and make it your own — The most flexible browser on the Web", 3 | "byline": null, 4 | "dir": "ltr", 5 | "excerpt": "It’s easier than ever to personalize Firefox and make it work the way\n you do.\n No other browser gives you so much choice and flexibility.", 6 | "readerable": true 7 | } 8 | -------------------------------------------------------------------------------- /test/test-pages/simplyfound-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Raspberry Pi 3 - The credit card sized PC that cost only $35 - All-time bestselling computer in UK", 3 | "byline": null, 4 | "excerpt": "The Raspberry Pi Foundation started by a handful of volunteers in 2012 when they released the original Raspberry Pi 256MB Model B without knowing what to expect. In a short four-year period they have grown to over sixty full-time employees and ha...", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/clean-links/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Bartleby the Scrivener Web Study Text", 3 | "byline": null, 4 | "excerpt": "Ere introducing the scrivener, as he first appeared to me, it is fit \n I make some mention of myself, my employees, my business, my chambers, \n and general surroundings; because some such description is indispensable \n to an adequate understanding of the chief character about to be presented.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/ehow-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "How to Build a Terrarium | eHow", 3 | "byline": "Lucy Akins", 4 | "excerpt": "How to Build a Terrarium. Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You...", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/lifehacker-working/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "How to Program Your Mind to Stop Buying Crap You Don’t Need", 3 | "byline": "Patrick Allan", 4 | "excerpt": "We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/webmd-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Babies Who Eat Peanuts Early May Avoid Allergy", 3 | "byline": "By Brenda Goodman, MA\n WebMD Health News", 4 | "excerpt": "Life-threatening peanut allergies have mysteriously been on the rise in the past decade, with little hope for a cure. But a groundbreaking new study may offer a way to stem that rise, while another may offer some hope for those who are already allergic.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/lifehacker-post-comment-load/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "How to Program Your Mind to Stop Buying Crap You Don’t Need", 3 | "byline": "Patrick Allan", 4 | "excerpt": "We all buy things from time to time that we don't really need. It's okay to appeal to your wants every once in a while, as long as you're in control. If you struggle with clutter, impulse buys, and buyer's remorse, here's how to put your mind in the right place before you even set foot in a store.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/iab-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Getting LEAN with Digital Ad UX | IAB", 3 | "byline": "By\n\t\t\tScott Cunningham", 4 | "excerpt": "We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience. Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. … Continued", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/svg-parsing/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "SVG parsing", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\ntempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\nquis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\nconsequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\ncillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\nproident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/lwn-1/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "LWN.net Weekly Edition for March 26, 2015 [LWN.net]", 3 | "byline": "By Nathan Willis\n March 25, 2015", 4 | "excerpt": "The Arduino has been one of the biggest success stories of the open-hardware movement, but that success does not protect it from internal conflict. In recent months, two of the project's founders have come into conflict about the direction of future efforts—and that conflict has turned into a legal dispute about who owns the rights to the Arduino trademark.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/base-url/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Base URL test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/embedded-videos/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Embedded videos test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/style-tags-removal/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Style tags removal", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/replace-font-tags/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Replace font tags test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/social-buttons/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Share buttons removal test", 3 | "byline": null, 4 | "excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/normalize-spaces/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Normalize space test", 3 | "byline": null, 4 | "excerpt": "Lorem\n ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n\ttab here\n incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", 5 | "readerable": false 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/reordering-paragraphs/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "", 3 | "byline": null, 4 | "excerpt": "Regarding item# 11111, under sufficiently extreme conditions, quarks may\n become deconfined and exist as free particles. In the course of asymptotic\n freedom, the strong interaction becomes weaker at higher temperatures.\n Eventually, color confinement would be lost and an extremely hot plasma\n of freely moving quarks and gluons would be formed. This theoretical phase\n of matter is called quark-gluon plasma.[81] The exact conditions needed\n to give rise to this state are unknown and have been the subject of a great\n deal of speculation and experimentation.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/test-pages/normalize-spaces/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tab here incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

5 |
6 |
-------------------------------------------------------------------------------- /test/test-pages/style-tags-removal/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

5 |
6 |
-------------------------------------------------------------------------------- /test/test-pages/basic-tags-cleaning/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

4 |

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

5 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |
7 |
8 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

9 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

10 |
11 |
-------------------------------------------------------------------------------- /test/test-pages/remove-script-tags/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

4 |

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

5 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |
7 |
8 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

9 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

10 |
11 |
-------------------------------------------------------------------------------- /test/test-pages/remove-extra-paragraphs/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

4 |

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

5 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |
7 |
8 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

9 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

10 |
11 |
-------------------------------------------------------------------------------- /test/test-pages/comment-inside-script-parsing/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

4 |

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

5 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |
7 |
8 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

9 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

10 |
11 |
-------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "readability", 3 | "version": "0.0.1", 4 | "description": "A standalone version of the readability library used for Firefox Reader View.", 5 | "main": "Readability.js", 6 | "scripts": { 7 | "lint": "eslint .", 8 | "test": "mocha test/test-*.js", 9 | "generate-testcase": "node test/generate-testcase.js", 10 | "perf": "matcha benchmarks/benchmarks.js", 11 | "perf-reference": "READABILITY_PERF_REFERENCE=1 matcha --reporter ./benchmarks/benchmark-reporter.js benchmarks/benchmarks.js" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "https://github.com/mozilla/readability" 16 | }, 17 | "author": "", 18 | "license": "Apache-2.0", 19 | "bugs": { 20 | "url": "https://github.com/mozilla/readability/issues" 21 | }, 22 | "homepage": "https://github.com/mozilla/readability", 23 | "devDependencies": { 24 | "chai": "^2.1.*", 25 | "eslint": ">1.10", 26 | "js-beautify": "^1.5.5", 27 | "jsdom": "^7.0", 28 | "matcha": "^0.6.0", 29 | "mocha": "^2.2.*" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/test-pages/remove-extra-brs/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

4 |

5 |

Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

6 |

7 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

8 |
9 |
10 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

11 |

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

12 |
13 |
-------------------------------------------------------------------------------- /test/test-pages/replace-font-tags/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

5 |
6 |
-------------------------------------------------------------------------------- /test/test-pages/replace-brs/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum

4 |

dolor sit

5 |

amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |
7 |
8 |

Tempor

9 |

incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

10 |
11 |
-------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | 4 | // We want to load Readability and JSDOMParser, which aren't set up as commonjs libraries, 5 | // and so we need to do some hocus-pocus with 'vm' to import them on a separate scope 6 | // (identical) scope context. 7 | var vm = require("vm"); 8 | var readabilityPath = path.join(__dirname, "Readability.js"); 9 | var jsdomPath = path.join(__dirname, "JSDOMParser.js"); 10 | 11 | 12 | var scopeContext = {}; 13 | // We generally expect dump() and console.{whatever} to work, so make these available 14 | // in the scope we're using: 15 | scopeContext.dump = console.log; 16 | scopeContext.console = console; 17 | 18 | // Actually load files. NB: if either of the files has parse errors, 19 | // node is dumb and shows you a syntax error *at this callsite* . Don't try to find 20 | // a syntax error on this line, there isn't one. Go look in the file it's loading instead. 21 | vm.runInNewContext(fs.readFileSync(jsdomPath), scopeContext, jsdomPath); 22 | vm.runInNewContext(fs.readFileSync(readabilityPath), scopeContext, readabilityPath); 23 | 24 | module.exports = { 25 | Readability: scopeContext.Readability, 26 | JSDOMParser: scopeContext.JSDOMParser 27 | }; 28 | -------------------------------------------------------------------------------- /benchmarks/benchmark-reporter.js: -------------------------------------------------------------------------------- 1 | var clean = require('matcha/lib/matcha/reporters/clean'); 2 | 3 | function average(list) { 4 | if (!list.length) 5 | return 0; 6 | 7 | var sum = list.reduce(function(previous, current) { 8 | return previous + current; 9 | }); 10 | return (sum / list.length).toFixed(0); 11 | } 12 | 13 | 14 | // Like clean, but also produces an average: 15 | module.exports = function(runner, utils) { 16 | var humanize = utils.humanize; 17 | var padBefore = utils.padBefore; 18 | var color = utils.color; 19 | var results = {}; 20 | var currentResults = []; 21 | runner.on('bench end', function(benchResults) { 22 | currentResults.push(benchResults.ops); 23 | }); 24 | runner.on('suite end', function(suite) { 25 | var avg = humanize(average(currentResults)); 26 | console.log(padBefore(avg + ' op/s', 22) + ' » ' + suite.title); 27 | console.log(); 28 | results[suite.title] = avg; 29 | currentResults = []; 30 | }); 31 | 32 | runner.on('end', function() { 33 | for (var k in results) { 34 | console.log(color(padBefore(k, 30) + ': ', 'gray') + results[k] + ' op/s'); 35 | } 36 | console.log(); 37 | }); 38 | 39 | clean(runner, utils); 40 | }; 41 | -------------------------------------------------------------------------------- /test/test-pages/missing-paragraphs/expected-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy\n eirmod tempor invidunt", 3 | "byline": "Henri Sivonen", 4 | "excerpt": "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy\n eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam\n voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit\n amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam\n nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,\n sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\n Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor\n sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed\n diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,\n sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum.\n Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor\n sit amet.", 5 | "readerable": true 6 | } 7 | -------------------------------------------------------------------------------- /test/utils.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | var prettyPrint = require("js-beautify").html; 4 | 5 | function readFile(filePath) { 6 | return fs.readFileSync(filePath, {encoding: "utf-8"}).trim(); 7 | } 8 | 9 | function readJSON(jsonPath) { 10 | return JSON.parse(readFile(jsonPath)); 11 | } 12 | 13 | var testPageRoot = path.join(__dirname, "test-pages"); 14 | 15 | exports.getTestPages = function() { 16 | return fs.readdirSync(testPageRoot).map(function(dir) { 17 | return { 18 | dir: dir, 19 | source: readFile(path.join(testPageRoot, dir, "source.html")), 20 | expectedContent: readFile(path.join(testPageRoot, dir, "expected.html")), 21 | expectedMetadata: readJSON(path.join(testPageRoot, dir, "expected-metadata.json")), 22 | }; 23 | }); 24 | }; 25 | 26 | exports.prettyPrint = function(html) { 27 | return prettyPrint(html, { 28 | "indent_size": 4, 29 | "indent_char": " ", 30 | "indent_level": 0, 31 | "indent_with_tabs": false, 32 | "preserve_newlines": false, 33 | "break_chained_methods": false, 34 | "eval_code": false, 35 | "unescape_strings": false, 36 | "wrap_line_length": 0, 37 | "wrap_attributes": "auto", 38 | "wrap_attributes_indent_size": 4 39 | }); 40 | }; 41 | -------------------------------------------------------------------------------- /test/test-pages/replace-brs/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Replace brs test 6 | 7 | 8 |
9 |

Lorem

10 |
11 | Lorem ipsum
dolor sit


amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 13 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 14 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 15 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 16 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 17 |
18 |

Foo

19 |
20 | Tempor

incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 21 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 22 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 23 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 24 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 25 |
26 |
27 | 28 | 29 | -------------------------------------------------------------------------------- /test/test-pages/normalize-spaces/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Normalize space test 6 | 7 | 8 |
9 |

Lorem

10 |
11 | Lorem 12 | ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 13 | tab here 14 | incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 15 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 16 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 17 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 18 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 19 |
20 |

Foo

21 |
22 | Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 23 | quis nostrud exercitation 24 | 25 | 26 | 27 | 28 | ullamco laboris nisi ut aliquip ex ea commodo 29 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 30 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 31 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 32 |
33 |
34 | 35 | 36 | -------------------------------------------------------------------------------- /test/test-pages/remove-extra-brs/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Remove trailing brs test 6 | 7 | 8 |
9 |

Lorem

10 |
11 |
12 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 13 | tempor incididunt ut labore et dolore magna aliqua.

14 |

Ut enim ad minim veniam, 15 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 16 | consequat.


17 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 18 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 19 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

20 |
21 |

Foo

22 |
23 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 24 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 25 | consequat.

26 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 27 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 28 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

29 |
30 |
31 | 32 | 33 | -------------------------------------------------------------------------------- /test/test-pages/replace-font-tags/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Replace font tags test 6 | 7 | 8 |
9 |

Lorem

10 |
11 | Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 13 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 14 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 15 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 16 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 17 |
18 |

Foo

19 |
20 | Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 21 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 22 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 23 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 24 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 25 |
26 |
27 | 28 | 29 | -------------------------------------------------------------------------------- /test/test-pages/comment-inside-script-parsing/source.html: -------------------------------------------------------------------------------- 1 | 2 | Test script parsing 3 | 4 | 10 |
11 |

Lorem

12 |
13 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 14 | tempor incididunt ut labore et dolore magna aliqua.

15 |

Ut enim ad minim veniam, 16 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 17 | consequat.

18 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 19 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 20 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

21 |
22 |

Foo

23 |
24 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 25 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 26 | consequat.

27 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 28 | cillum dolore eu fugiat nulla pariatur. 29 | Excepteur sint occaecat cupidatat non 30 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

31 |
32 |
33 | 34 | 35 | -------------------------------------------------------------------------------- /test/test-pages/remove-extra-paragraphs/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Replace font tags test 6 | 7 | 8 |
9 |

Lorem

10 |
11 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua.

13 |

14 |

Ut enim ad minim veniam, 15 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 16 | consequat.

17 |

18 |

19 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 20 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 21 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

22 |

23 |
24 |

Foo

25 |
26 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 27 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 28 | consequat.

29 |

30 |

31 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 32 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 33 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

34 |

35 | 36 | 37 |

38 |
39 |
40 | 41 | 42 | -------------------------------------------------------------------------------- /test/test-pages/style-tags-removal/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Style tags removal 6 | 7 | 8 | 9 |
10 |

Lorem

11 | 14 |
15 | Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 16 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 17 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 18 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 19 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 20 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 21 |
22 | 27 |

Foo

28 |
29 | Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 30 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 31 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 32 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 33 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 34 |
35 |
36 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /test/test-pages/basic-tags-cleaning/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Basic tag cleaning test 6 | 7 | 8 |
9 |

Lorem

10 |
11 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua.

13 |

Ut enim ad minim veniam, 14 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 15 | consequat.

16 | 17 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 18 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 19 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

20 |
21 |

Foo

22 |
23 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 24 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 25 | consequat.

26 | 27 | 28 | 29 | 30 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 31 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 32 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

33 |
34 |
35 | 36 | 37 | -------------------------------------------------------------------------------- /test/test-pages/remove-script-tags/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Remove script tags test 6 | 7 | 8 | 9 |
10 |

Lorem

11 |
12 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 13 | tempor incididunt ut labore et dolore magna aliqua.

14 |

Ut enim ad minim veniam, 15 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 16 | consequat.

17 | 20 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 21 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 22 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

23 |
24 | 27 |

Foo

28 |
29 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 30 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 31 | consequat.

32 | 33 |

Duis aute irure dolor in reprehenderit in voluptate velit esse 34 | cillum dolore eu fugiat nulla pariatur. 35 | 38 | Excepteur sint occaecat cupidatat non 39 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

40 |
41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /test/test-pages/reordering-paragraphs/expected.html: -------------------------------------------------------------------------------- 1 |
2 |

Regarding item# 11111, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

3 |

Regarding item# 22222, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

4 |

Regarding item# 33333, under sufficiently extreme conditions, quarks may become deconfined and exist as free particles. In the course of asymptotic freedom, the strong interaction becomes weaker at higher temperatures. Eventually, color confinement would be lost and an extremely hot plasma of freely moving quarks and gluons would be formed. This theoretical phase of matter is called quark-gluon plasma.[81] The exact conditions needed to give rise to this state are unknown and have been the subject of a great deal of speculation and experimentation.

5 |
-------------------------------------------------------------------------------- /test/test-pages/base-url/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Links

5 |

link

6 |

link

7 |

link

8 |

link

9 |

link

10 |

link

11 |

link

12 |

link

13 |

Images

14 |

15 |

16 |

17 |

18 |

19 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

20 |
21 |
22 | -------------------------------------------------------------------------------- /test/test-pages/tmz-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 |

$150K Pearl Oscar Dress ... STOLEN!!!!

5 |
6 | 2/26/2015 7:11 AM PST BY TMZ STAFF 7 |
8 |
9 |

EXCLUSIVE

10 |

0225-lupita-nyongo-getty-01Lupita Nyong'o's now-famous Oscar dress -- adorned in pearls -- was stolen right out of her hotel room ... TMZ has learned.

11 |

Law enforcement sources tell TMZ ... the dress was taken out of Lupita's room at The London West Hollywood. The dress is made of pearls ... 6,000 white Akoya pearls. It's valued at $150,000.

12 |

Our sources say Lupita told cops it was taken from her room sometime between 8 AM and 9 PM Wednesday ... while she was gone.  

13 |

We're told there is security footage that cops are looking at that could catch the culprit right in the act. 

14 |

update_graphic_red_bar12:00 PM PT -- Sheriff's deputies were at The London Thursday morning.  We know they were in the manager's office and we're told they have looked at security footage to determine if they can ID the culprit.

15 |

0226-SUB-london-hotel-swipe-tmz-02

16 | 17 |
18 |
19 |
-------------------------------------------------------------------------------- /test/test-pages/base-url/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Base URL test 6 | 7 | 8 |
9 |

Lorem

10 |
11 | Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 13 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 14 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 15 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 16 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 17 |
18 |

Links

19 |

link

20 |

link

21 |

link

22 |

link

23 |

link

24 |

link

25 |

link

26 |

link

27 |

Images

28 |

29 |

30 |

31 |

32 |

33 |

Foo

34 |
35 | Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 36 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 37 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 38 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 39 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 40 |
41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /test/test-pages/embedded-videos/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Videos

5 |

At root

6 | 7 | 8 | 9 |

In a paragraph

10 |

11 | 12 |

13 |

In a div

14 |

15 | 16 |

17 |

Foo

18 |

Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

19 |
20 |
-------------------------------------------------------------------------------- /benchmarks/benchmarks.js: -------------------------------------------------------------------------------- 1 | var getTestPages = require("../test/utils").getTestPages; 2 | 3 | var readability = require("../index.js"); 4 | var Readability = readability.Readability; 5 | var JSDOMParser = readability.JSDOMParser; 6 | 7 | var referenceTestPages = [ 8 | "002", 9 | "herald-sun-1", 10 | "lifehacker-working", 11 | "lifehacker-post-comment-load", 12 | "medium-1", 13 | "medium-2", 14 | "salon-1", 15 | "tmz-1", 16 | "wapo-1", 17 | "wapo-2", 18 | "webmd-1", 19 | ]; 20 | 21 | var testPages = getTestPages(); 22 | 23 | if (process.env.READABILITY_PERF_REFERENCE === "1") { 24 | testPages = testPages.filter(function(testPage) { 25 | return referenceTestPages.indexOf(testPage.dir) !== -1; 26 | }); 27 | } 28 | 29 | suite("JSDOMParser test page perf", function () { 30 | set("iterations", 1); 31 | set("type", "static"); 32 | 33 | testPages.forEach(function(testPage) { 34 | bench(testPage.dir + " document parse perf", function() { 35 | new JSDOMParser().parse(testPage.source); 36 | }); 37 | }); 38 | }); 39 | 40 | 41 | suite("Readability test page perf", function () { 42 | set("iterations", 1); 43 | set("type", "static"); 44 | 45 | var uri = { 46 | spec: "http://fakehost/test/page.html", 47 | host: "fakehost", 48 | prePath: "http://fakehost", 49 | scheme: "http", 50 | pathBase: "http://fakehost/test" 51 | }; 52 | testPages.forEach(function(testPage) { 53 | var doc = new JSDOMParser().parse(testPage.source); 54 | bench(testPage.dir + " readability perf", function() { 55 | new Readability(uri, doc).parse(); 56 | }); 57 | }); 58 | }); 59 | 60 | suite("isProbablyReaderable perf", function () { 61 | set("iterations", 1); 62 | set("type", "static"); 63 | 64 | var uri = { 65 | spec: "http://fakehost/test/page.html", 66 | host: "fakehost", 67 | prePath: "http://fakehost", 68 | scheme: "http", 69 | pathBase: "http://fakehost/test" 70 | }; 71 | testPages.forEach(function(testPage) { 72 | var doc = new JSDOMParser().parse(testPage.source); 73 | bench(testPage.dir + " readability perf", function() { 74 | new Readability(uri, doc).isProbablyReaderable(); 75 | }); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /test/test-pages/embedded-videos/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Embedded videos test 6 | 7 | 8 |
9 |

Lorem

10 |
11 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 12 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 13 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 14 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 15 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 16 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

17 |
18 |

Videos

19 |

At root

20 | 22 | 24 | 27 |

In a paragraph

28 |

30 |

In a div

31 |
33 |

Foo

34 |
35 | Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 36 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 37 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 38 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 39 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 40 |
41 |
42 | 43 | 44 | -------------------------------------------------------------------------------- /test/test-pages/reordering-paragraphs/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
7 |

Regarding item# 11111, under sufficiently extreme conditions, quarks may 8 | become deconfined and exist as free particles. In the course of asymptotic 9 | freedom, the strong interaction becomes weaker at higher temperatures. 10 | Eventually, color confinement would be lost and an extremely hot plasma 11 | of freely moving quarks and gluons would be formed. This theoretical phase 12 | of matter is called quark-gluon plasma.[81] The exact conditions needed 13 | to give rise to this state are unknown and have been the subject of a great 14 | deal of speculation and experimentation.

15 |

Regarding item# 22222, under sufficiently extreme conditions, quarks may 16 | become deconfined and exist as free particles. In the course of asymptotic 17 | freedom, the strong interaction becomes weaker at higher temperatures. 18 | Eventually, color confinement would be lost and an extremely hot plasma 19 | of freely moving quarks and gluons would be formed. This theoretical phase 20 | of matter is called quark-gluon plasma.[81] The exact conditions needed 21 | to give rise to this state are unknown and have been the subject of a great 22 | deal of speculation and experimentation.

23 |

Regarding item# 33333, under sufficiently extreme conditions, quarks may 24 | become deconfined and exist as free particles. In the course of asymptotic 25 | freedom, the strong interaction becomes weaker at higher temperatures. 26 | Eventually, color confinement would be lost and an extremely hot plasma 27 | of freely moving quarks and gluons would be formed. This theoretical phase 28 | of matter is called quark-gluon plasma.[81] The exact conditions needed 29 | to give rise to this state are unknown and have been the subject of a great 30 | deal of speculation and experimentation.

31 |
32 | 33 | 34 | -------------------------------------------------------------------------------- /test/test-pages/simplyfound-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

The Raspberry Pi Foundation started by a handful of volunteers in 2012 when they released the original Raspberry Pi 256MB Model B without knowing what to expect.  In a short four-year period they have grown to over sixty full-time employees and have shipped over eight million units to-date.  Raspberry Pi has achieved new heights by being shipped to the International Space Station for research and by being an affordable computing platforms used by teachers throughout the world.  "It has become the all-time best-selling computer in the UK".

4 |

Raspberry Pi 3 - A credit card sized PC that only costs $35 - Image: Raspberry Pi Foundation

5 |

Raspberry Pi Foundation is charity organization that pushes for a digital revolution with a mission to inspire kids to learn by creating computer-powered objects.  The foundation also helps teachers learn computing  skills through free training and readily available tutorials & example code for creating cool things such as music.

6 |

Raspberry Pi in educations - Image: Raspberry Pi Foundation

7 |

In celebration of their 4th year anniversary, the foundation has released Raspberry Pi 3 with the same price tag of $35 USD.  The 3rd revision features a 1.2GHz 64-bit quad-core ARM CPU with integrated Bluetooth 4.1 and 802.11n wireless LAN chipsets.  The ARM Cortex-A53 CPU along with other architectural enhancements making it the fastest Raspberry Pi to-date.  The 3rd revision is reportedly about 50-60% times faster than its predecessor Raspberry Pi 2 and about 10 times faster then the original Raspberry PI.

8 |

Raspberry Pi - Various Usage

9 |

Raspberry Pi 3 is now available via many online resellers.  At this time, you should use a recent 32-bit NOOBS or Raspbian image from their downloads page with a promise of a switch to a 64-bit version only if further investigation proves that there is indeed some value in moving to 64-bit mode.

10 |
11 |
-------------------------------------------------------------------------------- /test/test-pages/social-buttons/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

5 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

6 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

7 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

8 |
9 |
-------------------------------------------------------------------------------- /test/test-pages/liberation-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

Un troisième Français a été tué dans le tremblement de terre samedi au Népal, emporté par une avalanche, a déclaré jeudi le ministre des Affaires étrangères. Les autorités françaises sont toujours sans nouvelles «d’encore plus de 200» personnes. «Pour certains d’entre eux on est très interrogatif», a ajouté Laurent Fabius. Il accueillait à Roissy un premier avion spécial ramenant des rescapés. L’Airbus A350 affrété par les autorités françaises s’est posé peu avant 5h45 avec à son bord 206 passagers, dont 12 enfants et 26 blessés, selon une source du Quai d’Orsay. Quasiment tous sont français, à l’exception d’une quinzaine de ressortissants allemands, suisses, italiens, portugais ou encore turcs. Des psychologues, une équipe médicale et des personnels du centre de crise du Quai d’Orsay les attendent.

5 |

L’appareil, mis à disposition par Airbus, était arrivé à Katmandou mercredi matin avec 55 personnels de santé et humanitaires, ainsi que 25 tonnes de matériel (abris, médicaments, aide alimentaire). Un deuxième avion dépêché par Paris, qui était immobilisé aux Emirats depuis mardi avec 20 tonnes de matériel, est arrivé jeudi à Katmandou, dont le petit aéroport est engorgé par le trafic et l’afflux d’aide humanitaire. Il devait lui aussi ramener des Français, «les plus éprouvés» par la catastrophe et les «plus vulnérables (blessés, familles avec enfants)», selon le ministère des Affaires étrangères.

6 |

2 209 Français ont été localisés sains et saufs tandis que 393 n’ont pas encore pu être joints, selon le Quai d’Orsay. Environ 400 Français ont demandé à être rapatriés dans les vols mis en place par la France.

7 |

Le séisme a fait près de 5 500 morts et touche huit des 28 millions d’habitants du Népal. Des dizaines de milliers de personnes sont sans abri.

8 |

9 | 10 |

11 |
12 |
13 |
-------------------------------------------------------------------------------- /test/test-pages/missing-paragraphs/expected.html: -------------------------------------------------------------------------------- 1 |
2 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

3 |

Secondary header

4 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

5 |

Secondary header

6 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.

7 |
-------------------------------------------------------------------------------- /test/test-pages/heise/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

1Password scannt auch QR-Codes.

6 |

(Bild: Hersteller)

7 |
8 |
9 |

Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.

10 |

AgileBits hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben. Mit dem Update wird eine praktische Funktion nachgereicht, die die iOS-Version der Anwendung bereits seit längerem beherrscht: Das direkte Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der TOTP-Standard (Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.

11 |

Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann – etwa aus dem Webbrowser. Eine Einführung in die Technik gibt ein kurzes Video. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert – was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor mehr gibt.

12 |

Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt worden sein.

13 |

1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen Version mindestens OS X 10.10 voraus. (bsc) 14 |

15 |
16 |
-------------------------------------------------------------------------------- /test/test-pages/svg-parsing/expected.html: -------------------------------------------------------------------------------- 1 |
2 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

3 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

15 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

16 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

17 |
-------------------------------------------------------------------------------- /test/test-pages/svg-parsing/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | SVG parsing 5 | 6 | 7 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 8 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 9 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 10 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 11 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 12 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

13 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 14 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 15 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 16 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 17 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 18 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

19 | 20 | 21 | 22 | 23 | 24 | 25 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 26 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 27 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 28 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 29 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 30 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

31 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 32 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 33 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 34 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 35 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 36 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

37 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 38 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 39 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 40 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 41 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 42 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

43 | 44 | 45 | -------------------------------------------------------------------------------- /test/test-pages/webmd-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 |

Feb. 23, 2015 -- Life-threatening peanut allergies have mysteriously been on the rise in the past decade, with little hope for a cure.

5 |

But a groundbreaking new study may offer a way to stem that rise, while another may offer some hope for those who are already allergic.

6 |

Parents have been told for years to avoid giving foods containing peanuts to babies for fear of triggering an allergy. Now research shows the opposite is true: Feeding babies snacks made with peanuts before their first birthday appears to prevent that from happening.

7 |

The study is published in the New England Journal of Medicine, and it was presented at the annual meeting of the American Academy of Allergy, Asthma and Immunology in Houston. It found that among children at high risk for getting peanut allergies, eating peanut snacks by 11 months of age and continuing to eat them at least three times a week until age 5 cut their chances of becoming allergic by more than 80% compared to kids who avoided peanuts. Those at high risk were already allergic to egg, they had the skin condition eczema, or both.

8 |

Overall, about 3% of kids who ate peanut butter or peanut snacks before their first birthday got an allergy, compared to about 17% of kids who didn’t eat them.

9 |

“I think this study is an astounding and groundbreaking study, really,” says Katie Allen, MD, PhD. She's the director of the Center for Food and Allergy Research at the Murdoch Children’s Research Institute in Melbourne, Australia. Allen was not involved in the research.

10 |

Experts say the research should shift thinking about how kids develop food allergies, and it should change the guidance doctors give to parents.

11 |

Meanwhile, for children and adults who are already allergic to peanuts, another study presented at the same meeting held out hope of a treatment.

12 |

A new skin patch called Viaskin allowed people with peanut allergies to eat tiny amounts of peanuts after they wore it for a year.

13 | 14 |

A Change in Guidelines?

15 |

Allergies to peanuts and other foods are on the rise. In the U.S., more than 2% of people react to peanuts, a 400% increase since 1997. And reactions to peanuts and other tree nuts can be especially severe. Nuts are the main reason people get a life-threatening problem called anaphylaxis.

16 |
17 |
-------------------------------------------------------------------------------- /test/test-pages/daringfireball-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Daring Fireball is written and produced by John Gruber.

4 |

5 | Photograph of the author. 6 |
Portrait by George Del Barrio

7 |

Mac Apps

8 | 19 |

iPhone Apps

20 | 23 |

Server Software

24 |

The Daring Fireball website is hosted by Joyent.

25 |

Articles and links are published through Movable Type. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s MT-Regex and MT-IfEmpty, and Nat Irons’s Amputator.

26 |

Stats are tracked using Mint. Additional web nerdery, including the membership system, is fueled by Perl, PHP, and MySQL.

27 |

Web Standards

28 |

Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either HTML 5 or XHTML 4.01 Transitional, its layout is constructed using valid CSS, and its syndicated feed is valid Atom.

29 |

If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.

30 |
31 |
-------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Readability.js 2 | 3 | [![Build Status](https://travis-ci.org/mozilla/readability.svg?branch=master)](https://travis-ci.org/mozilla/readability) 4 | 5 | A standalone version of the readability library used for Firefox Reader View. Any changes to Readability.js itself should be reviewed by an appropriate Firefox/toolkit peer, such as [@leibovic](https://github.com/leibovic) or [@thebnich](https://github.com/thebnich), since these changes will be automatically merged to mozilla-central. 6 | 7 | ## Contributing 8 | 9 | For outstanding issues, see the issue list in this repo, as well as this bug list: https://bugzilla.mozilla.org/show_bug.cgi?id=1102450 10 | 11 | To test local changes to Readability.js, you can run your own instance of [readable-proxy](https://github.com/n1k0/readable-proxy/) to compare an original test page to its reader-ized content. 12 | 13 | ## Usage 14 | 15 | To parse a document, you must create a new `Readability` object from a URI object and a document, and then call `parse()`. Here's an example: 16 | 17 | ```javascript 18 | var loc = document.location; 19 | var uri = { 20 | spec: loc.href, 21 | host: loc.host, 22 | prePath: loc.protocol + "//" + loc.host, 23 | scheme: loc.protocol.substr(0, loc.protocol.indexOf(":")), 24 | pathBase: loc.protocol + "//" + loc.host + loc.pathname.substr(0, loc.pathname.lastIndexOf("/") + 1) 25 | }; 26 | var article = new Readability(uri, document).parse(); 27 | ``` 28 | 29 | This `article` object will contain the following properties: 30 | 31 | * `uri`: original `uri` object that was passed to constructor 32 | * `title`: article title 33 | * `content`: HTML string of processed article content 34 | * `length`: length of article, in characters 35 | * `excerpt`: article description, or short excerpt from content 36 | * `byline`: author metadata 37 | * `dir`: content direction 38 | 39 | ### Optional 40 | 41 | Readability's `parse()` works by modifying the DOM. This removes some elements in the web page. You could avoid this by passing the clone of the `document` object while creating a `Readability` object. 42 | 43 | 44 | ``` 45 | var documentClone = document.cloneNode(true); 46 | var article = new Readability(uri, documentClone).parse(); 47 | ``` 48 | 49 | ## Tests 50 | 51 | To run the test suite: 52 | 53 | $ mocha test/test-*.js 54 | 55 | To run a specific test page by its name: 56 | 57 | $ mocha test/test-*.js -g 001 58 | 59 | To run the test suite in TDD mode: 60 | 61 | $ mocha test/test-*.js -w 62 | 63 | Combo time: 64 | 65 | $ mocha test/test-*.js -w -g 001 66 | 67 | ## Benchmarks 68 | 69 | Benchmarks for all test pages: 70 | 71 | $ npm run perf 72 | 73 | Reference benchmark: 74 | 75 | $ npm run perf-reference 76 | 77 | ## License 78 | 79 | Copyright (c) 2010 Arc90 Inc 80 | 81 | Licensed under the Apache License, Version 2.0 (the "License"); 82 | you may not use this file except in compliance with the License. 83 | You may obtain a copy of the License at 84 | 85 | http://www.apache.org/licenses/LICENSE-2.0 86 | 87 | Unless required by applicable law or agreed to in writing, software 88 | distributed under the License is distributed on an "AS IS" BASIS, 89 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 90 | See the License for the specific language governing permissions and 91 | limitations under the License. 92 | -------------------------------------------------------------------------------- /test/test-pages/social-buttons/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Share buttons removal test 5 | 6 | 7 |
8 |

Lorem ipsum dolor

9 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 10 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 11 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 12 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 13 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 14 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

15 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 16 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 17 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 18 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 19 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 20 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

21 |
24 |

Like this:

25 | 31 | 32 | 33 |
34 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 35 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 36 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 37 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 38 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 39 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

40 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 41 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 42 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 43 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 44 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 45 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

46 |

Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 47 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 48 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 49 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 50 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 51 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

52 |
53 | 54 | 55 | -------------------------------------------------------------------------------- /test/test-pages/webmd-2/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 |

April 17, 2015 -- Imagine being sick in the hospital with a bacterial infection and doctors can't stop it from spreading. This so-called "superbug" scenario is not science fiction. It's an urgent, worldwide worry that is prompting swift action.

5 |

Every year, about 2 million people get sick from a superbug, according to the CDC. About 23,000 die. Earlier this year, an outbreak of CRE (carbapenem-resistant enterobacteriaceae) linked to contaminated medical tools sickened 11 people at two Los-Angeles area hospitals. Two people died, and more than 200 others may have been exposed.

6 |

The White House recently released a comprehensive plan outlining steps to combat drug-resistant bacteria. The plan identifies three "urgent" and several "serious" threats. We asked infectious disease experts to explain what some of them are and when to worry.

7 | 8 | 9 |

But First: What's a Superbug?

10 |

It's a term coined by the media to describe bacteria that cannot be killed using multiple antibiotics. "It resonates because it's scary," says Stephen Calderwood, MD, president of the Infectious Diseases Society of America. "But in fairness, there is no real definition."

11 |

Instead, doctors often use phrases like "multidrug-resistant bacteria." That's because a superbug isn't necessarily resistant to all antibiotics. It refers to bacteria that can't be treated using two or more, says Brian K. Coombes, PhD, of McMaster University in Ontario.

12 |

Any species of bacteria can turn into a superbug.

13 |

Misusing antibiotics (such as taking them when you don't need them or not finishing all of your medicine) is the "single leading factor" contributing to this problem, the CDC says. The concern is that eventually doctors will run out of antibiotics to treat them.

14 |

"What the public should know is that the more antibiotics you’ve taken, the higher your superbug risk," says Eric Biondi, MD, who runs a program to decrease unnecessary antibiotic use. "The more encounters you have with the hospital setting, the higher your superbug risk."

15 |

"Superbugs should be a concern to everyone," Coombes says. "Antibiotics are the foundation on which all modern medicine rests. Cancer chemotherapy, organ transplants, surgeries, and childbirth all rely on antibiotics to prevent infections. If you can't treat those, then we lose the medical advances we have made in the last 50 years."

16 |

Here are some of the growing superbug threats identified in the 2015 White House report.

17 |
18 |
-------------------------------------------------------------------------------- /test/test-pages/missing-paragraphs/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy 7 | eirmod tempor invidunt 8 | 10 | 11 | 12 | 13 | 14 | 15 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt

16 | 17 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy 18 | eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam 19 | voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet 20 | clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit 21 | amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam 22 | nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 23 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 24 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 25 | sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed 26 | diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 27 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 28 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 29 | sit amet.

30 | 31 |

Secondary header

32 | 33 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy 34 | eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam 35 | voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet 36 | clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit 37 | amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam 38 | nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 39 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 40 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 41 | sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed 42 | diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 43 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 44 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 45 | sit amet.

46 | 47 |

Secondary header

48 | 49 |

Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy 50 | eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam 51 | voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet 52 | clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit 53 | amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam 54 | nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 55 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 56 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 57 | sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed 58 | diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, 59 | sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. 60 | Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor 61 | sit amet.

62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /test/test-pages/buzzfeed-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

The mother of a woman who took suspected diet pills bought online has described how her daughter was “literally burning up from within” moments before her death.

5 |

West Merica Police

6 |
7 |
8 |

Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a lethal dose of highly toxic “slimming tablets”.

9 |

“The drug was in her system, there was no anti-dote, two tablets was a lethal dose – and she had taken eight,” her mother, Fiona, said in a statement yesterday.

10 |

“As Eloise deteriorated, the staff in A&E did all they could to stabilise her. As the drug kicked in and started to make her metabolism soar, they attempted to cool her down, but they were fighting an uphill battle.

11 |

“She was literally burning up from within.”

12 |

She added: “They never stood a chance of saving her. She burned and crashed.”

13 |
14 |
15 |
16 |
17 |
18 |

Facebook

19 |
20 |
21 |
22 |

Facebook

23 |
24 |
25 |
26 |
27 |

West Mercia police said the tablets were believed to contain dinitrophenol, known as DNP, which is a highly toxic industrial chemical.

28 |

“We are undoubtedly concerned over the origin and sale of these pills and are working with partner agencies to establish where they were bought from and how they were advertised,” said chief inspector Jennifer Mattinson from the West Mercia police.

29 |

The Food Standards Agency warned people to stay away from slimming products that contained DNP.

30 |

“We advise the public not to take any tablets or powders containing DNP, as it is an industrial chemical and not fit for human consumption,” it said in a statement.

31 |
32 |
33 |

Fiona Parry issued a plea for people to stay away from pills containing the chemical.

34 |

“[Eloise] just never really understood how dangerous the tablets that she took were,” she said. “Most of us don’t believe that a slimming tablet could possibly kill us.

35 |

“DNP is not a miracle slimming pill. It is a deadly toxin.”

36 |
37 |
38 |
-------------------------------------------------------------------------------- /test/generate-testcase.js: -------------------------------------------------------------------------------- 1 | var debug = false; 2 | 3 | var path = require("path"); 4 | var fs = require("fs"); 5 | var jsdom = require("jsdom").jsdom; 6 | var prettyPrint = require("./utils").prettyPrint; 7 | var serializeDocument = require("jsdom").serializeDocument; 8 | var http = require("http"); 9 | var urlparse = require("url").parse; 10 | 11 | var readability = require("../index"); 12 | var Readability = readability.Readability; 13 | var JSDOMParser = readability.JSDOMParser; 14 | 15 | var FFX_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:38.0) Gecko/20100101 Firefox/38.0"; 16 | 17 | if (process.argv.length < 3) { 18 | console.error("Need at least a destination slug and potentially a URL (if the slug doesn't have source)."); 19 | process.exit(0); 20 | throw "Abort"; 21 | } 22 | 23 | var slug = process.argv[2]; 24 | var argURL = process.argv[3]; // Could be undefined, we'll warn if it is if that is an issue. 25 | 26 | var destRoot = path.join(__dirname, "test-pages", slug); 27 | 28 | fs.mkdir(destRoot, function(err) { 29 | if (err) { 30 | var sourceFile = path.join(destRoot, "source.html"); 31 | fs.exists(sourceFile, function(exists) { 32 | if (exists) { 33 | fs.readFile(sourceFile, {encoding: "utf-8"}, function(readFileErr, data) { 34 | if (readFileErr) { 35 | console.error("Source existed but couldn't be read?"); 36 | process.exit(1); 37 | return; 38 | } 39 | onResponseReceived(data); 40 | }); 41 | } else { 42 | fetchSource(argURL, onResponseReceived); 43 | } 44 | }); 45 | return; 46 | } 47 | fetchSource(argURL, onResponseReceived); 48 | }); 49 | 50 | function fetchSource(url, callbackFn) { 51 | if (!url) { 52 | console.error("You should pass a URL if the source doesn't exist yet!"); 53 | process.exit(1); 54 | return; 55 | } 56 | var client = http; 57 | if (url.indexOf("https") == 0) { 58 | client = require("https"); 59 | } 60 | var options = urlparse(url); 61 | options.headers = {'User-Agent': FFX_UA}; 62 | 63 | client.get(options, function(response) { 64 | if (debug) { 65 | console.log("STATUS:", response.statusCode); 66 | console.log("HEADERS:", JSON.stringify(response.headers)); 67 | } 68 | response.setEncoding("utf-8"); 69 | var rv = ""; 70 | response.on("data", function(chunk) { 71 | rv += chunk; 72 | }); 73 | response.on("end", function() { 74 | if (debug) { 75 | console.log("End received"); 76 | } 77 | // Sanitize: 78 | rv = prettyPrint(serializeDocument(jsdom(rv))); 79 | callbackFn(rv); 80 | }); 81 | }); 82 | } 83 | 84 | function onResponseReceived(source) { 85 | if (debug) { 86 | console.log("writing"); 87 | } 88 | var sourcePath = path.join(destRoot, "source.html"); 89 | fs.writeFile(sourcePath, source, function(err) { 90 | if (err) { 91 | console.error("Couldn't write data to source.html!"); 92 | console.error(err); 93 | return; 94 | } 95 | if (debug) { 96 | console.log("Running readability stuff"); 97 | } 98 | runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json")); 99 | }); 100 | } 101 | 102 | function runReadability(source, destPath, metadataDestPath) { 103 | var doc = new JSDOMParser().parse(source); 104 | var uri = { 105 | spec: "http://fakehost/test/page.html", 106 | host: "fakehost", 107 | prePath: "http://fakehost", 108 | scheme: "http", 109 | pathBase: "http://fakehost/test/" 110 | }; 111 | var myReader, result, readerable; 112 | try { 113 | myReader = new Readability(uri, doc); 114 | readerable = myReader.isProbablyReaderable(); 115 | result = myReader.parse(); 116 | } catch (ex) { 117 | console.error(ex); 118 | ex.stack.forEach(console.log.bind(console)); 119 | } 120 | if (!result) { 121 | console.error("No content generated by readability, not going to write expected.html!"); 122 | return; 123 | } 124 | 125 | fs.writeFile(destPath, prettyPrint(result.content), function(fileWriteErr) { 126 | if (fileWriteErr) { 127 | console.error("Couldn't write data to expected.html!"); 128 | console.error(fileWriteErr); 129 | } 130 | 131 | // Delete the result data we don't care about checking. 132 | delete result.uri; 133 | delete result.content; 134 | delete result.length; 135 | 136 | // Add isProbablyReaderable result 137 | result.readerable = readerable; 138 | 139 | fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function(metadataWriteErr) { 140 | if (metadataWriteErr) { 141 | console.error("Couldn't write data to expected-metadata.json!"); 142 | console.error(metadataWriteErr); 143 | } 144 | 145 | process.exit(0); 146 | }); 147 | }); 148 | } 149 | 150 | -------------------------------------------------------------------------------- /test/test-pages/mozilla-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

It’s easier than ever to personalize Firefox and make it work the way you do. 6 |
No other browser gives you so much choice and flexibility.

7 |
8 |
9 |
10 |
11 |
12 |
13 |

Designed to
be redesigned

14 |

Get fast and easy access to the features you use most in the new menu. Open the “Customize” panel to add, move or remove any button you want. Keep your favorite features — add-ons, private browsing, Sync and more — one quick click away.

15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |

Themes

24 |

Make Firefox match your style. Choose from thousands of themes and dress up your browser with a single click.

25 |

26 | 27 | 28 | 29 | 30 |

Try it now 31 |
Learn more
32 |
Preview of the currently selected theme
33 |
34 |
35 |
36 |
37 |
38 |

Add-ons

39 |

Add-ons are like apps that you install to add features to Firefox. They let you compare prices, check the weather, listen to music, send a tweet and more.

40 |
    41 |
  • Read the latest news & blogs
    • 42 |
  • Manage your downloads
    • 43 |
  • Watch videos & view photos
    • 44 |
Here are a few of our favorites 45 |
Learn more
46 |
47 |
48 |
49 |
50 |
51 |
52 |

Awesome Bar

53 |

The Awesome Bar learns as you browse to make your version of Firefox unique. Find and return to your favorite sites without having to remember a URL.

See what it can do for you
54 |
Firefox Awesome Bar
55 |
56 |
57 |
58 |
59 |
60 |
-------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | // Braces only needed for multi-line arrow function blocks 4 | // "arrow-body-style": [2, "as-needed"], 5 | 6 | // Require spacing around => 7 | // "arrow-spacing": 2, 8 | 9 | // Always require spacing around a single line block 10 | // "block-spacing": 1, 11 | 12 | // No newline before open brace for a block 13 | "brace-style": 2, 14 | 15 | // No space before always a space after a comma 16 | "comma-spacing": [2, {"before": false, "after": true}], 17 | 18 | // Commas at the end of the line not the start 19 | // "comma-style": 2, 20 | 21 | // Don't require spaces around computed properties 22 | // "computed-property-spacing": [2, "never"], 23 | 24 | // Functions must always return something or nothing 25 | "consistent-return": 2, 26 | 27 | // Require braces around blocks that start a new line 28 | // Note that this rule is likely to be overridden on a per-directory basis 29 | // very frequently. 30 | // "curly": [2, "multi-line"], 31 | 32 | // Always require a trailing EOL 33 | "eol-last": 2, 34 | 35 | // Require function* name() 36 | // "generator-star-spacing": [2, {"before": false, "after": true}], 37 | 38 | // Two space indent 39 | "indent": [2, 2, { "SwitchCase": 1 }], 40 | 41 | // Space after colon not before in property declarations 42 | "key-spacing": [2, { "beforeColon": false, "afterColon": true, "mode": "minimum" }], 43 | 44 | // Unix linebreaks 45 | "linebreak-style": [2, "unix"], 46 | 47 | // Always require parenthesis for new calls 48 | "new-parens": 2, 49 | 50 | // Use [] instead of Array() 51 | // "no-array-constructor": 2, 52 | 53 | // No duplicate arguments in function declarations 54 | "no-dupe-args": 2, 55 | 56 | // No duplicate keys in object declarations 57 | "no-dupe-keys": 2, 58 | 59 | // No duplicate cases in switch statements 60 | "no-duplicate-case": 2, 61 | 62 | // No labels 63 | "no-labels": 2, 64 | 65 | // If an if block ends with a return no need for an else block 66 | "no-else-return": 2, 67 | 68 | // No empty statements 69 | "no-empty": 2, 70 | 71 | // No empty character classes in regex 72 | "no-empty-character-class": 2, 73 | 74 | // Disallow empty destructuring 75 | "no-empty-pattern": 2, 76 | 77 | // No assiging to exception variable 78 | // "no-ex-assign": 2, 79 | 80 | // No using !! where casting to boolean is already happening 81 | // "no-extra-boolean-cast": 2, 82 | 83 | // No double semicolon 84 | "no-extra-semi": 2, 85 | 86 | // No overwriting defined functions 87 | "no-func-assign": 2, 88 | 89 | // Declarations in Program or Function Body 90 | "no-inner-declarations": 2, 91 | 92 | // No invalid regular expresions 93 | "no-invalid-regexp": 2, 94 | 95 | // No odd whitespace characters 96 | "no-irregular-whitespace": 2, 97 | 98 | // No single if block inside an else block 99 | "no-lonely-if": 2, 100 | 101 | // No mixing spaces and tabs in indent 102 | "no-mixed-spaces-and-tabs": [2, "smart-tabs"], 103 | 104 | // No unnecessary spacing 105 | "no-multi-spaces": [2, { exceptions: { "AssignmentExpression": true, "VariableDeclarator": true, "ArrayExpression": true, "ObjectExpression": true } }], 106 | 107 | // No reassigning native JS objects 108 | "no-native-reassign": 2, 109 | 110 | // No (!foo in bar) 111 | "no-negated-in-lhs": 2, 112 | 113 | // Nested ternary statements are confusing 114 | "no-nested-ternary": 2, 115 | 116 | // Use {} instead of new Object() 117 | // "no-new-object": 2, 118 | 119 | // No Math() or JSON() 120 | "no-obj-calls": 2, 121 | 122 | // No octal literals 123 | "no-octal": 2, 124 | 125 | // No redeclaring variables 126 | "no-redeclare": 2, 127 | 128 | // No unnecessary comparisons 129 | "no-self-compare": 2, 130 | 131 | // No declaring variables from an outer scope 132 | "no-shadow": 2, 133 | 134 | // No declaring variables that hide things like arguments 135 | "no-shadow-restricted-names": 2, 136 | 137 | // No spaces between function name and parentheses 138 | "no-spaced-func": 2, 139 | 140 | // No trailing whitespace 141 | "no-trailing-spaces": 2, 142 | 143 | // No using undeclared variables 144 | // "no-undef": 2, 145 | 146 | // Error on newline where a semicolon is needed 147 | "no-unexpected-multiline": 2, 148 | 149 | // No unreachable statements 150 | "no-unreachable": 2, 151 | 152 | // No expressions where a statement is expected 153 | // "no-unused-expressions": 2, 154 | 155 | // No declaring variables that are never used 156 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 157 | 158 | // No using variables before defined 159 | // "no-use-before-define": [2, "nofunc"], 160 | 161 | // No using with 162 | "no-with": 2, 163 | 164 | // Always require semicolon at end of statement 165 | "semi": [2, "always"], 166 | 167 | // Require space after keywords 168 | "keyword-spacing": 2, 169 | 170 | // Require space before blocks 171 | "space-before-blocks": 2, 172 | 173 | // Never use spaces before function parentheses 174 | // "space-before-function-paren": [2, { "anonymous": "always", "named": "never" }], 175 | 176 | // Require spaces before finally, catch, etc. 177 | // "space-before-keywords": [2, "always"], 178 | 179 | // No space padding in parentheses 180 | // "space-in-parens": [2, "never"], 181 | 182 | // Require spaces around operators 183 | // "space-infix-ops": 2, 184 | 185 | // Require spaces after return, throw and case 186 | // "space-return-throw-case": 2, 187 | 188 | // ++ and -- should not need spacing 189 | // "space-unary-ops": [2, { "words": true, "nonwords": false }], 190 | 191 | // No comparisons to NaN 192 | "use-isnan": 2, 193 | 194 | // Only check typeof against valid results 195 | "valid-typeof": 2, 196 | }, 197 | } 198 | -------------------------------------------------------------------------------- /test/test-pages/001/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

So finally you're testing your frontend JavaScript code? Great! The more you 4 | write tests, the more confident you are with your code… but how much precisely? 5 | That's where code coverage might 6 | help.

7 |

The idea behind code coverage is to record which parts of your code (functions, statements, conditionals and so on) have been executed by your test suite, to compute metrics out of these data and usually to provide tools for navigating and inspecting them.

8 |

Not a lot of frontend developers I know actually test their frontend code, and I can barely imagine how many of them have ever setup code coverage… Mostly because there are not many frontend-oriented tools in this area I guess.

9 |

Actually I've only found one which provides an adapter for Mocha and actually works…

10 | 15 |

Blanket.js is an easy to install, easy to configure, 16 | and easy to use JavaScript code coverage library that works both in-browser and 17 | with nodejs.

18 |

Its use is dead easy, adding Blanket support to your Mocha test suite is just matter of adding this simple line to your HTML test file:

<script src="vendor/blanket.js"
19 |         data-cover-adapter="vendor/mocha-blanket.js"></script>
20 |
21 |

Source files: blanket.js, mocha-blanket.js

22 |

As an example, let's reuse the silly Cow example we used in a previous episode:

// cow.js
23 | (function(exports) {
24 |   "use strict";
25 | 
26 |   function Cow(name) {
27 |     this.name = name || "Anon cow";
28 |   }
29 |   exports.Cow = Cow;
30 | 
31 |   Cow.prototype = {
32 |     greets: function(target) {
33 |       if (!target)
34 |         throw new Error("missing target");
35 |       return this.name + " greets " + target;
36 |     }
37 |   };
38 | })(this);
39 |
40 |

And its test suite, powered by Mocha and Chai:

var expect = chai.expect;
41 | 
42 | describe("Cow", function() {
43 |   describe("constructor", function() {
44 |     it("should have a default name", function() {
45 |       var cow = new Cow();
46 |       expect(cow.name).to.equal("Anon cow");
47 |     });
48 | 
49 |     it("should set cow's name if provided", function() {
50 |       var cow = new Cow("Kate");
51 |       expect(cow.name).to.equal("Kate");
52 |     });
53 |   });
54 | 
55 |   describe("#greets", function() {
56 |     it("should greet passed target", function() {
57 |       var greetings = (new Cow("Kate")).greets("Baby");
58 |       expect(greetings).to.equal("Kate greets Baby");
59 |     });
60 |   });
61 | });
62 |
63 |

Let's create the HTML test file for it, featuring Blanket and its adapter for Mocha:

<!DOCTYPE html>
64 | <html>
65 | <head>
66 |   <meta charset="utf-8">
67 |   <title>Test</title>
68 |   <link rel="stylesheet" media="all" href="vendor/mocha.css">
69 | </head>
70 | <body>
71 |   <div id="mocha"></div>
72 |   <div id="messages"></div>
73 |   <div id="fixtures"></div>
74 |   <script src="vendor/mocha.js"></script>
75 |   <script src="vendor/chai.js"></script>
76 |   <script src="vendor/blanket.js"
77 |           data-cover-adapter="vendor/mocha-blanket.js"></script>
78 |   <script>mocha.setup('bdd');</script>
79 |   <script src="cow.js" data-cover></script>
80 |   <script src="cow_test.js"></script>
81 |   <script>mocha.run();</script>
82 | </body>
83 | </html>
84 |
85 |

Notes:

86 | 90 |

Running the tests now gives us something like this:

91 |

screenshot

92 |

As you can see, the report at the bottom highlights that we haven't actually tested the case where an error is raised in case a target name is missing. We've been informed of that, nothing more, nothing less. We simply know we're missing a test here. Isn't this cool? I think so!

93 |

Just remember that code coverage will only bring you numbers and raw information, not actual proofs that the whole of your code logic has been actually covered. If you ask me, the best inputs you can get about your code logic and implementation ever are the ones issued out of pair programming sessions and code reviews — but that's another story.

94 |

So is code coverage silver bullet? No. Is it useful? Definitely. Happy testing!

95 |
96 |
97 | -------------------------------------------------------------------------------- /test/test-pages/ars-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
6 |

A flaw in the wildly popular online game Minecraft makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability.

7 |

"I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a blog post published Thursday, 21 months, he said, after privately reporting the bug to Minecraft developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it."

8 |

The bug resides in the networking internals of the Minecraft protocol. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. Minecraft items can also store arbitrary metadata in a file format known as Named Binary Tag (NBT), which allows complex data structures to be kept in hierarchical nests. Askar has released proof-of-concept attack code he said exploits the vulnerability to crash any server hosting the game. Here's how it works.

9 |
10 |

The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to craft a packet that is incredibly complex for the server to deserialize but trivial for us to generate.

11 |

In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like.

12 | 
rekt: {
13 |     list: [
14 |         list: [
15 |             list: [
16 |                 list: [
17 |                     list: [
18 |                         list: [
19 |                         ]
20 |                         list: [
21 |                         ]
22 |                         list: [
23 |                         ]
24 |                         list: [
25 |                         ]
26 |                         ...
27 |                     ]
28 |                     ...
29 |                 ]
30 |                 ...
31 |             ]
32 |             ...
33 |         ]
34 |         ...
35 |     ]
36 |     ...
37 | }
38 |

The root of the object, rekt, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of 10^5 * 300 = 30,000,000 lists.

39 |

And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes.

40 |

Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to 2^15 - 1. Now that the length is a varint capable of storing integers up to 2^28, our potential for attack has increased significantly.

41 |

When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load.

42 |

This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the 0x08: Block Placement Packet and 0x10: Creative Inventory Action.

43 |

The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented.

44 |

These were the fixes that I recommended to Mojang 2 years ago.

45 |
46 |

Ars is asking Mojang for comment and will update this post if company officials respond.

47 |
48 |
49 | -------------------------------------------------------------------------------- /test/test-pages/herald-sun-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
A new Bill would require telecommunications service providers to store so-called ‘metadat
6 |

A new Bill would require telecommunications service providers to store so-called ‘metadata’ for two years. Source: 7 | Supplied

8 |
9 |
10 |

11 | A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed. 12 |

13 |

The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer, Justin Bassi, who advises Attorney-General George Brandis on crime and security matters, and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of Communications Minister Malcolm Turnbull also took part.

14 |

They held meetings with executives from News Corporation and Fairfax, representatives of the TV networks, the ABC top brass and a group from the media union and the Walkley journalism foundation. I was involved as a member of the Walkley board.

15 |

The initiative, from Tony Abbott’s office, is evidence that the Government has been alarmed by the strength of criticism from media of the Data Retention Bill it wants passed before Parliament rises in a fortnight. Bosses, journalists, even the Press Council, are up in arms, not only over this measure, but also over aspects of two earlier pieces of national security legislation that interfere with the ability of the media to hold government to account.

16 |
17 |
18 |

The Bill would require telecommunications service providers to store so-called “metadata” — the who, where, when and how of a communication, but not its content — for two years so security and law enforcement agencies can access it without warrant. Few would argue against the use of such material to catch criminals or terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has pointed out, it would also be used “for the purpose of determining the identity of a journalist’s sources”.

19 |

And that should ring warning bells for anyone genuinely concerned with the health of our democracy. Without the ability to protect the identity of sources, journalists would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence and misbehaviour by public officials.

20 |

The Press Council is concerned the laws would crush investigative journalism.

21 |

“These legitimate concerns cannot be addressed effectively short of exempting journalists and media organisations,” says president David Weisbrot.

22 |

The media union is adamant journalists’ metadata must be exempted from the law. That’s what media bosses want, too, though they have a fallback position based on new safeguards being implemented in Britain.

23 |

That would prevent access to the metadata of journalists or media organisations without a judicial warrant. There would be a code including — according to the explanatory notes of the British Bill — “provision to protect the public interest in the confidentiality of journalistic sources”.

24 |

In their meetings this week, the government team boasted of concessions in the new Data Retention Bill. The number of agencies able to access metadata will be reduced by excluding such organisations as the RSPCA and local councils. And whenever an authorisation is issued for access to information about a journalist’s sources, the Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and Security) will receive a copy.

25 |

That does nothing to solve the problem. The Government has effectively admitted as much by agreeing that the parliamentary committee should conduct a separate review of how to deal with the issue of journalists’ sources.

26 |

But another inquiry would be a waste of time — the committee has already received and considered dozens of submissions on the subject. The bottom line is that the Government does not deny that the legislation is flawed, but is demanding it be passed anyway with the possibility left open of a repair job down the track. That is a ridiculous approach.

27 |

Claims that immediate action is imperative do not stand up. These are measures that won’t come into full effect for two years. Anyway, amending the Bill to either exempt journalists or adopt the UK model could be done quickly, without any risk to national security.

28 |

AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press freedom concerns about mandatory data retention would ideally be addressed in this Bill to avoid the need for future additional amendments or procedures to be put in place in the future.”

29 |

The Data Retention Bill will be debated in the House of Representatives this week. Then, on Friday, CEOs from leading media organisations will front the parliamentary committee to air their concerns before the legislation goes to the Senate.

30 |

Those CEOs should make it clear they are just as angry about this as they were about Stephen Conroy’s attempt to impinge on press freedom through media regulation under the previous Labor government.

31 |

Memories of the grief Conroy brought down on his head would undoubtedly make Abbott sit up and take notice.

32 |

LAURIE OAKES IS THE NINE NETWORK POLITICAL EDITOR

33 |
34 |
35 |
36 |
-------------------------------------------------------------------------------- /test/test-pages/mozilla-2/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

Get to know the features that make it the most complete browser for building the Web.

6 |
7 |
    8 |
  • 9 | Screenshot 10 |

    WebIDE

    11 |

    Develop, deploy and debug Firefox OS apps directly in your browser, or on a Firefox OS device, with this tool that replaces App Manager.

    Learn more about WebIDE
    • 12 |
  • 13 | Screenshot 14 |

    Valence

    15 |

    Develop and debug your apps across multiple browsers and devices with this powerful extension that comes pre-installed with Firefox Developer Edition.

    Learn more about Valence
    • 16 |
17 |
18 |

Important: Sync your new profile

19 |

Developer Edition comes with a new profile so you can run it alongside other versions of Firefox. To access your bookmarks, browsing history and more, you need to sync the profile with your existing Firefox Account, or create a new one. Learn more

20 |
21 |
22 |
23 |
24 |
25 |

Features and tools

26 |
    27 |
  • 28 | Screenshot 29 |

    Page Inspector

    30 |

    Examine the HTML and CSS of any Web page and easily modify the structure and layout of a page.

    Learn more about Page Inspector
    • 31 |
  • 32 | Screenshot 33 |

    Web Console

    34 |

    See logged information associated with a Web page and use Web Console to interact with Web pages using JavaScript.

    Learn more about Web Console
    • 35 |
  • 36 | Screenshot 37 |

    JavaScript Debugger

    38 |

    Step through JavaScript code and examine or modify its state to help track down bugs.

    Learn more about JavaScript Debugger
    • 39 |
  • 40 | Screenshot 41 |

    Network Monitor

    42 |

    See all the network requests your browser makes, how long each request takes and details of each request.

    Learn more about Network Monitor
    • 43 |
  • 44 | Screenshot 45 |

    Web Audio Editor

    46 |

    Inspect and interact with Web Audio API in real time to ensure that all audio nodes are connected in the way you expect.

    Learn more about Web Audio Editor
    • 47 |
  • 48 | Screenshot 49 |

    Style Editor

    50 |

    View and edit CSS styles associated with a Web page, create new ones and apply existing CSS stylesheets to any page.

    Learn more about Style Editor
    • 51 |
52 |
53 |
54 |
55 |
-------------------------------------------------------------------------------- /test/test-pages/iab-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience.

4 |

Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. Many of us in the technical field felt compelled, and even empowered, to produce information as the distribution means for mass communication were no longer restricted by a high barrier to entry.

5 |

In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups were gone or that our divisions sustained by corporate parent companies needed to be in the black. It was a wakeup call that led to a renaissance age. Digital advertising became the foundation of an economic engine that, still now, sustains the free and democratic World Wide Web. In digital publishing, we strived to balance content, commerce, and technology. The content management systems and communication gateways we built to inform and entertain populations around the world disrupted markets and in some cases governments, informed communities of imminent danger, and liberated new forms of art and entertainment—all while creating a digital middle class of small businesses.

6 |

We engineered not just the technical, but also the social and economic foundation that users around the world came to lean on for access to real time information. And users came to expect this information whenever and wherever they needed it. And more often than not, for anybody with a connected device, it was free.

7 |

This was choice—powered by digital advertising—and premised on user experience.

8 |

But we messed up.

9 |

Through our pursuit of further automation and maximization of margins during the industrial age of media technology, we built advertising technology to optimize publishers’ yield of marketing budgets that had eroded after the last recession. Looking back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast, scalable systems of targeting users with ever-heftier advertisements have slowed down the public internet and drained more than a few batteries. We were so clever and so good at it that we over-engineered the capabilities of the plumbing laid down by, well, ourselves. This steamrolled the users, depleted their devices, and tried their patience.

10 |

The rise of ad blocking poses a threat to the internet and could potentially drive users to an enclosed platform world dominated by a few companies. We have let the fine equilibrium of content, commerce, and technology get out of balance in the open web. We had, and still do have, a responsibility to educate the business side, and in some cases to push back. We lost sight of our social and ethical responsibility to provide a safe, usable experience for anyone and everyone wanting to consume the content of their choice.

11 |

We need to bring that back into alignment, starting right now.

12 |

13 | Getting LEAN with Digital Ad UXToday, the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads. These are principles that will help guide the next phases of advertising technical standards for the global digital advertising supply chain.

14 |

As with any other industry, standards should be created by non-profit standards-setting bodies, with many diverse voices providing input. We will invite all parties for public comment, and make sure consumer interest groups have the opportunity to provide input.

15 |

L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy and engage with while consuming content on our sites across all IP enabled devices. Rather, these principles will guide an alternative set of standards that provide choice for marketers, content providers, and consumers.

16 |

Among the many areas of concentration, we must also address frequency capping on retargeting in Ad Tech and make sure a user is targeted appropriately before, but never AFTER they make a purchase. If we are so good at reach and scale, we can be just as good, if not better, at moderation. Additionally, we must address volume of ads per page as well as continue on the path to viewability. The dependencies here are critical to an optimized user experience.

17 |

The consumer is demanding these actions, challenging us to do better, and we must respond.

18 |

The IAB Tech Lab will continue to provide the tools for publishers in the digital supply chain to have a dialogue with users about their choices so that content providers can generate revenue while creating value. Publishers should have the opportunity to provide rich advertising experiences, L.E.A.N. advertising experiences, and subscription services. Or publishers can simply deny their service to users who choose to keep on blocking ads. That is all part of elasticity of consumer tolerance and choice.

19 |

Finally, we must do this in an increasingly fragmented market, across screens. We must do this in environments where entire sites are blocked, purposefully or not. Yes, it is disappointing that our development efforts will have to manage with multiple frameworks while we work to supply the economic engine to sustain an open internet. However, our goal is still to provide diverse content and voices to as many connected users as possible around the world.

20 |

That is user experience.

21 | 22 | 23 | 24 | 25 | 26 | 27 |
IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email adblocking@iab.com for more information.
28 |

Read more about ad blocking here.

29 |
30 |
-------------------------------------------------------------------------------- /test/test-pages/medium-2/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
Words need defenders.
6 |
7 |

On Behalf of “Literally”

8 |

You either are a “literally” abuser or know of one. If you’re anything like me, hearing the word “literally” used incorrectly causes a little piece of your soul to whither and die. Of course I do not mean that literally, I mean that figuratively. An abuser would have said: “Every time a person uses that word, a piece of my soul literally withers and dies.” Which is terribly, horribly wrong.

9 |

For whatever bizarre reason, people feel the need to use literally as a sort of verbal crutch. They use it to emphasize a point, which is silly because they’re already using an analogy or a metaphor to illustrate said point. For example: “Ugh, I literally tore the house apart looking for my remote control!” No, you literally did not tear apart your house, because it’s still standing. If you’d just told me you “tore your house apart” searching for your remote, I would’ve understood what you meant. No need to add “literally” to the sentence.

10 |

Maybe I should define literally.

11 |
Literally means actually. When you say something literally happened, you’re describing the scene or situation as it actually happened.
12 |

So you should only use literally when you mean it. It should not be used in hyperbole. Example: “That was so funny I literally cried.” Which is possible. Some things are funny enough to elicit tears. Note the example stops with “literally cried.” You cannot literally cry your eyes out. The joke wasn’t so funny your eyes popped out of their sockets.

13 |

When in Doubt, Leave it Out

14 |

“I’m so hungry I could eat a horse,” means you’re hungry. You don’t need to say “I’m so hungry I could literally eat a horse.” Because you can’t do that in one sitting, I don’t care how big your stomach is.

15 |

“That play was so funny I laughed my head off,” illustrates the play was amusing. You don’t need to say you literally laughed your head off, because then your head would be on the ground and you wouldn’t be able to speak, much less laugh.

16 |

“I drove so fast my car was flying,” we get your point: you were speeding. But your car is never going fast enough to fly, so don’t say your car was literally flying.

17 |

Insecurities?

18 |

Maybe no one believed a story you told as a child, and you felt the need to prove that it actually happened. No really, mom, I literally climbed the tree. In efforts to prove truth, you used literally to describe something real, however outlandish it seemed. Whatever the reason, now your overuse of literally has become a habit.

19 |

Hard Habit to Break?

20 |

Abusing literally isn’t as bad a smoking, but it’s still an unhealthy habit (I mean that figuratively). Help is required in order to break it.

21 |

This is my version of an intervention for literally abusers. I’m not sure how else to do it other than in writing. I know this makes me sound like a know-it-all, and I accept that. But there’s no excuse other than blatant ignorance to misuse the word “literally.” So just stop it.

22 |

Don’t say “Courtney, this post is so snobbish it literally burned up my computer.” Because nothing is that snobbish that it causes computers to combust. Or: “Courtney, your head is so big it literally cannot get through the door.” Because it can, unless it’s one of those tiny doors from Alice in Wonderland and I need to eat a mushroom to make my whole body smaller.

23 |

No One’s Perfect

24 |

And I’m not saying I am. I’m trying to restore meaning to a word that’s lost meaning. I’m standing up for literally. It’s a good word when used correctly. People are butchering it and destroying it every day (figuratively speaking) and the massacre needs to stop. Just as there’s a coalition of people against the use of certain fonts (like Comic Sans and Papyrus), so should there be a coalition of people against the abuse of literally.

25 |

Saying it to Irritate?

26 |

Do you misuse the word “literally” just to annoy your know-it-all or grammar police friends/acquaintances/total strangers? If so, why? Doing so would be like me going outside when it’s freezing, wearing nothing but a pair of shorts and t-shirt in hopes of making you cold by just looking at me. Who suffers more?

27 |

Graphical Representation

28 |

Matthew Inman of “The Oatmeal” wrote a comic about literally. Abusers and defenders alike should check it out. It’s clear this whole craze about literally is driving a lot of us nuts. You literally abusers are killing off pieces of our souls. You must be stopped, or the world will be lost to meaninglessness forever. Figuratively speaking.

29 |
30 |
-------------------------------------------------------------------------------- /test/test-pages/wapo-2/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

President Obama told the U.N. General Assembly 18 months ago that he would seek “real breakthroughs on these two issues — Iran’s nuclear program and ­Israeli-Palestinian peace.”

4 |

But Benjamin Netanyahu’s triumph in Tuesday’s parliamentary elections keeps in place an Israeli prime minister who has declared his intention to resist Obama on both of these fronts, guaranteeing two more years of difficult diplomacy between leaders who barely conceal their personal distaste for each other.

5 |

The Israeli election results also suggest that most voters there support Netanyahu’s tough stance on U.S.-led negotiations to limit Iran’s nuclear program and his vow on Monday that there would be no independent Palestinian state as long as he is prime minister.

6 |

“On the way to his election victory, Netanyahu broke a lot of crockery in the relationship,” said Martin Indyk, executive vice president of the Brookings Institution and a former U.S. ambassador to Israel. “It can’t be repaired unless both sides have an interest and desire to do so.”

7 |

Aside from Russian President Vladi­mir Putin, few foreign leaders so brazenly stand up to Obama and even fewer among longtime allies.

8 |
9 |

Israeli Prime Minister Benjamin Netanyahu pledged to form a new governing coalition quickly after an upset election victory that was built on a shift to the right. (Reuters)

10 |
11 |

In the past, Israeli leaders who risked damaging the country’s most important relationship, that with Washington, tended to pay a price. In 1991, when Prime Minister Yitzhak Shamir opposed the Madrid peace talks, President George H.W. Bush held back loan guarantees to help absorb immigrants from the former Soviet Union. Shamir gave in, but his government soon collapsed.

12 |

But this time, Netanyahu was not hurt by his personal and substantive conflicts with the U.S. president.

13 |

“While the United States is loved and beloved in Israel, President Obama is not,” said Robert M. Danin, a senior fellow at the Council on Foreign Relations. “So the perceived enmity didn’t hurt the way it did with Shamir when he ran afoul of Bush in ’91.”

14 |

Where do U.S.-Israeli relations go from here?

15 |

In the immediate aftermath of Tuesday’s elections, tensions between the two sides continued to run hot. The Obama administration’s first comments on the Israeli election came with a tough warning about some of the pre-election rhetoric from Netanyahu’s Likud party, which tried to rally right-wing support by saying that Arab Israeli voters were “coming out in droves.”

16 |

“The United States and this administration is deeply concerned about rhetoric that seeks to marginalize Arab Israeli citizens,” White House press secretary Josh Earnest told reporters aboard Air Force One. “It undermines the values and democratic ideals that have been important to our democracy and an important part of what binds the United States and Israel together.”

17 |

Earnest added that Netan­yahu’s election-eve disavowal of a two-state solution for Israelis and Palestinians would force the administration to reconsider its approach to peace in the region.

18 |

Over the longer term, a number of analysts say that Obama and Netan­yahu will seek to play down the friction between them and point to areas of continuing cooperation on military and economic issues.

19 |

“Both sides are going to want to turn down the rhetoric,” Danin said. “But it is also a structural problem. They have six years of accumulated history. That’s going to put limits on how far they can go together.”

20 |

The first substantive test could come as early as this month, when the United States hopes that it can finish hammering out the framework of an agreement with Iran.

21 |

Netanyahu strongly warned against making a “bad deal” during his March 3 address to a joint meeting of Congress, an appearance arranged by Republican congressional leaders and criticized by the Obama administration for making U.S.-Israeli relations partisan on both sides so close to the Israeli election.

22 |

If a deal is reached and does not pass muster with Netanyahu, he is likely to work with congressional Republicans to try to scuttle the accord.

23 |

“The Republicans have said they will do what they can to block a deal, and the prime minister has already made clear that he will work with the Republicans against the president,” Indyk said. “That’s where a clash could come, and it’s coming very quickly.”

24 |

The second test — talks with Palestinians — could be even more difficult. In his September 2013 address to the United Nations, Obama hailed signs of hope.

25 |

“Already, Israeli and Palestinian leaders have demonstrated a willingness to take significant political risks,” Obama said in his speech. Palestinian Authority President Mahmoud Abbas “has put aside efforts to shortcut the pursuit of peace and come to the negotiating table. Prime Minister Netanyahu has released Palestinian prisoners and reaffirmed his commitment to a Palestinian state.”

26 |

Today, the signals could not differ more. The Palestinian Authority has said that after it joins the International Criminal Court at The Hague on April 1, it will press war crimes charges against Israel for the bloody Gaza conflict during the summer. Israel, which controls tax receipts, has pledged to punish the Palestinian Authority by freezing its tax revenue.

27 |

The United States, which gives hundreds of millions of dollars of economic aid to the Palestinian Authority, would be caught in the middle. It has been trying to persuade both sides to stand down, but Netanyahu’s declaration that there would be no Palestinian state on his watch makes that more difficult.

28 |

“Now it’s hard to see what could persuade the Palestinians” to hold up on their ICC plans, Indyk said. “That has nothing to do with negotiations, but if both sides can’t be persuaded to back down, then they will be on a trajectory that could lead to the collapse of the Palestinian Authority because it can’t pay wages anymore.

29 |

“That could be an issue forced onto the agenda about the same time as a potential nuclear deal.”

30 |
31 |
32 | 33 |

Steven Mufson covers the White House. Since joining The Post, he has covered economics, China, foreign policy and energy.

34 |
35 |
-------------------------------------------------------------------------------- /test/test-readability.js: -------------------------------------------------------------------------------- 1 | var jsdom = require("jsdom").jsdom; 2 | var chai = require("chai"); 3 | chai.config.includeStack = true; 4 | var expect = chai.expect; 5 | 6 | var readability = require("../index"); 7 | var Readability = readability.Readability; 8 | var JSDOMParser = readability.JSDOMParser; 9 | 10 | var testPages = require("./utils").getTestPages(); 11 | 12 | function reformatError(err) { 13 | var formattedError = new Error(err.message); 14 | formattedError.stack = err.stack; 15 | return formattedError; 16 | } 17 | 18 | function inOrderTraverse(fromNode) { 19 | if (fromNode.firstChild) { 20 | return fromNode.firstChild; 21 | } 22 | while (fromNode && !fromNode.nextSibling) { 23 | fromNode = fromNode.parentNode; 24 | } 25 | return fromNode ? fromNode.nextSibling : null; 26 | } 27 | 28 | function inOrderIgnoreEmptyTextNodes(fromNode) { 29 | do { 30 | fromNode = inOrderTraverse(fromNode); 31 | } while (fromNode && fromNode.nodeType == 3 && !fromNode.textContent.trim()); 32 | return fromNode; 33 | } 34 | 35 | function traverseDOM(callback, expectedDOM, actualDOM) { 36 | var actualNode = actualDOM.documentElement || actualDOM.childNodes[0]; 37 | var expectedNode = expectedDOM.documentElement || expectedDOM.childNodes[0]; 38 | while (actualNode) { 39 | if (!callback(actualNode, expectedNode)) { 40 | break; 41 | } 42 | actualNode = inOrderIgnoreEmptyTextNodes(actualNode); 43 | expectedNode = inOrderIgnoreEmptyTextNodes(expectedNode); 44 | } 45 | } 46 | 47 | // Collapse subsequent whitespace like HTML: 48 | function htmlTransform(str) { 49 | return str.replace(/\s+/g, " "); 50 | } 51 | 52 | function runTestsWithItems(label, domGenerationFn, uri, source, expectedContent, expectedMetadata) { 53 | describe(label, function() { 54 | this.timeout(5000); 55 | 56 | var result; 57 | 58 | before(function() { 59 | try { 60 | var doc = domGenerationFn(source); 61 | var myReader = new Readability(uri, doc); 62 | var readerable = myReader.isProbablyReaderable(); 63 | result = myReader.parse(); 64 | result.readerable = readerable; 65 | } catch (err) { 66 | throw reformatError(err); 67 | } 68 | }); 69 | 70 | it("should return a result object", function() { 71 | expect(result).to.include.keys("content", "title", "excerpt", "byline"); 72 | }); 73 | 74 | it("should extract expected content", function() { 75 | function nodeStr(n) { 76 | if (n.nodeType == 3) { 77 | return "#text(" + htmlTransform(n.textContent) + ")"; 78 | } 79 | if (n.nodeType != 1) { 80 | return "some other node type: " + n.nodeType + " with data " + n.data; 81 | } 82 | var rv = n.localName; 83 | if (n.id) { 84 | rv += "#" + n.id; 85 | } 86 | if (n.className) { 87 | rv += ".(" + n.className + ")"; 88 | } 89 | return rv; 90 | } 91 | var actualDOM = domGenerationFn(result.content); 92 | var expectedDOM = domGenerationFn(expectedContent); 93 | traverseDOM(function(actualNode, expectedNode) { 94 | expect(!!actualNode).eql(!!expectedNode); 95 | if (actualNode && expectedNode) { 96 | var actualDesc = nodeStr(actualNode); 97 | var expectedDesc = nodeStr(expectedNode); 98 | if (actualDesc != expectedDesc) { 99 | expect(actualDesc).eql(expectedDesc); 100 | return false; 101 | } 102 | // Compare text for text nodes: 103 | if (actualNode.nodeType == 3) { 104 | var actualText = htmlTransform(actualNode.textContent); 105 | var expectedText = htmlTransform(expectedNode.textContent); 106 | expect(actualText).eql(expectedText); 107 | if (actualText != expectedText) { 108 | return false; 109 | } 110 | // Compare attributes for element nodes: 111 | } else if (actualNode.nodeType == 1) { 112 | expect(actualNode.attributes.length).eql(expectedNode.attributes.length); 113 | for (var i = 0; i < actualNode.attributes.length; i++) { 114 | var attr = actualNode.attributes[i].name; 115 | var actualValue = actualNode.getAttribute(attr); 116 | var expectedValue = expectedNode.getAttribute(attr); 117 | expect(expectedValue, "node '" + actualDesc + "' attribute " + attr + " should match").eql(actualValue); 118 | } 119 | } 120 | } else { 121 | return false; 122 | } 123 | return true; 124 | }, actualDOM, expectedDOM); 125 | }); 126 | 127 | it("should extract expected title", function() { 128 | expect(expectedMetadata.title).eql(result.title); 129 | }); 130 | 131 | it("should extract expected byline", function() { 132 | expect(expectedMetadata.byline).eql(result.byline); 133 | }); 134 | 135 | it("should extract expected excerpt", function() { 136 | expect(expectedMetadata.excerpt).eql(result.excerpt); 137 | }); 138 | 139 | it("should probably be readerable", function() { 140 | expect(expectedMetadata.readerable).eql(result.readerable); 141 | }); 142 | }); 143 | } 144 | 145 | function removeCommentNodesRecursively(node) { 146 | for (var i = node.childNodes.length - 1; i >= 0; i--) { 147 | var child = node.childNodes[i]; 148 | if (child.nodeType === child.COMMENT_NODE) { 149 | node.removeChild(child); 150 | } else if (child.nodeType === child.ELEMENT_NODE) { 151 | removeCommentNodesRecursively(child); 152 | } 153 | } 154 | } 155 | 156 | describe("Readability API", function() { 157 | describe("#constructor", function() { 158 | it("should accept a debug option", function() { 159 | expect(new Readability({}, {})._debug).eql(false); 160 | expect(new Readability({}, {}, {debug: true})._debug).eql(true); 161 | }); 162 | 163 | it("should accept a nbTopCandidates option", function() { 164 | expect(new Readability({}, {})._nbTopCandidates).eql(5); 165 | expect(new Readability({}, {}, {nbTopCandidates: 42})._nbTopCandidates).eql(42); 166 | }); 167 | 168 | it("should accept a maxPages option", function() { 169 | expect(new Readability({}, {})._maxPages).eql(5); 170 | expect(new Readability({}, {}, {maxPages: 42})._maxPages).eql(42); 171 | }); 172 | 173 | it("should accept a maxElemsToParse option", function() { 174 | expect(new Readability({}, {})._maxElemsToParse).eql(0); 175 | expect(new Readability({}, {}, {maxElemsToParse: 42})._maxElemsToParse).eql(42); 176 | }); 177 | }); 178 | 179 | describe("#parse", function() { 180 | it("shouldn't parse oversized documents as per configuration", function() { 181 | var doc = new JSDOMParser().parse("
yo
"); 182 | expect(function() { 183 | new Readability({}, doc, {maxElemsToParse: 1}).parse(); 184 | }).to.Throw("Aborting parsing document; 2 elements found"); 185 | }); 186 | }); 187 | }); 188 | 189 | describe("Test pages", function() { 190 | testPages.forEach(function(testPage) { 191 | describe(testPage.dir, function() { 192 | var uri = { 193 | spec: "http://fakehost/test/page.html", 194 | host: "fakehost", 195 | prePath: "http://fakehost", 196 | scheme: "http", 197 | pathBase: "http://fakehost/test/" 198 | }; 199 | 200 | runTestsWithItems("jsdom", function(source) { 201 | var doc = jsdom(source, { 202 | features: { 203 | FetchExternalResources: false, 204 | ProcessExternalResources: false 205 | } 206 | }); 207 | removeCommentNodesRecursively(doc); 208 | return doc; 209 | }, uri, testPage.source, testPage.expectedContent, testPage.expectedMetadata); 210 | 211 | runTestsWithItems("JSDOMParser", function(source) { 212 | var parser = new JSDOMParser(); 213 | var doc = parser.parse(source); 214 | if (parser.errorState) { 215 | console.error("Parsing this DOM caused errors:", parser.errorState); 216 | return null; 217 | } 218 | return doc; 219 | }, uri, testPage.source, testPage.expectedContent, testPage.expectedMetadata); 220 | }); 221 | }); 222 | }); 223 | -------------------------------------------------------------------------------- /test/test-pages/bbc-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

President Barack Obama has admitted that his failure to pass "common sense gun safety laws" in the US is the greatest frustration of his presidency.

4 |

In an interview with the BBC, Mr Obama said it was "distressing" not to have made progress on the issue "even in the face of repeated mass killings".

5 |

He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president did not sound very confident.

6 |

However, Mr Obama said race relations had improved during his presidency.

7 |

Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana, killing two people and injuring several others before shooting himself.

8 |

In a wide-ranging interview, President Obama also said:

9 | 17 |

Read the full transcript of his interview

18 |

Mr Obama lands in Kenya later on Friday for his first visit since becoming president.

19 |

But with just 18 months left in power, he said gun control was the area where he has been "most frustrated and most stymied" since coming to power in 2009.

20 |

"If you look at the number of Americans killed since 9/11 by terrorism, it's less than 100. If you look at the number that have been killed by gun violence, it's in the tens of thousands," Mr Obama said.

21 |
Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013 22 |
23 | The president said he would continue fighting for greater gun control laws 24 |
25 |
26 |

"For us not to be able to resolve that issue has been something that is distressing," he added.

27 |

Mr Obama has pushed for stricter gun control throughout his presidency but has been unable to secure any significant changes to the laws.

28 |

After nine African-American churchgoers were killed in South Carolina in June, he admitted "politics in this town" meant there were few options available.

29 |
line
30 |

Analysis: Jon Sopel, BBC News, Washington

31 |
President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015
32 |

Nine months ago, the president seemed like a spent force, after taking a beating in the midterm elections, during which members of his own party were reluctant to campaign on his record.

33 |

But the man sat before me today was relaxed and confident, buoyed by a string of "wins" on healthcare, Cuba and Iran, after bitter and ongoing battles with his many critics.

34 |

The only body swerve the president performed was when I asked him how many minds he had changed on the Iran nuclear deal after an intense sell aimed at Gulf allies and members of US Congress who remain implacably opposed.

35 |

There was a momentary flicker across the president's face as if to say "You think you got me?" before his smile returned and he proceeded to talk about how Congress would come round.

36 |

But notably, he did not give a direct answer to that question, which leaves me with the impression that he has persuaded precisely zero.

37 |

Five things we learned from Obama interview

38 |

The presidential body swerve

39 |
line
40 |

On race relations, Mr Obama said recent concerns around policing and mass incarcerations were "legitimate and deserve intense attention" but insisted progress had been made.

41 |

Children growing up during the eight years of his presidency "will have a different view of race relations in this country and what's possible," he said.

42 |

"There are going to be tensions that arise. But if you look at my daughters' generation, they have an attitude about race that's entirely different than even my generation."

43 |

Talking about how he was feeling after his recent successes, he said "every president, every leader has strengths and weaknesses".

44 |

"One of my strengths is I have a pretty even temperament. I don't get too high when it's high and I don't get too low when it's low," he said.

45 |
Customer looks at Obama shirts at a stall in Nairobi's Kibera slums, 23 July 2015 46 |
47 | Kenya is getting ready to welcome the US president 48 |
49 |
50 |

Kenya trip

51 |

Mr Obama was speaking to the BBC at the White House before departing for Kenya.

52 |

His father was Kenyan and the president is expected to meet relatives in Nairobi.

53 |

Mr Obama has faced criticism in the country after the US legalised gay marriage. However, in his interview, the president said he would not fall silent on the issue.

54 |

"I am not a fan of discrimination and bullying of anybody on the basis of race, on the basis of religion, on the basis of sexual orientation or gender," he said.

55 |

The president also admitted that some African governments, including Kenya's, needed to improve their records on human rights and democracy. However, he defended his decision to engage with and visit those governments.

56 |

"Well, they're not ideal institutions. But what we found is, is that when we combined blunt talk with engagement, that gives us the best opportunity to influence and open up space for civil society."

57 |

Mr Obama will become the first US president to address the African Union when he travels on to Ethiopia on Sunday.

58 |
59 |
60 | -------------------------------------------------------------------------------- /test/test-pages/daringfireball-1/source.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Daring Fireball: Colophon 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 |
20 | 23 | 62 | 63 |
64 |
65 |

About This Site

66 |

Daring Fireball is written and produced by John Gruber.

67 |

68 | Photograph of the author. 69 |
Portrait by George Del Barrio

70 |

Mac Apps

71 | 82 |

iPhone Apps

83 | 86 |

Server Software

87 |

The Daring Fireball website is hosted by Joyent.

88 |

Articles and links are published through Movable Type. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s MT-Regex and MT-IfEmpty, and Nat Irons’s Amputator.

89 |

Stats are tracked using Mint. Additional web nerdery, including the membership system, is fueled by Perl, PHP, and MySQL.

90 |

Web Standards

91 |

Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either HTML 5 or XHTML 4.01 Transitional, its layout is constructed using valid CSS, and its syndicated feed is valid Atom.

92 |

If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.

93 |
94 | 95 | 105 |
106 | 116 |

117 | Ads via The Deck Ads via The Deck 118 |

119 |
120 | 121 | 134 | 135 | 145 |
146 | 147 |
148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /test/test-pages/ehow-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
6 |

Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.

7 |
Glass cloche terrariums
8 |
Glass cloche terrariums (Lucy Akins)
9 |
10 |
11 |
12 |
13 |
What You'll Need: 14 |
    15 |
  • Cloche
    • 16 |
  • Planter saucer, small shallow dish or desired platform
    • 17 |
  • Floral foam oasis
    • 18 |
  • Ruler
    • 19 |
  • Spoon
    • 20 |
  • Floral wire pins or paper clips
    • 21 |
  • Small plants (from a florist or nursery)
    • 22 |
  • Moss
    • 23 |
  • Tweezers
    • 24 |
  • Other small decorative items (optional)
    • 25 |
26 |
27 |
28 |
29 |
30 |
31 |
Step 1 32 |

Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.

33 |
34 |
Dig a hole in the oasis.
35 |
Dig a hole in the oasis. (Lucy Akins)
36 |
37 |
38 |
39 |
40 |
Step 2 41 |

Insert your plant into the hole.

42 |
43 |
Orchid in foam oasis
44 |
Orchid in foam oasis (Lucy Akins)
45 |
46 |
47 |
48 |
49 |
Step 3 50 |

You can add various plants if you wish.

51 |
52 |
Various foliage
53 |
Various foliage (Lucy Akins)
54 |
55 |
56 |
57 |
58 |
Step 4 59 |

Using floral pins, attach enough moss around the oasis to cover it.

60 |
61 |
Attach moss.
62 |
Attach moss. (Lucy Akins)
63 |
64 |
65 |
66 |
67 |
Step 5 68 |

Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.

69 |
70 |
Place cloche over oasis.
71 |
Place cloche over oasis. (Lucy Akins)
72 |
73 |
74 |
75 |
76 |
Step 6 77 |

Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.

78 |
79 |
Rearrange moss.
80 |
Rearrange moss. (Lucy Akins)
81 |
82 |
83 |
84 |
85 |
Step 7 86 |

You can use any platform you wish. In this case, a small saucer was used.

87 |
88 |
Place cloche on a platform to sit on.
89 |
Place cloche on a platform to sit on. (Lucy Akins)
90 |
91 |
92 |
93 |
94 |
Step 8 95 |

This particular terrarium rests on a planter saucer and features a small white pumpkin.

96 |
97 |
Cloche placed on a terracotta saucer
98 |
Cloche placed on a terracotta saucer (Lucy Akins)
99 |
100 |
101 |
102 |
103 |
Step 9 104 |

This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.

105 |
106 |
Placed on a wooden slice
107 |
Placed on a wooden slice (Lucy Akins)
108 |
109 |
110 |
111 |
112 |
Finished Terrarium 113 |

Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.

114 |
115 |
Cloche terrarium
116 |
Cloche terrarium (Lucy Akins)
117 |
118 |
119 | 139 |
140 |
141 |
142 |
-------------------------------------------------------------------------------- /test/test-pages/wapo-1/expected.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

CAIRO — Gunmen opened fire on visitors at Tunisia’s most renowned museum on Wednesday, killing at least 19 people, including 17 foreigners, in an assault that threatened to upset the fragile stability of a country seen as the lone success of the Arab Spring.

4 |

It was the most deadly terrorist attack in the North African nation in more than a decade. Although no group claimed responsibility, the bloodshed raised fears that militants linked to the Islamic State were expanding their operations.

5 |

The attackers, clad in military uniforms, stormed the Bardo National Museum on Wednesday afternoon, seizing and gunning down foreign tourists before security forces raided the building to end the siege. The museum is a major tourist draw and is near the heavily guarded national parliament in downtown Tunis.

6 |

Tunisian Prime Minister Habib Essid said that in addition to the slain foreigners — from Italy, Poland, Germany and Spain — a local museum worker and a security official were killed. Two gunmen died, and three others may have escaped, officials said. About 50 other people were wounded, according to local news reports.

7 |

“Our nation is in danger,” Essid declared in a televised address Wednesday evening. He vowed that the country would be “merciless” in defending itself.

8 |

[Read: Why Tunisia, Arab Spring’s sole success story, suffers from Islamist violence]

9 |

Tunisia, a mostly Muslim nation of about 11 million people, was governed for decades by autocrats who imposed secularism. Its sun-drenched Mediterranean beaches drew thousands of bikini-clad tourists, and its governments promoted education and other rights for women. But the country has grappled with rising Islamist militancy since a popular uprising overthrew its dictator four years ago, setting the stage for the Arab Spring revolts across the region.

10 |

Thousands of Tunisians have flocked to join jihadist groups in Syria, including the Islamic State, making the country one of the major sources of foreign fighters in the conflict. Tunisian security forces have also fought increasing gunbattles with jihadists at home.

11 |

Despite this, the country has been hailed as a model of democratic transition as other governments that came to power after the Arab Spring collapsed, often in bloody confrontations. But the attack Wednesday — on a national landmark that showcases Tunisia’s rich heritage — could heighten tensions in a nation that has become deeply divided between pro- and anti-Islamist political factions.

12 |

Many Tunisians accuse the country’s political Islamists, who held power from 2011 to 2013, of having been slow to respond to the growing danger of terrorism. Islamist politicians have acknowledged that they did not realize the threat that would develop when radical Muslims, who had been repressed under authoritarian regimes, won the freedom to preach freely in mosques.

13 |

In Washington, White House press secretary Josh Earnest condemned the attack and said the U.S. government was willing to assist Tunisian authorities in the investigation.

14 |
15 |

Gunmen in military uniforms stormed Tunisia's national museum, killing at least 19 people, most of them foreign tourists. (Reuters)

16 |
17 |

“This attack today is meant to threaten authorities, to frighten tourists and to negatively affect the economy,” said Lotfi Azzouz, Tunisia country director for Amnesty International, a London-based rights group.

18 |

Tourism is critical to Tunisia’s economy, accounting for 15 percent of its gross domestic product in 2013, according to the World Travel and Tourism Council, an industry body. The Bardo museum hosts one of the world’s most outstanding collections of Roman mosaics and is popular with tourists and Tunisians alike.

19 |

[Bardo museum houses amazing Roman treasures]

20 |

The attack is “also aimed at the country’s security and stability during the transition period,” Azzouz said. “And it could have political repercussions — like the curtailing of human rights, or even less government transparency if there’s fear of further attacks.”

21 |

The attack raised concerns that the government, led by secularists, would be pressured to stage a wider crackdown on Islamists of all stripes. Lawmakers are drafting an anti-terrorism bill to give security forces additional tools to fight militants.

22 |

[Read: Tunisia sends most foreign fighters to Islamic State in Syria]

23 |

“We must pay attention to what is written” in that law, Azzouz said. “There is worry the government will use the attack to justify some draconian measures.”

24 |

Tunisian Islamists and secular forces have worked together — often reluctantly — to defuse the country’s political crises in the years since the revolt.

25 |

Last fall, Tunisians elected a secular-minded president and parliament dominated by liberal forces after souring on Islamist-led rule. In 2011, voters had elected a government led by the Ennahda party — a movement similar to Egypt’s Islamist Muslim Brotherhood. But a political stalemate developed as the party and others tried to draft the country’s new constitution. The Islamists failed to improve a slumping economy. And Ennahda came under fire for what many Tunisians saw as a failure to crack down on Islamist extremists.

26 |
Map: Flow of foreign fighters to Syria
27 |

After the collapse of the authoritarian system in 2011, hard-line Muslims known as Salafists attacked bars and art galleries. Then, in 2012, hundreds of Islamists assaulted the U.S. Embassy in Tunis, shattering windows and hurling gasoline bombs, after the release of a crude online video about the prophet Muhammad. 28 | The government outlawed the group behind the attack — Ansar al-Sharia, an al-Qaeda-linked organization — and began a crackdown. But the killing of two leftist politicians in 2013 prompted a fresh political crisis, and Ennahda stepped down, replaced by a technocratic government.

29 |

Tunisia’s current coalition government includes an Ennahda minister in the cabinet. Still, many leftist figures openly oppose collaboration with the movement’s leaders.

30 |

“Ennahda is responsible for the current deterioration of the situation, because they were careless with the extremists” while they were in power, Azzouz said.

31 |

The leader of Ennahda, Rachid Ghannouchi, condemned Wednesday’s attack, saying in a statement that it “will not break our people’s will and will not undermine our revolution and our democracy.”

32 |

Security officials are particularly concerned by the collapse of Libya, where various armed groups are vying for influence and jihadist militants have entrenched themselves in major cities. Tunisians worry that extremists can easily get arms and training in the neighboring country.

33 |

In January, Libyan militants loyal to the Islamic State beheaded 21 Christians — 20 of them Egyptian Copts — along the country’s coast. They later seized the Libyan city of Sirte.

34 |
35 |
36 |
37 |

Officials are worried about the number of Tunisian militants who may have joined the jihadists in Libya — with the goal of returning home to fight the Tunis government.

38 |

Ajmi Lourimi, a member of Ennahda’s general secretariat, said he believed the attack would unite Tunisians in the face of terrorism.

39 |

“There is a consensus here that this [attack] is alien to our culture, to our way of life. We want to unify against this danger,” Lourimi said. He said he did not expect a wider government campaign against Islamists.

40 |

“We have nothing to fear,” he said of himself and fellow Ennahda members. “We believe the Interior Ministry should be trained and equipped to fight and counter this militancy.”

41 |

The last major attack on a civilian target in Tunisia was in 2002, when al-Qaeda militants killed more than 20 people in a car bombing outside a synagogue in the city of Djerba.

42 |

Heba Habib contributed to this report.

43 |

Read more:

44 |

Tunisia’s Islamists get a sobering lesson in governing

45 |

Tunisia sends most foreign fighters to Islamic State in Syria

46 |

Tunisia’s Bardo museum is home to amazing Roman treasures

47 |
48 |
--------------------------------------------------------------------------------