├── .gitignore
├── .travis.yml
├── AUTHORS.txt
├── CHANGELOG.rst
├── LICENSE.rst
├── MANIFEST.in
├── Makefile
├── README.rst
├── breadability.1
├── breadability
    ├── __init__.py
    ├── _compat.py
    ├── annotated_text.py
    ├── document.py
    ├── readable.py
    ├── scoring.py
    ├── scripts
    │   ├── __init__.py
    │   ├── client.py
    │   └── test_helper.py
    └── utils.py
├── requirements.txt
├── setup.cfg
├── setup.py
├── tests
    ├── __init__.py
    ├── compat.py
    ├── data
    │   ├── articles
    │   │   ├── ars.001.html
    │   │   ├── automation_blog.html
    │   │   ├── django-tutorial.001.html
    │   │   ├── mitchie-blog.001.html
    │   │   ├── python.org-wiki.performancetips.html
    │   │   └── zdrojak_automaticke_zabezpeceni.html
    │   └── snippets
    │   │   ├── annotated_1.html
    │   │   ├── document_absolute_url.html
    │   │   ├── document_min.html
    │   │   ├── document_no_body.html
    │   │   ├── document_only_content.html
    │   │   ├── document_scripts.html
    │   │   ├── h1_and_2_paragraphs.html
    │   │   └── test_readable_unlikely.html
    ├── test_annotated_text.py
    ├── test_articles
    │   ├── __init__.py
    │   ├── test_antipope_org
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    │   ├── test_businessinsider-com
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    │   ├── test_businessinsider_com
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    │   ├── test_cz_zdrojak_tests
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    │   ├── test_scripting_com
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    │   └── test_sweetshark
    │   │   ├── __init__.py
    │   │   ├── article.html
    │   │   └── test_article.py
    ├── test_orig_document.py
    ├── test_readable.py
    ├── test_scoring.py
    └── utils.py
└── tox.ini


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.py[co]
 2 | *.prof
 3 | 
 4 | # hide all files starting with dot: .idea, .pytest_cache, .coverage, ...
 5 | .*
 6 | !.gitignore
 7 | !.travis.yml
 8 | 
 9 | .installed.cfg
10 | bin
11 | develop-eggs
12 | 
13 | *.egg-info
14 | 
15 | tmp
16 | build
17 | dist
18 | include/
19 | lib/
20 | local/
21 | man/
22 | share/
23 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | python:
 3 |   # https://github.com/travis-ci/travis-ci/issues/2219#issuecomment-41804942
 4 |   # https://snarky.ca/how-to-use-your-project-travis-to-help-test-python-itself/
 5 |   - "2.7"
 6 |   - "3.3"
 7 |   - "3.4"
 8 |   - "3.5"
 9 |   - "3.5-dev"
10 |   - "3.6"
11 |   - "3.6-dev"
12 |   - "3.7-dev"
13 |   - "nightly"
14 | before_install: sudo apt-get install libxml2-dev libxslt-dev
15 | # command to install dependencies
16 | install:
17 |   - pip install -U pip wheel setuptools
18 |   - python setup.py install
19 |   - make deps
20 | # command to run tests
21 | script: py.test -q --tb native -s tests
22 | 


--------------------------------------------------------------------------------
/AUTHORS.txt:
--------------------------------------------------------------------------------
1 | Rick Harding (original author)
2 | nhnifong
3 | Craig Maloney
4 | Mišo Belica
5 | 


--------------------------------------------------------------------------------
/CHANGELOG.rst:
--------------------------------------------------------------------------------
  1 | .. :changelog:
  2 | 
  3 | Changelog for breadability
  4 | ==========================
  5 | 
  6 | 0.1.20 (April 13th 2014)
  7 | -------------------------
  8 | - Don't include tests in sdist builds.
  9 | 
 10 | 0.1.19 (April 13th 2014)
 11 | --------------------------
 12 | - Replace charade with chardet for easier packaging.
 13 | 
 14 | 0.1.18 (April 6th 2014)
 15 | ------------------------
 16 | - Improved decoding of the page into Unicode.
 17 | 
 18 | 0.1.17 (Jan 22nd 2014)
 19 | ----------------------
 20 | - More log quieting down to INFO vs WARN
 21 | 
 22 | 0.1.16 (Jan 22nd 2014)
 23 | ----------------------
 24 | - Clean up logging output at warning when it's not a true warning
 25 | 
 26 | 0.1.15 (Nov 29th 2013)
 27 | ----------------------
 28 | - Merge changes from 0.1.14 of breadability with the fork https://github.com/miso-belica/readability.py and tweaking to return to the name breadability.
 29 | - Fork: Added property ``Article.main_text`` for getting text annotated with
 30 |   semantic HTML tags (<em>, <strong>, ...).
 31 | - Fork: Join node with 1 child of the same type. From
 32 |   ``<div><div>...</div></div>`` we get ``<div>...</div>``.
 33 | - Fork: Don't change <div> to <p> if it contains <p> elements.
 34 | - Fork: Renamed test generation helper 'readability_newtest' -> 'readability_test'.
 35 | - Fork: Renamed package to readability. (Renamed back)
 36 | - Fork: Added support for Python >= 3.2.
 37 | - Fork: Py3k compatible package 'charade' is used instead of 'chardet'.
 38 | 
 39 | 0.1.14 (Nov 7th 2013)
 40 | ---------------------
 41 | - Update sibling append to only happen when sibling doesn't already exist.
 42 | 
 43 | 0.1.13 (Aug 31st 2013)
 44 | ----------------------
 45 | - Give images in content boy a better chance of survival
 46 | - Add tests
 47 | 
 48 | 0.1.12 (July 28th 2013)
 49 | -----------------------
 50 | - Add a user agent to requests.
 51 | 
 52 | 0.1.11 (Dec 12th 2012)
 53 | ----------------------
 54 | - Add argparse to the install requires for python < 2.7
 55 | 
 56 | 0.1.10 (Sept 13th 2012)
 57 | -----------------------
 58 | - Updated scoring bonus and penalty with , and " characters.
 59 | 
 60 | 0.1.9 (Aug 27nd 2012)
 61 | ---------------------
 62 | - In case of an issue dealing with candidates we need to act like we didn't
 63 |   find any candidates for the article content. #10
 64 | 
 65 | 0.1.8 (Aug 27nd 2012)
 66 | ---------------------
 67 | - Add code/tests for an empty document.
 68 | - Fixes #9 to handle xml parsing issues.
 69 | 
 70 | 0.1.7 (July 21nd 2012)
 71 | ----------------------
 72 | - Change the encode 'replace' kwarg into a normal arg for older python
 73 |   version.
 74 | 
 75 | 0.1.6 (June 17th 2012)
 76 | ----------------------
 77 | - Fix the link removal, add tests and a place to process other bad links.
 78 | 
 79 | 0.1.5 (June 16th 2012)
 80 | ----------------------
 81 | - Start to look at removing bad links from content in the conditional cleaning
 82 |   state. This was really used for the scripting.com site's garbage.
 83 | 
 84 | 0.1.4 (June 16th 2012)
 85 | ----------------------
 86 | - Add a test generation helper readability_newtest script.
 87 | - Add tests and fixes for the scripting news parse failure.
 88 | 
 89 | 0.1.3 (June 15th 2012)
 90 | ----------------------
 91 | - Add actual testing of full articles for regression tests.
 92 | - Update parser to properly clean after winner doc node is chosen.
 93 | 
 94 | 0.1.2 (May 28th 2012)
 95 | ---------------------
 96 | - Bugfix: #4 issue with logic of the 100char bonus points in scoring
 97 | - Garden with PyLint/PEP8
 98 | - Add a bunch of tests to readable/scoring code.
 99 | 
100 | 0.1.1 (May 11th 2012)
101 | ---------------------
102 | - Fix bugs in scoring to help in getting right content
103 | - Add concept of -d which shows scoring/decisions on nodes
104 | - Update command line client to be able to pipe output to other tools
105 | 
106 | 0.1.0 (May 6th 2012)
107 | --------------------
108 | - Initial release and upload to PyPi
109 | 


--------------------------------------------------------------------------------
/LICENSE.rst:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2013 Rick Harding and contributors
 2 | 
 3 | All rights reserved.
 4 | 
 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 6 | 
 7 | - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 8 | - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 9 | 
10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
11 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include README.rst
2 | include CHANGELOG.rst
3 | include LICENSE.rst
4 | include AUTHORS.txt
5 | recursive-exclude * __pycache__
6 | recursive-exclude * *.py[co]
7 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # Makefile to help automate tasks
 2 | WD := $(shell pwd)
 3 | PY := bin/python
 4 | PIP := bin/pip
 5 | PEP8 := bin/pep8
 6 | PYTEST := bin/py.test
 7 | 
 8 | # ###########
 9 | # Tests rule!
10 | # ###########
11 | .PHONY: test
12 | test: venv develop $(PYTEST)
13 | 	$(PYTEST) -q --tb native -s tests
14 | 
15 | .PHONY: test-coverage
16 | test-coverage: venv develop $(PYTEST)
17 | 	$(PYTEST) -q --cov breadability tests
18 | 
19 | $(PYTEST):
20 | 	$(PIP) install -r requirements.txt
21 | 
22 | # #######
23 | # INSTALL
24 | # #######
25 | .PHONY: all
26 | all: venv deps develop
27 | 
28 | venv: bin/python
29 | bin/python:
30 | 	virtualenv .
31 | 
32 | .PHONY: deps
33 | deps: venv
34 | 	$(PIP) install -r requirements.txt
35 | 
36 | .PHONY: clean_venv
37 | clean_venv:
38 | 	rm -rf bin include lib local man share
39 | 
40 | .PHONY: develop
41 | develop: lib/python*/site-packages/breadability.egg-link
42 | lib/python*/site-packages/breadability.egg-link:
43 | 	$(PY) setup.py develop
44 | 
45 | 
46 | # ###########
47 | # Development
48 | # ###########
49 | .PHONY: clean-all
50 | clean-all: clean_venv
51 | 	if [ -d dist ]; then \
52 | 		rm -r dist; \
53 |     fi
54 | 
55 | 
56 | bin/flake8: venv
57 | 	bin/pip install flake8
58 | 
59 | lint: bin/flake8
60 | 	flake8 breadability
61 | 
62 | 
63 | # ###########
64 | # Deploy
65 | # ###########
66 | .PHONY: dist
67 | dist:
68 | 	$(PY) setup.py sdist
69 | 
70 | .PHONY: upload
71 | upload:
72 | 	$(PY) setup.py sdist upload
73 | 
74 | .PHONY: version_update
75 | version_update:
76 | 	$(EDITOR) setup.py CHANGELOG.rst
77 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | breadability - another readability Python (v2.6-v3.3) port
  2 | ==========================================================
  3 | .. image:: https://api.travis-ci.org/bookieio/breadability.png?branch=master
  4 |    :target: https://travis-ci.org/bookieio/breadability
  5 | 
  6 | I've tried to work with the various forks of some ancient codebase that ported
  7 | `readability`_ to Python. The lack of tests, unused regex's, and commented out
  8 | sections of code in other Python ports just drove me nuts.
  9 | 
 10 | I put forth an effort to bring in several of the better forks into one
 11 | code base, but they've diverged so much that I just can't work with it.
 12 | 
 13 | So what's any sane person to do? Re-port it with my own repo, add some tests,
 14 | infrastructure, and try to make this port better. OSS FTW (and yea, NIH FML,
 15 | but oh well I did try)
 16 | 
 17 | This is a pretty straight port of the JS here:
 18 | 
 19 | - http://code.google.com/p/arc90labs-readability/source/browse/trunk/js/readability.js#82
 20 | - http://www.minvolai.com/blog/decruft-arc90s-readability-in-python/
 21 | 
 22 | 
 23 | Alternatives
 24 | ------------
 25 | 
 26 | - https://github.com/codelucas/newspaper
 27 | - https://github.com/grangier/python-goose
 28 | - https://github.com/aidanf/BTE
 29 | - http://www.unixuser.org/~euske/python/webstemmer/#extract
 30 | - https://github.com/al3xandru/readability.py
 31 | - https://github.com/rcarmo/soup-strainer
 32 | - https://github.com/bcampbell/decruft
 33 | - https://github.com/gfxmonk/python-readability
 34 | - https://github.com/srid/readability
 35 | - https://github.com/dcramer/decruft
 36 | - https://github.com/reorx/readability
 37 | - https://github.com/mote/python-readability
 38 | - https://github.com/predatell/python-readability-lxml
 39 | - https://github.com/Harshavardhana/boilerpipy
 40 | - https://github.com/raptium/hitomi
 41 | - https://github.com/kingwkb/readability
 42 | 
 43 | 
 44 | Installation
 45 | ------------
 46 | This does depend on lxml so you'll need some C headers in order to install
 47 | things from pip so that it can compile.
 48 | 
 49 | .. code-block:: bash
 50 | 
 51 |     $ [sudo] apt-get install libxml2-dev libxslt-dev
 52 |     $ [sudo] pip install git+git://github.com/bookieio/breadability.git
 53 | 
 54 | Tests
 55 | -----
 56 | .. code-block:: bash
 57 | 
 58 |     $ pytest tests
 59 | 
 60 | 
 61 | Usage
 62 | -----
 63 | Command line
 64 | ~~~~~~~~~~~~
 65 | 
 66 | .. code-block:: bash
 67 | 
 68 |     $ breadability http://wiki.python.org/moin/BeginnersGuide
 69 | 
 70 | Options
 71 | ```````
 72 | 
 73 | - **b** will write out the parsed content to a temp file and open it in a
 74 |   browser for viewing.
 75 | - **d** will write out debug scoring statements to help track why a node was
 76 |   chosen as the document and why some nodes were removed from the final
 77 |   product.
 78 | - **f** will override the default behaviour of getting an html fragment (<div>)
 79 |   and give you back a full <html> document.
 80 | - **v** will output in verbose debug mode and help let you know why it parsed
 81 |   how it did.
 82 | 
 83 | 
 84 | Python API
 85 | ~~~~~~~~~~
 86 | .. code-block:: python
 87 | 
 88 |     from __future__ import print_function
 89 | 
 90 |     from breadability.readable import Article
 91 | 
 92 | 
 93 |     if __name__ == "__main__":
 94 |         document = Article(html_as_text, url=source_url)
 95 |         print(document.readable)
 96 | 
 97 | 
 98 | Work to be done
 99 | ---------------
100 | Yep, I've got some catching up to do. I don't do pagination, I've got a lot of
101 | custom tweaks I need to get going, there are some articles that fail to parse.
102 | I also have more tests to write on a lot of the cleaning helpers, but
103 | hopefully things are setup in a way that those can/will be added.
104 | 
105 | Fortunately, I need this library for my tools:
106 | 
107 | - https://bmark.us
108 | - http://r.bmark.us
109 | 
110 | so I really need this to be an active and improving project.
111 | 
112 | 
113 | Off the top of my heads TODO list:
114 | 
115 | - Support metadata from parsed article [url, confidence scores, all
116 |   candidates we thought about?]
117 | - More tests, more thorough tests
118 | - More sample articles we need to test against in the test_articles
119 | - Tests that run through and check for regressions of the test_articles
120 | - Tidy'ing the HTML that comes out, might help with regression tests ^^
121 | - Multiple page articles
122 | - Performance tuning, we do a lot of looping and re-drop some nodes that
123 |   should be skipped. We should have a set of regression tests for this so
124 |   that if we implement a change that blows up performance we know it right
125 |   away.
126 | - More docs for things, but sphinx docs and in code comments to help
127 |   understand wtf we're doing and why. That's the biggest hurdle to some of
128 |   this stuff.
129 | 
130 | 
131 | Inspiration
132 | ~~~~~~~~~~~
133 | 
134 | - `python-readability`_
135 | - `decruft`_
136 | - `readability`_
137 | 
138 | 
139 | 
140 | .. _readability: http://code.google.com/p/arc90labs-readability/
141 | .. _TravisCI: http://travis-ci.org/
142 | .. _decruft: https://github.com/dcramer/decruft
143 | .. _python-readability: https://github.com/buriy/python-readability
144 | 


--------------------------------------------------------------------------------
/breadability.1:
--------------------------------------------------------------------------------
 1 | .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.45.1.
 2 | .TH BREADABILITY "1" "April 2014" "breadability 0.1.20" "User Commands"
 3 | .SH NAME
 4 | breadability \- readability formatter
 5 | .SH DESCRIPTION
 6 | A fast python port of arc90's readability tool.
 7 | .SS "Usage:"
 8 | .IP
 9 | breadability [options] <resource>
10 | .SS "Arguments:"
11 | .TP
12 | <resource>
13 | URL or file path to process in readable form.
14 | .SH OPTIONS
15 | .TP
16 | \fB\-f\fR, \fB\-\-fragment\fR
17 | Output html fragment by default.
18 | .TP
19 | \fB\-b\fR, \fB\-\-browser\fR
20 | Open the parsed content in your web browser.
21 | .TP
22 | \fB\-d\fR, \fB\-\-debug\fR
23 | Output the detailed scoring information for debugging
24 | parsing.
25 | .TP
26 | \fB\-v\fR, \fB\-\-verbose\fR
27 | Increase logging verbosity to DEBUG.
28 | .TP
29 | \fB\-\-version\fR
30 | Display program's version number and exit.
31 | .TP
32 | \fB\-h\fR, \fB\-\-help\fR
33 | Display this help message and exit.
34 | .SH "SEE ALSO"
35 | http://github.com/bookie/breadability
36 | .SH "LICENSE"
37 | BSD License (2 clause)
38 | .SH "AUTHOR"
39 | Rick Harding et al.
40 | 


--------------------------------------------------------------------------------
/breadability/__init__.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | from __future__ import (
 4 |     absolute_import,
 5 |     division,
 6 |     print_function,
 7 |     unicode_literals
 8 | )
 9 | import pkg_resources
10 | 
11 | __version__ = pkg_resources.get_distribution("breadability").version
12 | 


--------------------------------------------------------------------------------
/breadability/_compat.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | from __future__ import absolute_import
  4 | from __future__ import division, print_function, unicode_literals
  5 | 
  6 | from sys import version_info
  7 | 
  8 | 
  9 | PY3 = version_info[0] == 3
 10 | 
 11 | 
 12 | if PY3:
 13 |     bytes = bytes
 14 |     unicode = str
 15 | else:
 16 |     bytes = str
 17 |     unicode = unicode
 18 | string_types = (bytes, unicode,)
 19 | 
 20 | 
 21 | try:
 22 |     # Assert to hush pyflakes about the unused import. This is a _compat
 23 |     # module and we expect this to aid in other code importing urllib.
 24 |     import urllib2 as urllib
 25 |     assert urllib
 26 | except ImportError:
 27 |     import urllib.request as urllib
 28 |     assert urllib
 29 | 
 30 | 
 31 | def unicode_compatible(cls):
 32 |     """
 33 |     Decorator for unicode compatible classes. Method ``__unicode__``
 34 |     has to be implemented to work decorator as expected.
 35 |     """
 36 |     if PY3:
 37 |         cls.__str__ = cls.__unicode__
 38 |         cls.__bytes__ = lambda self: self.__str__().encode("utf8")
 39 |     else:
 40 |         cls.__str__ = lambda self: self.__unicode__().encode("utf8")
 41 | 
 42 |     return cls
 43 | 
 44 | 
 45 | def to_string(object):
 46 |     return to_unicode(object) if PY3 else to_bytes(object)
 47 | 
 48 | 
 49 | def to_bytes(object):
 50 |     try:
 51 |         if isinstance(object, bytes):
 52 |             return object
 53 |         elif isinstance(object, unicode):
 54 |             return object.encode("utf8")
 55 |         else:
 56 |             # try encode instance to bytes
 57 |             return instance_to_bytes(object)
 58 |     except UnicodeError:
 59 |         # recover from codec error and use 'repr' function
 60 |         return to_bytes(repr(object))
 61 | 
 62 | 
 63 | def to_unicode(object):
 64 |     try:
 65 |         if isinstance(object, unicode):
 66 |             return object
 67 |         elif isinstance(object, bytes):
 68 |             return object.decode("utf8")
 69 |         else:
 70 |             # try decode instance to unicode
 71 |             return instance_to_unicode(object)
 72 |     except UnicodeError:
 73 |         # recover from codec error and use 'repr' function
 74 |         return to_unicode(repr(object))
 75 | 
 76 | 
 77 | def instance_to_bytes(instance):
 78 |     if PY3:
 79 |         if hasattr(instance, "__bytes__"):
 80 |             return bytes(instance)
 81 |         elif hasattr(instance, "__str__"):
 82 |             return unicode(instance).encode("utf8")
 83 |     else:
 84 |         if hasattr(instance, "__str__"):
 85 |             return bytes(instance)
 86 |         elif hasattr(instance, "__unicode__"):
 87 |             return unicode(instance).encode("utf8")
 88 | 
 89 |     return to_bytes(repr(instance))
 90 | 
 91 | 
 92 | def instance_to_unicode(instance):
 93 |     if PY3:
 94 |         if hasattr(instance, "__str__"):
 95 |             return unicode(instance)
 96 |         elif hasattr(instance, "__bytes__"):
 97 |             return bytes(instance).decode("utf8")
 98 |     else:
 99 |         if hasattr(instance, "__unicode__"):
100 |             return unicode(instance)
101 |         elif hasattr(instance, "__str__"):
102 |             return bytes(instance).decode("utf8")
103 | 
104 |     return to_unicode(repr(instance))
105 | 


--------------------------------------------------------------------------------
/breadability/annotated_text.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | from __future__ import absolute_import
 4 | from __future__ import division, print_function, unicode_literals
 5 | 
 6 | from itertools import groupby
 7 | from lxml.sax import saxify, ContentHandler
 8 | from .utils import is_blank, shrink_text
 9 | from ._compat import to_unicode
10 | 
11 | 
12 | _SEMANTIC_TAGS = frozenset((
13 |     "a", "abbr", "acronym", "b", "big", "blink", "blockquote", "cite", "code",
14 |     "dd", "del", "dfn", "dir", "dl", "dt", "em", "h", "h1", "h2", "h3", "h4",
15 |     "h5", "h6", "i", "ins", "kbd", "li", "marquee", "menu", "ol", "pre", "q",
16 |     "s", "samp", "strike", "strong", "sub", "sup", "tt", "u", "ul", "var",
17 | ))
18 | 
19 | 
20 | class AnnotatedTextHandler(ContentHandler):
21 |     """A class for converting a HTML DOM into annotated text."""
22 | 
23 |     @classmethod
24 |     def parse(cls, dom):
25 |         """Converts DOM into paragraphs."""
26 |         handler = cls()
27 |         saxify(dom, handler)
28 |         return handler.content
29 | 
30 |     def __init__(self):
31 |         self._content = []
32 |         self._paragraph = []
33 |         self._dom_path = []
34 | 
35 |     @property
36 |     def content(self):
37 |         return self._content
38 | 
39 |     def startElementNS(self, name, qname, attrs):
40 |         namespace, name = name
41 | 
42 |         if name in _SEMANTIC_TAGS:
43 |             self._dom_path.append(to_unicode(name))
44 | 
45 |     def endElementNS(self, name, qname):
46 |         namespace, name = name
47 | 
48 |         if name == "p" and self._paragraph:
49 |             self._append_paragraph(self._paragraph)
50 |         elif name in ("ol", "ul", "pre") and self._paragraph:
51 |             self._append_paragraph(self._paragraph)
52 |             self._dom_path.pop()
53 |         elif name in _SEMANTIC_TAGS:
54 |             self._dom_path.pop()
55 | 
56 |     def endDocument(self):
57 |         if self._paragraph:
58 |             self._append_paragraph(self._paragraph)
59 | 
60 |     def _append_paragraph(self, paragraph):
61 |         paragraph = self._process_paragraph(paragraph)
62 |         self._content.append(paragraph)
63 |         self._paragraph = []
64 | 
65 |     def _process_paragraph(self, paragraph):
66 |         current_paragraph = []
67 | 
68 |         for annotation, items in groupby(paragraph, key=lambda i: i[1]):
69 |             if annotation and "li" in annotation:
70 |                 for text, _ in items:
71 |                     text = shrink_text(text)
72 |                     current_paragraph.append((text, annotation))
73 |             else:
74 |                 text = "".join(i[0] for i in items)
75 |                 text = shrink_text(text)
76 |                 current_paragraph.append((text, annotation))
77 | 
78 |         return tuple(current_paragraph)
79 | 
80 |     def characters(self, content):
81 |         if is_blank(content):
82 |             return
83 | 
84 |         if self._dom_path:
85 |             pair = (content, tuple(sorted(frozenset(self._dom_path))))
86 |         else:
87 |             pair = (content, None)
88 | 
89 |         self._paragraph.append(pair)
90 | 


--------------------------------------------------------------------------------
/breadability/document.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | """Generate a clean nice starting html document to process for an article."""
  4 | 
  5 | from __future__ import absolute_import
  6 | 
  7 | import logging
  8 | import re
  9 | 
 10 | import chardet
 11 | from lxml.etree import ParserError, XMLSyntaxError, tounicode
 12 | from lxml.html import HTMLParser, document_fromstring
 13 | 
 14 | from ._compat import to_bytes, to_unicode, unicode, unicode_compatible
 15 | from .utils import cached_property, ignored
 16 | 
 17 | logger = logging.getLogger("breadability")
 18 | 
 19 | 
 20 | TAG_MARK_PATTERN = re.compile(to_bytes(r"</?[^>]*>\s*"))
 21 | UTF8_PARSER = HTMLParser(encoding="utf8")
 22 | CHARSET_META_TAG_PATTERN = re.compile(
 23 |     br"""<meta[^>]+charset=["']?([^'"/>\s]+)""",
 24 |     re.IGNORECASE
 25 | )
 26 | 
 27 | 
 28 | def decode_html(html):
 29 |     """
 30 |     Converts bytes stream containing an HTML page into Unicode.
 31 |     Tries to guess character encoding from meta tag of by "chardet" library.
 32 |     """
 33 |     if isinstance(html, unicode):
 34 |         return html
 35 | 
 36 |     match = CHARSET_META_TAG_PATTERN.search(html)
 37 |     if match:
 38 |         declared_encoding = match.group(1).decode("ASCII")
 39 |         # proceed unknown encoding as if it wasn't found at all
 40 |         with ignored(LookupError):
 41 |             return html.decode(declared_encoding, "ignore")
 42 | 
 43 |     # try to enforce UTF-8 firstly
 44 |     with ignored(UnicodeDecodeError):
 45 |         return html.decode("utf8")
 46 | 
 47 |     text = TAG_MARK_PATTERN.sub(to_bytes(" "), html)
 48 |     diff = text.decode("utf8", "ignore").encode("utf8")
 49 |     sizes = len(diff), len(text)
 50 | 
 51 |     # 99% of text is UTF-8
 52 |     if abs(len(text) - len(diff)) < max(sizes) * 0.01:
 53 |         return html.decode("utf8", "ignore")
 54 | 
 55 |     # try detect encoding
 56 |     encoding = "utf8"
 57 |     encoding_detector = chardet.detect(text)
 58 |     if encoding_detector["encoding"]:
 59 |         encoding = encoding_detector["encoding"]
 60 | 
 61 |     return html.decode(encoding, "ignore")
 62 | 
 63 | 
 64 | BREAK_TAGS_PATTERN = re.compile(
 65 |     to_unicode(r"(?:<\s*[bh]r[^>]*>\s*)+"),
 66 |     re.IGNORECASE
 67 | )
 68 | 
 69 | 
 70 | def convert_breaks_to_paragraphs(html):
 71 |     """
 72 |     Converts <hr> tag and multiple <br> tags into paragraph.
 73 |     """
 74 |     logger.debug("Converting multiple <br> & <hr> tags into <p>.")
 75 | 
 76 |     return BREAK_TAGS_PATTERN.sub(_replace_break_tags, html)
 77 | 
 78 | 
 79 | def _replace_break_tags(match):
 80 |     tags = match.group()
 81 | 
 82 |     if to_unicode("<hr") in tags:
 83 |         return to_unicode("</p><p>")
 84 |     elif tags.count(to_unicode("<br")) > 1:
 85 |         return to_unicode("</p><p>")
 86 |     else:
 87 |         return tags
 88 | 
 89 | 
 90 | def build_document(html_content, base_href=None):
 91 |     """Requires that the `html_content` not be None"""
 92 |     assert html_content is not None
 93 | 
 94 |     if isinstance(html_content, unicode):
 95 |         html_content = html_content.encode("utf8", "xmlcharrefreplace")
 96 | 
 97 |     try:
 98 |         document = document_fromstring(html_content, parser=UTF8_PARSER)
 99 |     except (ParserError, XMLSyntaxError):
100 |         raise ValueError("Failed to parse document contents.")
101 | 
102 |     if base_href:
103 |         document.make_links_absolute(base_href, resolve_base_href=True)
104 |     else:
105 |         document.resolve_base_href()
106 | 
107 |     return document
108 | 
109 | 
110 | @unicode_compatible
111 | class OriginalDocument(object):
112 |     """The original document to process."""
113 | 
114 |     def __init__(self, html, url=None):
115 |         self._html = html
116 |         self._url = url
117 | 
118 |     @property
119 |     def url(self):
120 |         """Source URL of HTML document."""
121 |         return self._url
122 | 
123 |     def __unicode__(self):
124 |         """Renders the document as a string."""
125 |         return tounicode(self.dom)
126 | 
127 |     @cached_property
128 |     def dom(self):
129 |         """Parsed HTML document from the input."""
130 |         html = self._html
131 |         if not isinstance(html, unicode):
132 |             html = decode_html(html)
133 | 
134 |         html = convert_breaks_to_paragraphs(html)
135 |         document = build_document(html, self._url)
136 | 
137 |         return document
138 | 
139 |     @cached_property
140 |     def links(self):
141 |         """Links within the document."""
142 |         return self.dom.findall(".//a")
143 | 
144 |     @cached_property
145 |     def title(self):
146 |         """Title attribute of the parsed document."""
147 |         title_element = self.dom.find(".//title")
148 |         if title_element is None or title_element.text is None:
149 |             return ""
150 |         else:
151 |             return title_element.text.strip()
152 | 


--------------------------------------------------------------------------------
/breadability/readable.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | from __future__ import absolute_import
  4 | 
  5 | import logging
  6 | 
  7 | from copy import deepcopy
  8 | from operator import attrgetter
  9 | from pprint import PrettyPrinter
 10 | from lxml.html.clean import Cleaner
 11 | from lxml.etree import tounicode, tostring
 12 | from lxml.html import fragment_fromstring, fromstring
 13 | 
 14 | from .document import OriginalDocument
 15 | from .annotated_text import AnnotatedTextHandler
 16 | from .scoring import (
 17 |     get_class_weight,
 18 |     get_link_density,
 19 |     is_unlikely_node,
 20 |     score_candidates,
 21 | )
 22 | from .utils import cached_property, shrink_text
 23 | 
 24 | 
 25 | html_cleaner = Cleaner(
 26 |     scripts=True, javascript=True, comments=True,
 27 |     style=True, links=True, meta=False, add_nofollow=False,
 28 |     page_structure=False, processing_instructions=True,
 29 |     embedded=False, frames=False, forms=False,
 30 |     annoying_tags=False, remove_tags=None, kill_tags=("noscript", "iframe"),
 31 |     remove_unknown_tags=False, safe_attrs_only=False)
 32 | 
 33 | 
 34 | SCORABLE_TAGS = ("div", "p", "td", "pre", "article")
 35 | ANNOTATION_TAGS = (
 36 |     "a", "abbr", "acronym", "b", "big", "blink", "blockquote", "br", "cite",
 37 |     "code", "dd", "del", "dir", "dl", "dt", "em", "font", "h", "h1", "h2",
 38 |     "h3", "h4", "h5", "h6", "hr", "i", "ins", "kbd", "li", "marquee", "menu",
 39 |     "ol", "p", "pre", "q", "s", "samp", "span", "strike", "strong", "sub",
 40 |     "sup", "tt", "u", "ul", "var",
 41 | )
 42 | NULL_DOCUMENT = """
 43 | <html>
 44 |     <head>
 45 |         <meta http-equiv="Content-Type" content="text/html;charset=UTF-8">
 46 |     </head>
 47 |     <body>
 48 |     </body>
 49 | </html>
 50 | """
 51 | 
 52 | logger = logging.getLogger("breadability")
 53 | 
 54 | 
 55 | def ok_embedded_video(node):
 56 |     """Check if this embed/video is an ok one to count."""
 57 |     good_keywords = ('youtube', 'blip.tv', 'vimeo')
 58 | 
 59 |     node_str = tounicode(node)
 60 |     for key in good_keywords:
 61 |         if key in node_str:
 62 |             return True
 63 | 
 64 |     return False
 65 | 
 66 | 
 67 | def build_base_document(dom, return_fragment=True):
 68 |     """
 69 |     Builds a base document with the body as root.
 70 | 
 71 |     :param dom: Parsed lxml tree (Document Object Model).
 72 |     :param bool return_fragment: If True only <div> fragment is returned.
 73 |         Otherwise full HTML document is returned.
 74 |     """
 75 |     body_element = dom.find(".//body")
 76 | 
 77 |     if body_element is None:
 78 |         fragment = fragment_fromstring('<div id="readabilityBody"/>')
 79 |         fragment.append(dom)
 80 |     else:
 81 |         body_element.tag = "div"
 82 |         body_element.set("id", "readabilityBody")
 83 |         fragment = body_element
 84 | 
 85 |     return document_from_fragment(fragment, return_fragment)
 86 | 
 87 | 
 88 | def build_error_document(dom, return_fragment=True):
 89 |     """
 90 |     Builds an empty erorr document with the body as root.
 91 | 
 92 |     :param bool return_fragment: If True only <div> fragment is returned.
 93 |         Otherwise full HTML document is returned.
 94 |     """
 95 |     fragment = fragment_fromstring(
 96 |         '<div id="readabilityBody" class="parsing-error"/>')
 97 | 
 98 |     return document_from_fragment(fragment, return_fragment)
 99 | 
100 | 
101 | def document_from_fragment(fragment, return_fragment):
102 |     if return_fragment:
103 |         document = fragment
104 |     else:
105 |         document = fromstring(NULL_DOCUMENT)
106 |         body_element = document.find(".//body")
107 |         body_element.append(fragment)
108 | 
109 |     document.doctype = "<!DOCTYPE html>"
110 |     return document
111 | 
112 | 
113 | def check_siblings(candidate_node, candidate_list):
114 |     """
115 |     Looks through siblings for content that might also be related.
116 |     Things like preambles, content split by ads that we removed, etc.
117 |     """
118 |     candidate_css = candidate_node.node.get("class")
119 |     potential_target = candidate_node.content_score * 0.2
120 |     sibling_target_score = potential_target if potential_target > 10 else 10
121 |     parent = candidate_node.node.getparent()
122 |     siblings = parent.getchildren() if parent is not None else []
123 | 
124 |     for sibling in siblings:
125 |         append = False
126 |         content_bonus = 0
127 | 
128 |         if sibling is candidate_node.node:
129 |             append = True
130 | 
131 |         # Give a bonus if sibling nodes and top candidates have the example
132 |         # same class name
133 |         if candidate_css and sibling.get("class") == candidate_css:
134 |             content_bonus += candidate_node.content_score * 0.2
135 | 
136 |         if sibling in candidate_list:
137 |             adjusted_score = \
138 |                 candidate_list[sibling].content_score + content_bonus
139 | 
140 |             if adjusted_score >= sibling_target_score:
141 |                 append = True
142 | 
143 |         if sibling.tag == "p":
144 |             link_density = get_link_density(sibling)
145 |             content = sibling.text_content()
146 |             content_length = len(content)
147 | 
148 |             if content_length > 80 and link_density < 0.25:
149 |                 append = True
150 |             elif content_length < 80 and link_density == 0:
151 |                 if ". " in content:
152 |                     append = True
153 | 
154 |         if append:
155 |             logger.debug(
156 |                 "Sibling appended: %s %r", sibling.tag, sibling.attrib)
157 |             if sibling.tag not in ("div", "p"):
158 |                 # We have a node that isn't a common block level element, like
159 |                 # a form or td tag. Turn it into a div so it doesn't get
160 |                 # filtered out later by accident.
161 |                 sibling.tag = "div"
162 | 
163 |             if candidate_node.node != sibling:
164 |                 candidate_node.node.append(sibling)
165 | 
166 |     return candidate_node
167 | 
168 | 
169 | def clean_document(node):
170 |     """Cleans up the final document we return as the readable article."""
171 |     if node is None or len(node) == 0:
172 |         return None
173 | 
174 |     logger.debug("\n\n-------------- CLEANING DOCUMENT -----------------")
175 |     to_drop = []
176 | 
177 |     for n in node.iter():
178 |         # clean out any in-line style properties
179 |         if "style" in n.attrib:
180 |             n.set("style", "")
181 | 
182 |         # remove embended objects unless it's wanted video
183 |         if n.tag in ("object", "embed") and not ok_embedded_video(n):
184 |             logger.debug("Dropping node %s %r", n.tag, n.attrib)
185 |             to_drop.append(n)
186 | 
187 |         # clean headings with bad css or high link density
188 |         if n.tag in ("h1", "h2", "h3", "h4") and get_class_weight(n) < 0:
189 |             logger.debug("Dropping <%s>, it's insignificant", n.tag)
190 |             to_drop.append(n)
191 | 
192 |         if n.tag in ("h3", "h4") and get_link_density(n) > 0.33:
193 |             logger.debug("Dropping <%s>, it's insignificant", n.tag)
194 |             to_drop.append(n)
195 | 
196 |         # drop block element without content and children
197 |         if n.tag in ("div", "p"):
198 |             text_content = shrink_text(n.text_content())
199 |             if len(text_content) < 5 and not n.getchildren():
200 |                 logger.debug(
201 |                     "Dropping %s %r without content.", n.tag, n.attrib)
202 |                 to_drop.append(n)
203 | 
204 |         # finally try out the conditional cleaning of the target node
205 |         if clean_conditionally(n):
206 |             to_drop.append(n)
207 | 
208 |     drop_nodes_with_parents(to_drop)
209 | 
210 |     return node
211 | 
212 | 
213 | def drop_nodes_with_parents(nodes):
214 |     for node in nodes:
215 |         if node.getparent() is None:
216 |             continue
217 | 
218 |         node.drop_tree()
219 |         logger.debug(
220 |             "Dropped node with parent %s %r %s",
221 |             node.tag,
222 |             node.attrib,
223 |             node.text_content()[:50]
224 |         )
225 | 
226 | 
227 | def clean_conditionally(node):
228 |     """Remove the clean_el if it looks like bad content based on rules."""
229 |     if node.tag not in ('form', 'table', 'ul', 'div', 'p'):
230 |         return  # this is not the tag we are looking for
231 | 
232 |     weight = get_class_weight(node)
233 |     # content_score = LOOK up the content score for this node we found
234 |     # before else default to 0
235 |     content_score = 0
236 | 
237 |     if weight + content_score < 0:
238 |         logger.debug('Dropping conditional node')
239 |         logger.debug('Weight + score < 0')
240 |         return True
241 | 
242 |     commas_count = node.text_content().count(',')
243 |     if commas_count < 10:
244 |         logger.debug(
245 |             "There are %d commas so we're processing more.", commas_count)
246 | 
247 |         # If there are not very many commas, and the number of
248 |         # non-paragraph elements is more than paragraphs or other ominous
249 |         # signs, remove the element.
250 |         p = len(node.findall('.//p'))
251 |         img = len(node.findall('.//img'))
252 |         li = len(node.findall('.//li')) - 100
253 |         inputs = len(node.findall('.//input'))
254 | 
255 |         embed = 0
256 |         embeds = node.findall('.//embed')
257 |         for e in embeds:
258 |             if ok_embedded_video(e):
259 |                 embed += 1
260 |         link_density = get_link_density(node)
261 |         content_length = len(node.text_content())
262 | 
263 |         remove_node = False
264 | 
265 |         if li > p and node.tag != 'ul' and node.tag != 'ol':
266 |             logger.debug('Conditional drop: li > p and not ul/ol')
267 |             remove_node = True
268 |         elif inputs > p / 3.0:
269 |             logger.debug('Conditional drop: inputs > p/3.0')
270 |             remove_node = True
271 |         elif content_length < 25 and (img == 0 or img > 2):
272 |             logger.debug('Conditional drop: len < 25 and 0/>2 images')
273 |             remove_node = True
274 |         elif weight < 25 and link_density > 0.2:
275 |             logger.debug('Conditional drop: weight small (%f) and link is dense (%f)', weight, link_density)
276 |             remove_node = True
277 |         elif weight >= 25 and link_density > 0.5:
278 |             logger.debug('Conditional drop: weight big but link heavy')
279 |             remove_node = True
280 |         elif (embed == 1 and content_length < 75) or embed > 1:
281 |             logger.debug(
282 |                 'Conditional drop: embed w/o much content or many embed')
283 |             remove_node = True
284 | 
285 |         if remove_node:
286 |             logger.debug('Node will be removed: %s %r %s', node.tag, node.attrib, node.text_content()[:30])
287 | 
288 |         return remove_node
289 | 
290 |     return False  # nope, don't remove anything
291 | 
292 | 
293 | def prep_article(doc):
294 |     """Once we've found our target article we want to clean it up.
295 | 
296 |     Clean out:
297 |     - inline styles
298 |     - forms
299 |     - strip empty <p>
300 |     - extra tags
301 |     """
302 |     return clean_document(doc)
303 | 
304 | 
305 | def find_candidates(document):
306 |     """
307 |     Finds cadidate nodes for the readable version of the article.
308 | 
309 |     Here's we're going to remove unlikely nodes, find scores on the rest,
310 |     clean up and return the final best match.
311 |     """
312 |     nodes_to_score = set()
313 |     should_remove = set()
314 | 
315 |     for node in document.iter():
316 |         if is_unlikely_node(node):
317 |             logger.debug(
318 |                 "We should drop unlikely: %s %r", node.tag, node.attrib)
319 |             should_remove.add(node)
320 |         elif is_bad_link(node):
321 |             logger.debug(
322 |                 "We should drop bad link: %s %r", node.tag, node.attrib)
323 |             should_remove.add(node)
324 |         elif node.tag in SCORABLE_TAGS:
325 |             nodes_to_score.add(node)
326 | 
327 |     return score_candidates(nodes_to_score), should_remove
328 | 
329 | 
330 | def is_bad_link(node):
331 |     """
332 |     Helper to determine if the node is link that is useless.
333 | 
334 |     We've hit articles with many multiple links that should be cleaned out
335 |     because they're just there to pollute the space. See tests for examples.
336 |     """
337 |     if node.tag != "a":
338 |         return False
339 | 
340 |     name = node.get("name")
341 |     href = node.get("href")
342 |     if name and not href:
343 |         return True
344 | 
345 |     if href:
346 |         href_parts = href.split("#")
347 |         if len(href_parts) == 2 and len(href_parts[1]) > 25:
348 |             return True
349 | 
350 |     return False
351 | 
352 | 
353 | class Article(object):
354 |     """Parsed readable object"""
355 | 
356 |     def __init__(self, html, url=None, return_fragment=True):
357 |         """
358 |         Create the Article we're going to use.
359 | 
360 |         :param html: The string of HTML we're going to parse.
361 |         :param url: The url so we can adjust the links to still work.
362 |         :param return_fragment: Should we return a <div> fragment or
363 |             a full <html> document.
364 |         """
365 |         self._original_document = OriginalDocument(html, url=url)
366 |         self._return_fragment = return_fragment
367 | 
368 |     def __str__(self):
369 |         return tostring(self._readable())
370 | 
371 |     def __unicode__(self):
372 |         return tounicode(self._readable())
373 | 
374 |     @cached_property
375 |     def dom(self):
376 |         """Parsed lxml tree (Document Object Model) of the given html."""
377 |         try:
378 |             dom = self._original_document.dom
379 |             # cleaning doesn't return, just wipes in place
380 |             html_cleaner(dom)
381 |             return leaf_div_elements_into_paragraphs(dom)
382 |         except ValueError:
383 |             return None
384 | 
385 |     @cached_property
386 |     def candidates(self):
387 |         """Generates list of candidates from the DOM."""
388 |         dom = self.dom
389 |         if dom is None or len(dom) == 0:
390 |             return None
391 | 
392 |         candidates, unlikely_candidates = find_candidates(dom)
393 |         drop_nodes_with_parents(unlikely_candidates)
394 | 
395 |         return candidates
396 | 
397 |     @cached_property
398 |     def main_text(self):
399 |         dom = deepcopy(self.readable_dom).get_element_by_id("readabilityBody")
400 |         return AnnotatedTextHandler.parse(dom)
401 | 
402 |     @cached_property
403 |     def readable(self):
404 |         return tounicode(self.readable_dom)
405 | 
406 |     @cached_property
407 |     def readable_dom(self):
408 |         return self._readable()
409 | 
410 |     def _readable(self):
411 |         """The readable parsed article"""
412 |         if not self.candidates:
413 |             logger.info("No candidates found in document.")
414 |             return self._handle_no_candidates()
415 | 
416 |         # right now we return the highest scoring candidate content
417 |         best_candidates = sorted(
418 |             (c for c in self.candidates.values()),
419 |             key=attrgetter("content_score"), reverse=True)
420 | 
421 |         printer = PrettyPrinter(indent=2)
422 |         logger.debug(printer.pformat(best_candidates))
423 | 
424 |         # since we have several candidates, check the winner's siblings
425 |         # for extra content
426 |         winner = best_candidates[0]
427 |         updated_winner = check_siblings(winner, self.candidates)
428 |         updated_winner.node = prep_article(updated_winner.node)
429 |         if updated_winner.node is not None:
430 |             dom = build_base_document(
431 |                 updated_winner.node, self._return_fragment)
432 |         else:
433 |             logger.info(
434 |                 'Had candidates but failed to find a cleaned winning DOM.')
435 |             dom = self._handle_no_candidates()
436 | 
437 |         return self._remove_orphans(dom.get_element_by_id("readabilityBody"))
438 | 
439 |     def _remove_orphans(self, dom):
440 |         for node in dom.iterdescendants():
441 |             if len(node) == 1 and tuple(node)[0].tag == node.tag:
442 |                 node.drop_tag()
443 | 
444 |         return dom
445 | 
446 |     def _handle_no_candidates(self):
447 |         """
448 |         If we fail to find a good candidate we need to find something else.
449 |         """
450 |         # since we've not found a good candidate we're should help this
451 |         if self.dom is not None and len(self.dom):
452 |             dom = prep_article(self.dom)
453 |             dom = build_base_document(dom, self._return_fragment)
454 |             return self._remove_orphans(
455 |                 dom.get_element_by_id("readabilityBody"))
456 |         else:
457 |             logger.info("No document to use.")
458 |             return build_error_document(self._return_fragment)
459 | 
460 | 
461 | def leaf_div_elements_into_paragraphs(document):
462 |     """
463 |     Turn some block elements that don't have children block level
464 |     elements into <p> elements.
465 | 
466 |     Since we can't change the tree as we iterate over it, we must do this
467 |     before we process our document.
468 |     """
469 |     for element in document.iter(tag="div"):
470 |         child_tags = tuple(n.tag for n in element.getchildren())
471 |         if "div" not in child_tags and "p" not in child_tags:
472 |             logger.debug(
473 |                 "Changing leaf block element <%s> into <p>", element.tag)
474 |             element.tag = "p"
475 | 
476 |     return document
477 | 


--------------------------------------------------------------------------------
/breadability/scoring.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | """Handle dealing with scoring nodes and content for our parsing."""
  4 | 
  5 | from __future__ import absolute_import
  6 | from __future__ import division, print_function
  7 | 
  8 | import re
  9 | import logging
 10 | 
 11 | from hashlib import md5
 12 | from lxml.etree import tostring
 13 | from ._compat import to_bytes
 14 | from .utils import normalize_whitespace
 15 | 
 16 | 
 17 | # A series of sets of attributes we check to help in determining if a node is
 18 | # a potential candidate or not.
 19 | CLS_UNLIKELY = re.compile(
 20 |     "combx|comment|community|disqus|extra|foot|header|menu|remark|rss|"
 21 |     "shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|perma|popup|"
 22 |     "tweet|twitter|social|breadcrumb",
 23 |     re.IGNORECASE
 24 | )
 25 | CLS_MAYBE = re.compile(
 26 |     "and|article|body|column|main|shadow|entry",
 27 |     re.IGNORECASE
 28 | )
 29 | CLS_WEIGHT_POSITIVE = re.compile(
 30 |     "article|body|content|entry|main|page|pagination|post|text|blog|story",
 31 |     re.IGNORECASE
 32 | )
 33 | CLS_WEIGHT_NEGATIVE = re.compile(
 34 |     "combx|comment|com-|contact|foot|footer|footnote|head|masthead|media|meta|"
 35 |     "outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|"
 36 |     "tool|widget",
 37 |     re.IGNORECASE
 38 | )
 39 | 
 40 | logger = logging.getLogger("breadability")
 41 | 
 42 | 
 43 | def check_node_attributes(pattern, node, *attributes):
 44 |     """
 45 |     Searches match in attributes against given pattern and if
 46 |     finds the match against any of them returns True.
 47 |     """
 48 |     for attribute_name in attributes:
 49 |         attribute = node.get(attribute_name)
 50 |         if attribute is not None and pattern.search(attribute):
 51 |             return True
 52 | 
 53 |     return False
 54 | 
 55 | 
 56 | def generate_hash_id(node):
 57 |     """
 58 |     Generates a hash_id for the node in question.
 59 | 
 60 |     :param node: lxml etree node
 61 |     """
 62 |     try:
 63 |         content = tostring(node)
 64 |     except Exception:
 65 |         logger.exception("Generating of hash failed")
 66 |         content = to_bytes(repr(node))
 67 | 
 68 |     hash_id = md5(content).hexdigest()
 69 |     return hash_id[:8]
 70 | 
 71 | 
 72 | def get_link_density(node, node_text=None):
 73 |     """
 74 |     Computes the ratio for text in given node and text in links
 75 |     contained in the node. It is computed from number of
 76 |     characters in the texts.
 77 | 
 78 |     :parameter Element node:
 79 |         HTML element in which links density is computed.
 80 |     :parameter string node_text:
 81 |         Text content of given node if it was obtained before.
 82 |     :returns float:
 83 |         Returns value of computed 0 <= density <= 1, where 0 means
 84 |         no links and 1 means that node contains only links.
 85 |     """
 86 |     if node_text is None:
 87 |         node_text = node.text_content()
 88 |     node_text = normalize_whitespace(node_text.strip())
 89 | 
 90 |     text_length = len(node_text)
 91 |     if text_length == 0:
 92 |         return 0.0
 93 | 
 94 |     links_length = sum(map(_get_normalized_text_length, node.findall(".//a")))
 95 |     # Give 50 bonus chars worth of length for each img.
 96 |     # Tweaking this 50 down a notch should help if we hit false positives.
 97 |     img_bonuses = 50 * len(node.findall(".//img"))
 98 |     links_length = max(0, links_length - img_bonuses)
 99 | 
100 |     return links_length / text_length
101 | 
102 | 
103 | def _get_normalized_text_length(node):
104 |     return len(normalize_whitespace(node.text_content().strip()))
105 | 
106 | 
107 | def get_class_weight(node):
108 |     """
109 |     Computes weight of element according to its class/id.
110 | 
111 |     We're using sets to help efficiently check for existence of matches.
112 |     """
113 |     weight = 0
114 | 
115 |     if check_node_attributes(CLS_WEIGHT_NEGATIVE, node, "class"):
116 |         weight -= 25
117 |     if check_node_attributes(CLS_WEIGHT_POSITIVE, node, "class"):
118 |         weight += 25
119 | 
120 |     if check_node_attributes(CLS_WEIGHT_NEGATIVE, node, "id"):
121 |         weight -= 25
122 |     if check_node_attributes(CLS_WEIGHT_POSITIVE, node, "id"):
123 |         weight += 25
124 | 
125 |     return weight
126 | 
127 | 
128 | def is_unlikely_node(node):
129 |     """
130 |     Short helper for checking unlikely status.
131 | 
132 |     If the class or id are in the unlikely list, and there's not also a
133 |     class/id in the likely list then it might need to be removed.
134 |     """
135 |     unlikely = check_node_attributes(CLS_UNLIKELY, node, "class", "id")
136 |     maybe = check_node_attributes(CLS_MAYBE, node, "class", "id")
137 | 
138 |     return bool(unlikely and not maybe and node.tag != "body")
139 | 
140 | 
141 | def score_candidates(nodes):
142 |     """Given a list of potential nodes, find some initial scores to start"""
143 |     MIN_HIT_LENTH = 25
144 |     candidates = {}
145 | 
146 |     for node in nodes:
147 |         logger.debug("* Scoring candidate %s %r", node.tag, node.attrib)
148 | 
149 |         # if the node has no parent it knows of then it ends up creating a
150 |         # body & html tag to parent the html fragment
151 |         parent = node.getparent()
152 |         if parent is None:
153 |             logger.debug("Skipping candidate - parent node is 'None'.")
154 |             continue
155 | 
156 |         grand = parent.getparent()
157 |         if grand is None:
158 |             logger.debug("Skipping candidate - grand parent node is 'None'.")
159 |             continue
160 | 
161 |         # if paragraph is < `MIN_HIT_LENTH` characters don't even count it
162 |         inner_text = node.text_content().strip()
163 |         if len(inner_text) < MIN_HIT_LENTH:
164 |             logger.debug(
165 |                 "Skipping candidate - inner text < %d characters.",
166 |                 MIN_HIT_LENTH)
167 |             continue
168 | 
169 |         # initialize readability data for the parent
170 |         # add parent node if it isn't in the candidate list
171 |         if parent not in candidates:
172 |             candidates[parent] = ScoredNode(parent)
173 | 
174 |         if grand not in candidates:
175 |             candidates[grand] = ScoredNode(grand)
176 | 
177 |         # add a point for the paragraph itself as a base
178 |         content_score = 1
179 | 
180 |         if inner_text:
181 |             # add 0.25 points for any commas within this paragraph
182 |             commas_count = inner_text.count(",")
183 |             content_score += commas_count * 0.25
184 |             logger.debug("Bonus points for %d commas.", commas_count)
185 | 
186 |             # subtract 0.5 points for each double quote within this paragraph
187 |             double_quotes_count = inner_text.count('"')
188 |             content_score += double_quotes_count * -0.5
189 |             logger.debug(
190 |                 "Penalty points for %d double-quotes.", double_quotes_count)
191 | 
192 |             # for every 100 characters in this paragraph, add another point
193 |             # up to 3 points
194 |             length_points = len(inner_text) / 100
195 |             content_score += min(length_points, 3.0)
196 |             logger.debug("Bonus points for length of text: %f", length_points)
197 | 
198 |         # add the score to the parent
199 |         logger.debug(
200 |             "Bonus points for parent %s %r with score %f: %f",
201 |             parent.tag, parent.attrib, candidates[parent].content_score,
202 |             content_score)
203 |         candidates[parent].content_score += content_score
204 |         # the grand node gets half
205 |         logger.debug(
206 |             "Bonus points for grand %s %r with score %f: %f",
207 |             grand.tag, grand.attrib, candidates[grand].content_score,
208 |             content_score / 2.0)
209 |         candidates[grand].content_score += content_score / 2.0
210 | 
211 |         if node not in candidates:
212 |             candidates[node] = ScoredNode(node)
213 |         candidates[node].content_score += content_score
214 | 
215 |     for candidate in candidates.values():
216 |         adjustment = 1.0 - get_link_density(candidate.node)
217 |         candidate.content_score *= adjustment
218 |         logger.debug(
219 |             "Link density adjustment for %s %r: %f",
220 |             candidate.node.tag, candidate.node.attrib, adjustment)
221 | 
222 |     return candidates
223 | 
224 | 
225 | class ScoredNode(object):
226 |     """
227 |     We need Scored nodes we use to track possible article matches
228 | 
229 |     We might have a bunch of these so we use __slots__ to keep memory usage
230 |     down.
231 |     """
232 |     __slots__ = ('node', 'content_score')
233 | 
234 |     def __init__(self, node):
235 |         """Given node, set an initial score and weigh based on css and id"""
236 |         self.node = node
237 |         self.content_score = 0
238 | 
239 |         if node.tag in ('div', 'article'):
240 |             self.content_score = 5
241 |         if node.tag in ('pre', 'td', 'blockquote'):
242 |             self.content_score = 3
243 | 
244 |         if node.tag in ('address', 'ol', 'ul', 'dl', 'dd', 'dt', 'li', 'form'):
245 |             self.content_score = -3
246 |         if node.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
247 |             self.content_score = -5
248 | 
249 |         self.content_score += get_class_weight(node)
250 | 
251 |     @property
252 |     def hash_id(self):
253 |         return generate_hash_id(self.node)
254 | 
255 |     def __repr__(self):
256 |         if self.node is None:
257 |             return "<NullScoredNode with score {2:0.1F}>" % self.content_score
258 | 
259 |         return "<ScoredNode {0} {1}: {2:0.1F}>".format(
260 |             self.node.tag,
261 |             self.node.attrib,
262 |             self.content_score
263 |         )
264 | 


--------------------------------------------------------------------------------
/breadability/scripts/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/breadability/scripts/__init__.py


--------------------------------------------------------------------------------
/breadability/scripts/client.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """
 4 | A fast python port of arc90's readability tool
 5 | 
 6 | Usage:
 7 |     breadability [options] <resource>
 8 |     breadability --version
 9 |     breadability --help
10 | 
11 | Arguments:
12 |   <resource>      URL or file path to process in readable form.
13 | 
14 | Options:
15 |   -f, --fragment  Output html fragment by default.
16 |   -b, --browser   Open the parsed content in your web browser.
17 |   -d, --debug     Output the detailed scoring information for debugging
18 |                   parsing.
19 |   -v, --verbose   Increase logging verbosity to DEBUG.
20 |   --version       Display program's version number and exit.
21 |   -h, --help      Display this help message and exit.
22 | """
23 | 
24 | from __future__ import absolute_import
25 | from __future__ import division, print_function, unicode_literals
26 | 
27 | 
28 | import logging
29 | import locale
30 | import webbrowser
31 | 
32 | from tempfile import NamedTemporaryFile
33 | from docopt import docopt
34 | from .. import __version__
35 | from .._compat import urllib
36 | from ..readable import Article
37 | 
38 | 
39 | HEADERS = {
40 |     "User-Agent": 'breadability/{version} ({url})'.format(
41 |         url="https://github.com/bookieio/breadability",
42 |         version=__version__
43 |     )
44 | }
45 | 
46 | 
47 | def parse_args():
48 |     return docopt(__doc__, version=__version__)
49 | 
50 | 
51 | def main():
52 |     args = parse_args()
53 |     logger = logging.getLogger("breadability")
54 | 
55 |     if args["--verbose"]:
56 |         logger.setLevel(logging.DEBUG)
57 | 
58 |     resource = args["<resource>"]
59 |     if resource.startswith("www"):
60 |         resource = "http://" + resource
61 | 
62 |     url = None
63 |     if resource.startswith("http://") or resource.startswith("https://"):
64 |         url = resource
65 | 
66 |         request = urllib.Request(url, headers=HEADERS)
67 |         response = urllib.urlopen(request)
68 |         content = response.read()
69 |         response.close()
70 |     else:
71 |         with open(resource, "r") as file:
72 |             content = file.read()
73 | 
74 |     document = Article(content, url=url, return_fragment=args["--fragment"])
75 |     if args["--browser"]:
76 |         html_file = NamedTemporaryFile(mode="wb", suffix=".html", delete=False)
77 | 
78 |         content = document.readable.encode("utf8")
79 |         html_file.write(content)
80 |         html_file.close()
81 | 
82 |         webbrowser.open(html_file.name)
83 |     else:
84 |         encoding = locale.getpreferredencoding()
85 |         content = document.readable.encode(encoding)
86 |         print(content)
87 | 
88 | 
89 | if __name__ == '__main__':
90 |     main()
91 | 


--------------------------------------------------------------------------------
/breadability/scripts/test_helper.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | """
  4 | Helper to generate a new set of article test files for breadability.
  5 | 
  6 | Usage:
  7 |     breadability_test --name <name> <url>
  8 |     breadability_test --version
  9 |     breadability_test --help
 10 | 
 11 | Arguments:
 12 |   <url>                   The url of content to fetch for the article.html
 13 | 
 14 | Options:
 15 |   -n <name>, --name=<name>  Name of the test directory.
 16 |   --version                 Show program's version number and exit.
 17 |   -h, --help                Show this help message and exit.
 18 | """
 19 | 
 20 | from __future__ import absolute_import
 21 | from __future__ import division, print_function, unicode_literals
 22 | 
 23 | from os import mkdir
 24 | from os.path import join, dirname, pardir, exists as path_exists
 25 | from docopt import docopt
 26 | from .. import __version__
 27 | from .._compat import to_unicode, urllib
 28 | 
 29 | 
 30 | TEST_PATH = join(
 31 |     dirname(__file__),
 32 |     pardir, pardir,
 33 |     "tests/test_articles"
 34 | )
 35 | 
 36 | TEST_TEMPLATE = '''# -*- coding: utf8 -*-
 37 | 
 38 | """
 39 | Test the scoring and parsing of the article from URL below:
 40 | %(source_url)s
 41 | """
 42 | 
 43 | from __future__ import absolute_import, division, print_function, unicode_literals
 44 | 
 45 | import os
 46 | 
 47 | import pytest
 48 | 
 49 | from breadability.readable import Article
 50 | 
 51 | 
 52 | @pytest.fixture(scope="module")
 53 | def article():
 54 |     """Load up the article for us"""
 55 |     article_path = os.path.join(os.path.dirname(__file__), "article.html")
 56 |     with open(article_path, "rb") as file:
 57 |         return Article(file.read(), "%(source_url)s")
 58 | 
 59 | 
 60 | def test_parses(article):
 61 |     """Verify we can parse the document."""
 62 |     assert 'id="readabilityBody"' in article.readable
 63 | 
 64 | 
 65 | def test_content_exists(article):
 66 |     """Verify that some content exists."""
 67 |     assert "#&@#&@#&@" in article.readable
 68 | 
 69 | 
 70 | def test_content_does_not_exist(article):
 71 |     """Verify we cleaned out some content that shouldn't exist."""
 72 |     assert "" not in article.readable
 73 | '''
 74 | 
 75 | 
 76 | def parse_args():
 77 |     return docopt(__doc__, version=__version__)
 78 | 
 79 | 
 80 | def make_test_directory(name):
 81 |     """Generates a new directory for tests."""
 82 |     directory_name = "test_" + name.replace(" ", "_")
 83 |     directory_path = join(TEST_PATH, directory_name)
 84 | 
 85 |     if not path_exists(directory_path):
 86 |         mkdir(directory_path)
 87 | 
 88 |     return directory_path
 89 | 
 90 | 
 91 | def make_test_files(directory_path, url):
 92 |     init_file = join(directory_path, "__init__.py")
 93 |     open(init_file, "a").close()
 94 | 
 95 |     data = TEST_TEMPLATE % {
 96 |         "source_url": to_unicode(url)
 97 |     }
 98 | 
 99 |     test_file = join(directory_path, "test_article.py")
100 |     with open(test_file, "w") as file:
101 |         file.write(data)
102 | 
103 | 
104 | def fetch_article(directory_path, url):
105 |     """Get the content of the url and make it the article.html"""
106 |     opener = urllib.build_opener()
107 |     opener.addheaders = [("Accept-Charset", "utf-8")]
108 | 
109 |     response = opener.open(url)
110 |     html_data = response.read()
111 |     response.close()
112 | 
113 |     path = join(directory_path, "article.html")
114 |     with open(path, "wb") as file:
115 |         file.write(html_data)
116 | 
117 | 
118 | def main():
119 |     """Run the script."""
120 |     args = parse_args()
121 |     directory = make_test_directory(args["--name"])
122 |     make_test_files(directory, args["<url>"])
123 |     fetch_article(directory, args["<url>"])
124 | 
125 | 
126 | if __name__ == "__main__":
127 |     main()
128 | 


--------------------------------------------------------------------------------
/breadability/utils.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | from __future__ import absolute_import
 4 | from __future__ import division, print_function, unicode_literals
 5 | 
 6 | import re
 7 | 
 8 | try:
 9 |     from contextlib import ignored
10 | except ImportError:
11 |     from contextlib import contextmanager
12 | 
13 |     @contextmanager
14 |     def ignored(*exceptions):
15 |         try:
16 |             yield
17 |         except tuple(exceptions):
18 |             pass
19 | 
20 | 
21 | MULTIPLE_WHITESPACE_PATTERN = re.compile(r"\s+", re.UNICODE)
22 | 
23 | 
24 | def is_blank(text):
25 |     """
26 |     Returns ``True`` if string contains only whitespace characters
27 |     or is empty. Otherwise ``False`` is returned.
28 |     """
29 |     return not text or text.isspace()
30 | 
31 | 
32 | def shrink_text(text):
33 |     return normalize_whitespace(text.strip())
34 | 
35 | 
36 | def normalize_whitespace(text):
37 |     """
38 |     Translates multiple whitespace into single space character.
39 |     If there is at least one new line character chunk is replaced
40 |     by single LF (Unix new line) character.
41 |     """
42 |     return MULTIPLE_WHITESPACE_PATTERN.sub(_replace_whitespace, text)
43 | 
44 | 
45 | def _replace_whitespace(match):
46 |     text = match.group()
47 | 
48 |     if "\n" in text or "\r" in text:
49 |         return "\n"
50 |     else:
51 |         return " "
52 | 
53 | 
54 | def cached_property(getter):
55 |     """
56 |     Decorator that converts a method into memoized property.
57 |     The decorator works as expected only for classes with
58 |     attribute '__dict__' and immutable properties.
59 |     """
60 |     def decorator(self):
61 |         key = "_cached_property_" + getter.__name__
62 | 
63 |         if not hasattr(self, key):
64 |             setattr(self, key, getter(self))
65 | 
66 |         return getattr(self, key)
67 | 
68 |     decorator.__name__ = getter.__name__
69 |     decorator.__module__ = getter.__module__
70 |     decorator.__doc__ = getter.__doc__
71 | 
72 |     return property(decorator)
73 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | docopt>=0.6.1,<0.7
 2 | chardet
 3 | lxml[html_clean]
 4 | 
 5 | pytest
 6 | pytest-cov
 7 | coverage
 8 | pylint
 9 | pep8
10 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [tool:pytest]
2 | addopts = --quiet --tb=native --color=yes
3 | 
4 | [bdist_wheel]
5 | universal = 1
6 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | 
 3 | from os.path import (
 4 |     abspath,
 5 |     dirname,
 6 |     join,
 7 | )
 8 | from setuptools import setup
 9 | 
10 | VERSION = "0.1.20"
11 | 
12 | VERSION_SUFFIX = "%d.%d" % sys.version_info[:2]
13 | CURRENT_DIRECTORY = abspath(dirname(__file__))
14 | 
15 | 
16 | with open(join(CURRENT_DIRECTORY, "README.rst")) as readme:
17 |     with open(join(CURRENT_DIRECTORY, "CHANGELOG.rst")) as changelog:
18 |         long_description = "%s\n\n%s" % (readme.read(), changelog.read())
19 | 
20 | 
21 | install_requires = [
22 |     "docopt>=0.6.1,<0.7",
23 |     "chardet",
24 |     "lxml[html_clean]>=2.0",
25 | ]
26 | tests_require = [
27 |     "pytest",
28 |     "pytest-cov",
29 |     "coverage",
30 |     "pylint",
31 |     "pep8",
32 | ]
33 | 
34 | 
35 | console_script_targets = [
36 |     "breadability = breadability.scripts.client:main",
37 |     "breadability-{0} = breadability.scripts.client:main",
38 |     "breadability_test = breadability.scripts.test_helper:main",
39 |     "breadability_test-{0} = breadability.scripts.test_helper:main",
40 | ]
41 | console_script_targets = [
42 |     target.format(VERSION_SUFFIX) for target in console_script_targets
43 | ]
44 | 
45 | 
46 | setup(
47 |     name="breadability",
48 |     version=VERSION,
49 |     description="Port of Readability HTML parser in Python",
50 |     long_description=long_description,
51 |     keywords=[
52 |         "bookie",
53 |         "breadability",
54 |         "content",
55 |         "HTML",
56 |         "parsing",
57 |         "readability",
58 |         "readable",
59 |     ],
60 |     author="Rick Harding",
61 |     author_email="rharding@mitechie.com",
62 |     url="https://github.com/bookieio/breadability",
63 |     license="BSD",
64 |     classifiers=[
65 |         "Development Status :: 5 - Production/Stable",
66 |         "Intended Audience :: Developers",
67 |         "License :: OSI Approved :: BSD License",
68 |         "Operating System :: OS Independent",
69 |         "Programming Language :: Python",
70 |         "Programming Language :: Python :: 2",
71 |         "Programming Language :: Python :: 2.6",
72 |         "Programming Language :: Python :: 2.7",
73 |         "Programming Language :: Python :: 3",
74 |         "Programming Language :: Python :: 3.2",
75 |         "Programming Language :: Python :: 3.3",
76 |         "Programming Language :: Python :: 3.4",
77 |         "Programming Language :: Python :: 3.5",
78 |         "Programming Language :: Python :: 3.6",
79 |         "Programming Language :: Python :: Implementation :: CPython",
80 |         "Topic :: Internet :: WWW/HTTP",
81 |         "Topic :: Software Development :: Pre-processors",
82 |         "Topic :: Text Processing :: Filters",
83 |         "Topic :: Text Processing :: Markup :: HTML",
84 |     ],
85 |     packages=['breadability', 'breadability.scripts'],
86 |     include_package_data=True,
87 |     zip_safe=False,
88 |     install_requires=install_requires,
89 |     tests_require=tests_require,
90 |     test_suite="tests",
91 |     entry_points={
92 |         "console_scripts": console_script_targets,
93 |     }
94 | )
95 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/__init__.py


--------------------------------------------------------------------------------
/tests/compat.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf8 -*-
2 | 
3 | from __future__ import absolute_import
4 | from __future__ import division, print_function, unicode_literals
5 | 


--------------------------------------------------------------------------------
/tests/data/articles/ars.001.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  2 |   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  3 | 
  4 | <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US">
  5 |   <head>
  6 |       <script type='text/javascript'>var _sf_startpt=(new Date()).getTime()</script>
  7 |         <title>1859's &quot;Great Auroral Storm&quot;&mdash;the week the Sun touched the earth</title>
  8 | 
  9 |   <!-- Begin CSS -->
 10 |   <link rel="stylesheet" type="text/css" href="http://static.arstechnica.net//public/v6/styles/light/light.c.css?1335816272" media="screen" />
 11 |   <link rel="stylesheet" type="text/css" href="http://static.arstechnica.net//public/v6/styles/print/print.css?1335816272" media="print" />
 12 |   <!-- End CSS -->
 13 | 
 14 |   <link rel="apple-touch-icon" href="http://static.arstechnica.net/apple-touch-icon.png" />
 15 |   <link rel="canonical" href="http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars" /> 
 16 |     <link rel="shorturl" href="http://arst.ch/tmc" />
 17 |   <link rel="shortlink" href="http://arst.ch/tmc" />
 18 |   <link rev="canonical" href="http://arst.ch/tmc" />
 19 |     
 20 |   <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="Ars Technica" /> 
 21 |   <link rel="shortcut icon" href="http://static.arstechnica.net/favicon.ico" />
 22 |   <link rel="icon" type="image/x-icon" href="http://static.arstechnica.net/favicon.ico" />
 23 | 
 24 |   <!-- Begin Feeds -->
 25 |     <link rel="alternate" type="application/rssxml" title="Nobel Intent" href="http://feeds.arstechnica.com/arstechnica/science/" />
 26 |   
 27 |     <link rel="alternate" type="application/rss+xml" title="All Articles " href="http://feeds.arstechnica.com/arstechnica/everything" />
 28 |     <!-- End Feeds -->
 29 | 
 30 |   <!-- C-razy IE9 stuff -->
 31 |   <meta name="application-name" content="Ars Technica"/>
 32 |   <meta name="msapplication-starturl" content="http://arstechnica.com/"/>
 33 |   <meta name="msapplication-tooltip" content="Ars Technica: Serving the technologist for 1.2 decades"/>
 34 |   <meta name="msapplication-task" content="name=News;action-uri=http://arstechnica.com/;icon-uri=http://arstechnica.com/favicon.ico"/>
 35 |   <meta name="msapplication-task" content="name=Features;action-uri=http://arstechnica.com/features/;icon-uri=http://static.arstechnica.net/ie-jump-menu/jump-features.ico"/>
 36 |   <meta name="msapplication-task" content="name=OpenForum;action-uri=http://arstechnica.com/civis/;icon-uri=http://static.arstechnica.net/ie-jump-menu/jump-forum.ico"/>
 37 |   <meta name="msapplication-task" content="name=One Microsoft Way;action-uri=http://arstechnica.com/microsoft/;icon-uri=http://static.arstechnica.net/ie-jump-menu/jump-omw.ico"/>
 38 |   <meta name="msapplication-task" content="name=Subscribe;action-uri=http://arstechnica.com/subscriptions/;icon-uri=http://static.arstechnica.net/ie-jump-menu/jump-subscribe.ico"/>
 39 | 
 40 |   
 41 |   <!-- Begin Metadata -->
 42 |   <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 43 |   <meta name="viewport" content="width=1000" />
 44 |     <meta name="description" content="150 years ago, the world was engulfed in the largest electromagnetic storm on record, the &quot;Carrington Event&quot; of 1859. How did humanity react to the week-long solar maelstrom, and what will we do when it happens again?" />
 45 |       <meta name="keywords" content="carringtonevent, greatauroralstormof1859, magneticstorm" />
 46 |       <meta name="title" content="1859's &quot;Great Auroral Storm&quot;&mdash;the week the Sun touched the earth" />
 47 |   <link rel="image_src" href="http://static.arstechnica.net/assets/2012/04/sun-eruption-4f9e9cc-listing-thumb-300x169-33537-f.jpg" />
 48 |   <meta name="medium" content="news" />
 49 |   
 50 |   <meta name="entry_id" content="55765" />
 51 |   <meta property="og:title" content="1859's &quot;Great Auroral Storm&quot;&mdash;the week the Sun touched the earth"/>
 52 |   <meta property="og:site_name" content="Ars Technica"/>
 53 |   <meta property="og:image" content="http://static.arstechnica.net/assets/2012/04/sun-eruption-4f9e9cc-listing-thumb-300x169-33537-f.jpg"/>
 54 |   
 55 |     <meta name="advertising" content="ask" />
 56 |   <meta property="fb:admins" content="13703630" />
 57 |   <!-- End Metadata -->
 58 |   <!-- Entry - culture_science -->
 59 | <style type="text/css" id="resource-styles">  </style>
 60 |       <script type="text/javascript" src="http://static.arstechnica.com/public/shared/scripts/da-1.5.js"></script>
 61 |     <script type="text/javascript">
 62 |     try {
 63 | 			      cnp.ad.dart.setSite("ars.dart");
 64 | 			      cnp.ad.dart.setZone('culture_science');
 65 |       //cnp.ad.dart.addParameterString('kw=1859s-great-auroral-stormthe-week-the-sun-touched-the-earth;kw=05;kw=2012;kw=news;kw=science;');
 66 |       cnp.ad.emptyFrameSrc="/public/shared/scripts/empty.html";
 67 |       cnp.ad.loaderFrameSrc="/public/shared/scripts/ad-loader-frame.html";
 68 |     } catch(e) {}
 69 |     </script>
 70 |      
 71 |   <script type="text/javascript" charset="utf-8">
 72 |     // In case someone on a desktop clicks a mobile #! link
 73 |     var l = window.location;
 74 |     if(l.hash.indexOf('#!') !== -1){
 75 |       window.location = l.protocol + '//' + l.host + l.hash.slice(2);
 76 |     }
 77 |   </script>
 78 |   </head>
 79 |   <body class="individual">
 80 |     <div id="page" class="">
 81 |       
 82 | <div id="masthead" class="">
 83 |       <div id="logo"><a href="/"><img src="http://static.arstechnica.net//public/v6/styles/light/images/masthead/logo.png?1335816272" alt="Ars Technica: The Art of Technology" width="110" height="81" /></a></div>      
 84 |     <div id="ebc51ce07629d0e14d2fbc4236e44067" >
 85 | <script type="text/javascript">
 86 |   var pbanner_start = new Date();
 87 |   try {
 88 |     var pbanner = cnp.ad.create(cnp.ad.refreshable, false);
 89 |     //pbanner.addParameter({'dcopt':'ist'});
 90 |     pbanner.addParameterString('kw=top;kw=1859s-great-auroral-stormthe-week-the-sun-touched-the-earth;kw=05;kw=2012;kw=news;kw=science;');
 91 |     pbanner.addParameter({'sz': '728x90' });
 92 |   } catch(e) {}
 93 | </script>
 94 | </div>
 95 |   </div>
 96 |       
 97 | <div id="search-navigation">
 98 |   <div id="search">
 99 |     <a id="search-link" href="http://www.google.com/cse?cx=011835048811694782689:7zpko-isndo">Search</a>
100 |     
101 |     <div class="form">
102 |     <span>Search:</span>
103 |     <form action="http://www.google.com/cse" id="search-form">
104 |       <div>
105 |         <input type="hidden" value="011835048811694782689:7zpko-isndo" name="cx"/>
106 |         <input type="hidden" value="UTF-8" name="ie"/>
107 |         <input type="text" id="search-form-text" value="" name="q"/>
108 |       </div>
109 |     </form>
110 |     </div>
111 |   </div>
112 |   <div id="navigation">
113 |     <ul id="primary-navigation">
114 |           <li class=""><a href="/">All</a></li>
115 |           <li class="apple"><a href="/apple/">Apple</a></li>
116 |           <li class="ask-ars"><a href="/ask-ars/">Ask Ars</a></li>
117 |           <li class="business"><a href="/business/">Business</a></li>
118 |           <li class="gadgets"><a href="/gadgets/">Gadgets</a></li>
119 |           <li class="gaming"><a href="/gaming/">Gaming</a></li>
120 |           <li class="microsoft"><a href="/microsoft/">Microsoft</a></li>
121 |           <li class="open-source"><a href="/open-source/">Open Source</a></li>
122 |           <li class="science selected"><a href="/science/">Science</a></li>
123 |           <li class="tech-policy"><a href="/tech-policy/">Tech Policy</a></li>
124 |           <li id="primary-navigation-more" style="display:none;">
125 |         More
126 |         <ul >
127 |                   <li><a href="/hardware/">Hardware</a></li>
128 |                   <li><a href="/media/">Media</a></li>
129 |                   <li><a href="/security/">Security</a></li>
130 |                   <li><a href="/software/">Software</a></li>
131 |                   <li><a href="/staff/">Staff</a></li>
132 |                   <li><a href="/telecom/">Telecom</a></li>
133 |                   <li><a href="/web/">Web</a></li>
134 |                   <li><a href="/site/tv.ars" title="Ars Technica TV">Ars.TV</a></li>
135 |           <li style="padding:0;"><span style="display:inline;background-color: #920404; padding: 3px; color:white; -webkit-border-radius: 4px;">New</span> <a style="display:inline;" href="/getting-it-done" title="Getting IT Done">Getting IT Done</a></li>
136 |         </ul>
137 |       </li>
138 |     </ul>
139 |    
140 |     <ul id="secondary-navigation" class="science">
141 |                             <li class="news selected"><a href="/science/news/">News</a></li>
142 |                                           <li class="guides"><a href="/science/guides/">Guides</a></li>
143 |                                           <li class="reviews"><a href="/science/reviews/">Reviews</a></li>
144 |                       </ul>
145 |     <ul id="auxiliary-navigation">
146 |         <li class="subscribe"><a href="/subscriptions/">Upgrade to a Premier Subscription</a>
147 |          
148 |     </li>
149 |     <li class="customize" style="display:none;">
150 |       <a href="#">Customize ▾</a>
151 |       <ul>
152 |         <li>
153 |           <p>Site Theme:</p>
154 |           <label><input type="radio" checked="checked" value="light.css" class="site-style" name="site-style" /> White</label>
155 |           <label><input type="radio" value="dark.css" class="site-style" name="site-style" /> Black</label>
156 |         </li>
157 |         <li>
158 |           <p>Choose body font:</p>
159 |           <label><input type="radio" checked="checked" value="arial" class="body_font" name="body_font" /> Arial</label>
160 |           <label><input type="radio" value="helvetica" class="body_font" name="body_font" /> Helvetica</label>
161 |         </li>
162 |       </ul>
163 |     </li>
164 |     
165 |     <li class="openforum"><a href="http://arstechnica.com/civis/">OpenForum</a></li>
166 |     
167 |           <li class="login-join"><a href="/civis/ucp.php?mode=login&amp;return_to=http%3A%2F%2Farstechnica.com%2Fscience%2Fnews%2F2012%2F05%2F1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars">Login/Join</a></li>
168 |           </ul>
169 |   </div>
170 | </div>
171 |       
172 |                   <div id="main">
173 |         
174 | <div id="silo-header" class="">
175 |   <h1 class="science"><a href="/science/" title="Go to Nobel Intent">Nobel Intent</a></h1>
176 | </div>
177 |         
178 |         <div id="content" class="normal">   <div id="content-inner">
179 |              <div id="story">
180 |                         <h2 class="title">1859's "Great Auroral Storm"&#8212;the week the Sun touched the earth</h2>
181 |         <div class="byline"><span class="author">By <a rel="author" href="/author/matthew-lasar/">Matthew Lasar</a>
182 | </span> | <span class="posted"><span class="published"><span class="name">Published </span> <abbr class="timeago datetime" title="2012-05-03T1:00:00Z">May 2, 2012 8:00 PM</abbr></span></span></div>
183 |         
184 |       <div class="story-image CenteredImage" style="width:640px;">  
185 |   <img width="640" src="http://static.arstechnica.net/assets/2012/04/magnetogram2-4f9c6a2-intro-thumb-640xauto-33524.jpg" alt="1859's &quot;Great Auroral Storm&quot;&amp;#8212;the week the Sun touched the earth" /> 
186 |   
187 |     <div class="news-item-figure-caption"><div class="news-item-figure-caption-text">A magnetogram recorded at the Greenwich Observatory in London during the Carrington Event of 1859.</div></div>
188 |   </div>
189 | 
190 |         <div id="" class="body" style="">          
191 |         
192 |         <!--body--><p>Noon approached on September 1, 1859, and British astronomer <a href="http://en.wikipedia.org/wiki/Richard_Christopher_Carrington">Richard Christopher Carrington</a> was busy with his favorite pastime: tracking sunspots, those huge regions of the star darkened by shifts in its magnetic field. He projected the Sun's image from his viewing device onto a plate of glass stained a "pale straw colour," which gave him a picture of the fiery globe one inch shy of a foot in diameter.</p>
193 | 
194 | <p>The morning's work went as normal. Carrington patiently counted and charted spots, time-lining changes in their positions with a chronometer. Then he saw something unusual.</p><!--page 1-->
195 | 
196 | <div style="width: 300px;" class="news-item-figure ImageRight"><div class="news-item-figure-image" style="height: 169px;"><img src="http://static.arstechnica.net/2012/04/28/carrington-4f9c68d-listing.jpg" height="169" alt="Richard Carrington's 1859 drawing of the solar flares he identified while sunspot watching. The intensely bright patches are marked A and B." width="300" /></div><div class="news-item-figure-caption"><div class="news-item-figure-caption-text">Richard Carrington's 1859 drawing of the solar flares he identified while sunspot watching. The "intensely bright" patches are marked A and B.</div><div class="news-item-figure-caption-byline"><a href="http://articles.adsabs.harvard.edu//full/seri/MNRAS/0020//0000013.000.html">SAO/Nasa Astrophysics</a></div></div></div>
197 | 
198 | <p>"Two patches of intensely bright and white light broke out," <a href="http://adsbit.harvard.edu/cgi-bin/nph-iarticle_query?bibcode=1859MNRAS..20...13C">he later wrote</a>. Carrington puzzled over the flashes. "My first impression was that by some chance a ray of light had penetrated a hole in the screen attached to the object-glass," he explained, given that "the brilliancy was fully equal to that of direct sun-light."</p>
199 | <p>The astronomer checked his gear. He moved the apparatus around a bit. To his surprise, the intense white patches stayed put. Realizing that he was an "unprepared witness of a very different affair," Carrington ran out of his studios to find a second observer. But when he brought this person back, he was "mortified to find" that the bright sections were "already much changed and enfeebled."</p>
200 | <p>"Very shortly afterwards the last trace was gone," Carrington wrote. He kept watch on the region for another hour, but saw nothing more. Meanwhile, the explosive energy that he had seen rushed towards him and everyone else on earth.</p>
201 | 
202 | <h3>Better than batteries</h3>
203 | 
204 | 
205 | <p>It hit quickly. Twelve hours after Carrington's discovery and a continent away, "We were high up on the Rocky Mountains sleeping in the open air," wrote a correspondent to the <em>Rocky Mountain News</em>. "A little after midnight we were awakened by the auroral light, so bright that one could easily read common print." As the sky brightened further, some of the party began making breakfast on the mistaken assumption that dawn had arrived.</p>
206 | 
207 | <p>Across the United States and Europe, telegraph operators struggled to keep service going as the electromagnetic gusts enveloped the globe. In 1859, the US telegraph system was about 20 years old, and <a href="http://arstechnica.com/telecom/news/2010/02/the-lost-souls-of-telecommunications-history.ars/2">Cyrus Field</a> had just built his transatlantic cable from Newfoundland to Ireland, which would not succeed in transmitting messages until after the American Civil War. </p>
208 | <p>"Never in my experience of fifteen years in working telegraph lines have I witnessed anything like the extraordinary 
209 | effect of the Aurora Borealis between Quebec and Farther Point last night," wrote one telegraph manager to the <em>Rochester Union &amp; Advertiser</em> on August 30:</p>
210 | <blockquote>
211 |   <p>The line was in most perfect order, and well skilled operators worked incessantly from 8 o'clock last evening till one this morning to get over in an intelligible form four hundred words of the report per steamer Indian for the Associated Press, and at the latter hour so completely were the wires under the influence of the Aurora Borealis that it was found utterly impossible to communicate between the telegraph stations, and the line had to be closed.</p>
212 | </blockquote>
213 | <p>But if the following newspaper transcript of a telegraph operator exchange between Portland and Boston is to be believed, some plucky telegraphers improvised, letting the storm do the work that their disrupted batteries couldn't:</p>
214 | 
215 | 
216 | <blockquote>
217 |   <p>Boston operator, (to Portland operator) - "Please cut off your battery entirely from the line for fifteen minutes." </p>
218 |    <p> Portland operator - "Will do so. It is now disconnected."</p>
219 |   <p>  Boston - "Mine is disconnected, and we are working with the auroral current. How do you receive my writing?"</p>
220 |   <p>  Portland - "Better than with our batteries on. Current comes and goes gradually."</p>
221 |   <p>  Boston - "My current is very strong at times, and we can work better without the batteries, as the Aurora  seems to neutralize and augment our batteries alternately, making current too strong at times for our relay magnets. </p>
222 |   <p>  Suppose we work without batteries while we are affected by this trouble."</p>
223 |  <p>   Portland - "Very well. Shall I go ahead with business?"</p>
224 | <p>    Boston - "Yes. Go ahead."</p>
225 | </blockquote>
226 | 
227 | 
228 | <p>Telegraphers around the US reported similar experiences. "The wire was then worked for about two hours without the usual batteries on the auroral current, working better than with the batteries connected," said the <em>Washington Daily National Intelligencer</em>. "Who now will dispute the theory that the Aurora Borealis is caused by electricity?" asked the <em>Washington Evening Star</em>.</p>
229 | 
230 | 
231 | <div style="width: 550px;" class="news-item-figure CenteredImage"><div class="news-item-figure-image" style="height: 319px;"><img src="http://static.arstechnica.net/2012/04/28/magnetogram1-4f9c6be-intro.jpg" height="319" alt="A magnetogram recorded at the Greenwich Observatory in London during the Carrington Event of 1859. The lower line (D) represents compass direction; the upper line (H) represents magnetic field horizontal force." width="550" /></div><div class="news-item-figure-caption"><div class="news-item-figure-caption-text">A magnetogram recorded at the Greenwich Observatory in London during the Carrington Event of 1859. The lower line (D) represents compass direction; the upper line (H) represents horizontal force.</div><div class="news-item-figure-caption-byline"><a href="http://www.geomag.bgs.ac.uk/education/carrington.html">British Geological Survey</a></div></div></div>
232 | 
233 | <p>Working with solar-powered telegraph lines sometimes proved to be risky, however. As the sky filled with light, East Coast telegrapher Frederic Royce struggled to get his messages to Richmond, Virginia, his hand resting on the iron plate of his gear. Distracted by the displays, he also leaned on the system's "sounder," which indicated by audio whether the circuit was connected or not. At the same time, his forehead touched its ground wire. </p>
234 | <p>"Immediately, I received a very severe electric shock, which stunned me for an instant," Royce wrote to <em>The New York Times</em>. "An old man who was sitting facing me, and but a few feet distant, said that he saw a spark of fire jump from my forehead." </p>
235 | 
236 | <p>The night of Carrington's discovery, the electrical hurricane that had swept the globe peaked. The Great Auroral Storm had actually begun several days earlier with a similar incident on August 28, but it was Carrington and another astronomer, Richard Hodgson, who identified one of the solar flares that enveloped the earth in a week-long magnetic maelstrom. Because of their work, the episode was dubbed the "Carrington Event," and it consumed the world's attention for the week.</p>
237 | 
238 | 
239 | 
240 | <h3>A livid red flame</h3>
241 | 
242 | 
243 | 
244 | <div style="width: 640px;" class="news-item-figure CenteredImage"><div class="news-item-figure-image" style="height: 425px;"><img src="http://static.arstechnica.net/2012/04/28/churchauroraborealis-4f9c6ef-intro.jpg" height="425" alt="Frederic Edwin Church's 1859 painting Aurora Borealis. Some speculate that Church took his inspiration from the Great Auroral Storm of 1859." width="640" /></div><div class="news-item-figure-caption"><div class="news-item-figure-caption-text">Frederic Edwin Church's 1865 painting "Aurora Borealis." Some speculate that Church took his inspiration from the Great Auroral Storm of 1859.</div><div class="news-item-figure-caption-byline"><a href="http://en.wikipedia.org/wiki/File:Frederic_Edwin_Church_Aurora_Borealis.jpg">Wikipedia</a></div></div></div>
245 | 
246 | 
247 | 
248 | <p>In New York City, San Francisco, Boston, and Chicago,  thousands of sky gazers wandered about the midnight streets,  astounded at what they could see. "Crowds of people gathered at the street corners, admiring and commenting upon the singular spectacle," observed the <em>New Orleans Daily Picayune</em>. When the September 1 aurora "was at its greatest brilliancy, the northern heavens were perfectly illuminated," wrote a reporter for <em>The New York Times.</em> He continued:</p>
249 | 
250 | <blockquote>
251 |   <p>At that time almost the whole southern heavens were in a livid red flame, brightest still in the southeast and southwest. Streamers of yellow and orange shot up and met and crossed each other, like the bayonets upon a stack of guns, in the open space between the constellations Aries, Taurus and the Head of Medusa&#8212;about 15 degrees south of the zenith. In this manner&#8212;alternating great pillars, rolling cumuli shooting streamers, curdled and wisped and fleecy waves&#8212;rapidly changing its hue from red to orange, orange to yellow, and yellow to white, and back in the same order to brilliant red, the magnificent auroral glory continued its grand and inexplicable movements until the light of morning overpowered to radiance and it was lost in the beams of the rising sun.</p>
252 | </blockquote>
253 | 
254 | <p>Popular descriptions of the spectacle appeared everywhere. In 2006, a team of space scientists assembled <a href="http://www.sciencedirect.com/science/article/pii/S0273117706000160">a collection of eyewitness newspaper accounts</a> of the storm. What stands out in these reports is the astonishment, awe, and even pleasure that the world experienced for a week&#8212;followed by a sobering realization of how close our planet is to its indispensable star. </p>
255 | 
256 | <p>In Cincinnati, the aurora came "like that preceding the rising moon, while in the west a delicate crimson seemed to be thrown upwards, as if from the sun, long since gone down," wrote a journalist for the <em>Cincinnati Daily Commercial.</em></p>
257 | <blockquote>
258 |   <p>Later, these strange fires overran the entire heavens&#8212;now separating into streamers, gathered at the zenith, and forming a glorious canopy&#8212;then spreading evenly like a vapor, shedding on all things a soft radiance; again, across the sky waves of light would flit, like the almost undistinguishable ripple produced by the faintest breeze upon the quiet surface of an inland lake; a pale green would now cover half the firmament from the east, while rich crimson met it from the west&#8212;then the ruddy light would concentrate itself at the zenith, while beneath it fell in folds of beauty the mild purple and green. To the east and to the west lay huge fields of luminous clouds, tinted with a bright rosy flush, wholly unlike that produced by the rising sun and if possible even more beautiful.</p>
259 | </blockquote>
260 | <p>And so it went around the United States and the world. The sky appeared "blood red," noted the <em>New York Herald.</em> The storm produced "a beautiful halo, and at another period it had the effect of falling from the apex in showers of nebulous matter like star-dust," reported the <em>The Hobart Town Mercury</em> from Tasmania.</p>
261 | 
262 | 
263 | <p>Not all reporters welcomed these images with pleasure. "Half-past eleven. The appearance now is positively awful," wrote a horrified correspondent for the <em>San Francisco Chronicle</em> on September 5. "The red glare is over houses, streets, and fields, and the most dreadful of conflagrations could not cast a deeper hue abroad."</p>
264 | <p>Others took a pragmatic approach to the moment. </p>
265 | 
266 | <p>"Singular as it may appear, a gentleman actually killed three birds with a gun yesterday morning about one o'clock [in the morning]," disclosed the <em>New Orleans Times Picayune</em>, "a circumstance which perhaps never had its like before. The birds were killed while the beautiful aurora borealis was at its height, and being a very early species&#8212;larks&#8212;were, no doubt, deceived by the bright appearance of everything, and came forth innocently, supposing it was day." </p>
267 |         
268 |                     <div class="bottom-image-credit">
269 |                                     <a href="http://www.geomag.bgs.ac.uk/education/carrington.html">Photograph by British Geological Survey</a>
270 |                     </div>
271 |                   
272 |         
273 |         
274 |                 
275 |         </div>
276 |         
277 |         
278 |         <!-- Article Pager -->
279 |                             <div class="pager">
280 |             <div class="label">Page:</div>
281 |             <ul>
282 |                                               <li class="selected">1</li>
283 |                                                               <li><a href="/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars/2">2</a></li>
284 |                                           </ul>
285 |             <div class="next-page">
286 |                             <a href="/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars/2">Next &gt;</a>
287 |                           </div>
288 |         </div>
289 |         
290 |     
291 | 
292 |     </div>
293 |     
294 |     <noscript>
295 | <img style="position: absolute; bottom: 0px; right: 0px; width: 1px; height: 1px;" src="http://arstechnica.com/dragons/brains.gif?id=55765&amp;91047084" alt="" />
296 | </noscript>
297 | <script type="text/javascript">
298 | document.write('<img style="position: absolute; bottom: 0px; right: 0px; width: 1px; height: 1px;" src="http://arstechnica.com/dragons/brains.gif?id=55765&amp;' + (parseInt(Math.random()*99999999, 10)).toString() + '" alt="" />');
299 | </script>
300 |     
301 |           
302 | <!--googleoff: all-->
303 | 
304 | <div id="comments-bar" class="with-bubble">
305 |     <h2>User comments</h2>
306 |      
307 |     <div class="comments-link">
308 |     <a name="comments-bar" rel="nofollow" href="/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars?comments=1#comments-bar">Click here to view the 30 comments on this story</a>
309 |     </div>
310 |       </div>
311 | 
312 |   <div id="hiddencomment"></div>
313 | <!--<div id="alert"><p><img src="http://arstechnica.com/civis/images/smilies/flail.gif" /> We're making some updates to the commenting system.  We should have the kinks worked out soon.</p></div>-->
314 | <!--googleon: all-->
315 |               <div id="links-bar">
316 |   <ul>
317 |     
318 |     
319 |         <li class="facebook">
320 |       <iclint src="http://www.facebook.com/plugins/like.php?href=http%3A%2F%2Farstechnica.com%2Fscience%2Fnews%2F2012%2F05%2F1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars&amp;layout=button_count&amp;show_faces=false&amp;width=85&amp;action=like&amp;font=arial&amp;colorscheme=light&amp;height=21" scrolling="no" frameborder="0" style="border:none; overflow:hidden; width:85px; height:21px;" allowTransparency="true"></iclint>
321 |     </li>
322 |         
323 |     <li class="gplus">
324 |     <g:plusone size="medium" href="http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars"></g:plusone>
325 |     </li>
326 |     <li class="linkedin"><script type="IN/Share" data-counter="right"></script></li>
327 |     		<li><a href="http://twitter.com/share" class="twitter-share-button" data-url="http://arst.ch/tmc" data-counturl="http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars" data-count="horizontal" data-via="arstechnica" data-related="matthewlasar:Matthew Lasar">Tweet</a></li>
328 |         
329 |     <li class="reddit">
330 |         <iclint src="http://www.reddit.com/static/button/button1.html?width=120&url=http%3A%2F%2Farstechnica.com%2Fscience%2Fnews%2F2012%2F05%2F1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars&amp;title=1859%27s%20%22Great%20Auroral%20Storm%22%26%238212%3Bthe%20week%20the%20Sun%20touched%20the%20earth&amp;bgcolor=fff&amp;bordercolor=eee" width="120" height="20" scrolling="no" frameborder="0"></iclint>
331 |         </li>
332 |     
333 | 
334 |     
335 |     <li class="share">
336 |       <a class="a2a_dd" href="http://www.addtoany.com/share_save?linkname=1859%27s%20%22Great%20Auroral%20Storm%22%26%238212%3Bthe%20week%20the%20Sun%20touched%20the%20earth&amp;linkurl=http%3A%2F%2Farstechnica.com%2Fscience%2Fnews%2F2012%2F05%2F1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars"><img src="http://static.addtoany.com/buttons/favicon.png" width="16" height="16" border="0" alt="Share/Bookmark" style="display:inline;vertical-align:middle;"/> Share/Email</a>
337 |       <script type="text/javascript">
338 |         var a2a_linkname="1859\'s \"Great Auroral Storm\"&#8212;the week the Sun touched the earth",
339 |         a2a_linkurl="http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars",
340 |         a2a_onclick=1,
341 |         a2a_show_title=1,
342 |         a2a_hide_embeds=0,
343 |         a2a_num_services=8,
344 |         a2a_color_main="989EA3",
345 |         a2a_color_border="989EA3",
346 |         a2a_color_link_text="FF5B00",
347 |         a2a_color_link_text_hover="ffffff",
348 |         a2a_track_links='ga',
349 |         a2a_prioritize= [
350 |           "digg",
351 |           "yahoo_buzz",
352 |           "stumbleupon",
353 |           "instapaper",
354 |           "slashdot",
355 |           "linkedin",
356 |           "delicious",
357 |           "google_reader",
358 |           "tumblr",
359 |           "posterous"
360 |           ];
361 |         var a2a_config = a2a_config || {};
362 |         a2a_config.no_3p = 1;
363 |       </script>
364 |       <style type="text/css">#a2apage_BROWSER { display:none !important; }</style>
365 |     </li>
366 |     <li class="copypasta copy-pasta-button">Make a correction</li>
367 | 
368 |   </ul>
369 | </div>
370 | <script src="http://platform.linkedin.com/in.js" type="text/javascript"></script>
371 |               <!--googleoff: all-->
372 |     <div id="read-more-stories">
373 | <h2>Read more stories</h2>
374 |   <div class="story-navigation">
375 |       <a href="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars" title="Read the previously published article">&lt; Older Story</a> 
376 |     
377 |     
378 |     </div>
379 |   <!--googleoff: all-->
380 |   <script language='JavaScript'>
381 |     var OB_langJS = "http://static.arstechnica.net//public/v6/scripts/outbrain.lang_en_ars.js",OBITm = '1306449288604',OB_raterMode = 'singlethumb',OB_recMode = 'strip',OutbrainPermaLink='http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars';
382 |     if (typeof(OB_Script)!='undefined' ){OutbrainStart();}else{var OB_Script = true,str = unescape("%3Cscript src=\'http://widgets.outbrain.com/OutbrainRater.js\' type=\'text/javascript\'%3E%3C/script%3E");document.write(str);}
383 |   </script>
384 |   <!--googleon: all-->
385 | </div>
386 |     <!--googleon: all-->
387 |     
388 |   </div>   
389 | </div>
390 |         <!--googleoff: all-->
391 | <div id="sidebar">
392 |       
393 | <div id="article-links" class="with-divider" style="display:none;">
394 |   
395 |   <ul>
396 |     <li class="enlarge-text"><a href="#">Increase text size</a></li>
397 |     <li class="shrink-text"><a href="#">Reduce text size</a></li>
398 |         <li class="print"><a href="#">Print this story</a></li>
399 |         
400 |     <li class="comment"><a href="/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars?comments=1#comments-bar#comments-bar">Leave a comment (30)</a></li>
401 |     <li class="copy-pasta-button edit-suggestion" style="display: none;"><a href="#">Make a correction</a></li>
402 |         <li class="shorturl"><a rel="nofollow" href="http://arst.ch/tmc">http://arst.ch/tmc</a></li>
403 |         
404 |   </ul>
405 | 
406 |     <div class="premier">Premier features ( <a href="/subscriptions/">learn more</a> ) :</div>
407 |   <ul>
408 |                   <li class="single-page"><a href="/subscriptions/" class="nonsub">View on single page</a></li>
409 |                   <li class="configure"><a href="/subscriptions/">Configure these features</a></li>
410 |       </ul>
411 |   </div>
412 | 
413 |  
414 |     <div id="tile-top" class="">
415 | 
416 | <noscript>
417 | <div id="help-by-subscribing">
418 | <a href="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars/2"><img src="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars/4" alt="Please subscribe" /></a></div>
419 | </noscript>
420 | 
421 | <script type="text/javascript">
422 | try {
423 |   var ppanel = cnp.ad.create(cnp.ad.refreshable, false);
424 |   ppanel.addParameterString('kw=top;kw=1859s-great-auroral-stormthe-week-the-sun-touched-the-earth;kw=05;kw=2012;kw=news;kw=science;');
425 |   ppanel.addParameter({'sz':'300x250'});
426 |   ppanel.load();
427 | } catch(e) {}
428 | </script>
429 | </div>
430 |   <div id="journals-box" class="with-divider">
431 |   <h2 class="title">Latest Top Stories</h2>
432 |   <ul class="category">
433 |         <li class="all selected">
434 |       <span class="tab-inner">
435 |         <a href="/" title="All">All</a>
436 |       </span>
437 |     </li>
438 |         <li class="apple">
439 |       <span class="tab-inner">
440 |         <a href="/apple/" title="Apple">Apple</a>
441 |       </span>
442 |     </li>
443 |         <li class="gaming">
444 |       <span class="tab-inner">
445 |         <a href="/gaming/" title="Gaming">Gaming</a>
446 |       </span>
447 |     </li>
448 |         <li class="microsoft">
449 |       <span class="tab-inner">
450 |         <a href="/microsoft/" title="Microsoft">Microsoft</a>
451 |       </span>
452 |     </li>
453 |         <li class="gadgets">
454 |       <span class="tab-inner">
455 |         <a href="/gadgets/" title="Gadgets">Gadgets</a>
456 |       </span>
457 |     </li>
458 |         <li class="open-source">
459 |       <span class="tab-inner">
460 |         <a href="/open-source/" title="Open Source">Open Source</a>
461 |       </span>
462 |     </li>
463 |         <li class="business">
464 |       <span class="tab-inner">
465 |         <a href="/business/" title="Business">Business</a>
466 |       </span>
467 |     </li>
468 |         <li class="science">
469 |       <span class="tab-inner">
470 |         <a href="/science/" title="Science">Science</a>
471 |       </span>
472 |     </li>
473 |         <li class="tech-policy">
474 |       <span class="tab-inner">
475 |         <a href="/tech-policy/" title="Tech Policy">Tech Policy</a>
476 |       </span>
477 |     </li>
478 |         <li class="staff">
479 |       <span class="tab-inner">
480 |         <a href="/staff/" title="Staff">Staff</a>
481 |       </span>
482 |     </li>
483 |       </ul>
484 |   <ul class="stories">
485 |     <li id="journal-box-0" class="science">
486 |     <a href="/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars?clicked=related_right">1859's "Great Auroral Storm"&#8212;the week the Sun touched the earth</a>
487 |       </li>
488 |     <li id="journal-box-1" class="gadgets">
489 |     <a href="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars?clicked=related_right">Android users targeted in drive-by download attacks </a>
490 |       </li>
491 |     <li id="journal-box-2" class="getting-it-done">
492 |     <a href="/getting-it-done/2012/05/google-beefs-up-apps-scripts-for-docs.ars?clicked=related_right">Google beefs up Apps Scripts for Docs</a>
493 |       </li>
494 |     <li id="journal-box-3" class="tech-policy">
495 |     <a href="/tech-policy/news/2012/05/furious-judge-decries-blizzard-of-copyright-troll-lawsuits.ars?clicked=related_right">Furious judge decries "blizzard" of copyright troll lawsuits</a>
496 |       </li>
497 |     <li id="journal-box-4" class="microsoft">
498 |     <a href="/microsoft/news/2012/05/windows-8-to-integrate-cloud-services-ditch-windows-live-branding.ars?clicked=related_right">Windows 8 to integrate cloud services, ditch Windows Live branding</a>
499 |       </li>
500 |     <li id="journal-box-5" class="gaming">
501 |     <a href="/gaming/news/2012/05/virtual-shooting-games-may-improve-accuracy-but-wont-make-you-a-sniper.ars?clicked=related_right">Virtual shooting games may improve real-world accuracy, won't make you a sniper</a>
502 |       </li>
503 |     <li id="journal-box-6" class="science">
504 |     <a href="/science/news/2012/05/sequestration-meets-extraction-us-swaps-carbon-dioxide-for-methane-under-the-sea.ars?clicked=related_right">Dept. of Energy extracts hydrocarbons from sea, sequesters CO2 in Davy Jones' locker</a>
505 |       </li>
506 |     <li id="journal-box-7" class="business">
507 |     <a href="/business/news/2012/05/html5-based-meego-successor-tizen-hits-10-milestone.ars?clicked=related_right">HTML5-based MeeGo successor Tizen hits 1.0 milestone</a>
508 |       </li>
509 |     <li id="journal-box-8" class="tech-policy">
510 |     <a href="/tech-policy/news/2012/05/eus-top-court-apis-cant-be-copyrighted-would-monopolise-ideas.ars?clicked=related_right">EU's top court: APIs can't be copyrighted, would "monopolise ideas"</a>
511 |       </li>
512 |     <li id="journal-box-9" class="science">
513 |     <a href="/science/news/2012/05/usdas-wildlife-services-program-kills-50000-harmless-animals.ars?clicked=related_right">USDA's Wildlife Services program reportedly kills 50,000 harmless animals</a>
514 |       </li>
515 |     <li id="journal-box-10" class="business">
516 |     <a href="/business/news/2012/05/amazon-funding-original-comedy-childrens-television-for-instant-streaming.ars?clicked=related_right">Amazon wants original comedy, children's television ideas for Instant streaming</a>
517 |       </li>
518 |     <li id="journal-box-11" class="apple">
519 |     <a href="/apple/guides/2012/05/creating-videos-with-51-audio-for-the-apple-tv-mac-and-ios.ars?clicked=related_right">Creating videos with 5.1 audio for the Apple TV, Mac, and iOS</a>
520 |       </li>
521 |     <li id="journal-box-12" class="business">
522 |     <a href="/business/news/2012/05/european-privacy-police-say-google-duped-them-over-street-view.ars?clicked=related_right">European privacy police say Google "duped" them over Street View </a>
523 |       </li>
524 |     <li id="journal-box-13" class="science">
525 |     <a href="/science/news/2012/05/a-burp-of-light-from-a-black-hole-reveals-that-it-ate-a-star.ars?clicked=related_right">A burp of light from a black hole reveals that it ate a star</a>
526 |       </li>
527 |     <li id="journal-box-14" class="science">
528 |     <a href="/science/news/2012/05/research-finally-published-showing-how-bird-flu-virus-can-spread-among-mammals.ars?clicked=related_right">Research finally published showing how bird flu virus can spread among mammals</a>
529 |       </li>
530 |     </ul>
531 | </div>
532 |     <div class="with-divider" id="fb">
533 |     <iclint src="http://www.facebook.com/plugins/likebox.php?href=http%3A%2F%2Ffacebook.com%2Farstechnica&amp;width=300&amp;colorscheme=light&amp;show_faces=false&amp;stream=false&amp;header=false&amp;height=62&amp;border_color=%23FFFFFF" scrolling="no" frameborder="0" style="border:none; overflow:hidden; width:300px; height:62px;" allowTransparency="true"></iclint>
534 |     <iclint src="http://www.facebook.com/plugins/activity.php?site=arstechnica.com&amp;width=300&amp;height=370&amp;header=false&amp;colorscheme=light&amp;recommendations=false&amp;border_color=%23FFFFFF" scrolling="no" frameborder="0" style="border:none; overflow:hidden; width:300px; height:370px;" allowTransparency="true"></iclint>
535 | 
536 |       <p><a href="#" class="anonymous">Disable Facebook on Ars</a></p>
537 |       </div>
538 |     <div id="tile-bottom" class="">
539 | 
540 | <noscript>
541 | <div id="help-by-subscribing">
542 | <a href="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars/2"><img src="/gadgets/news/2012/05/android-users-targeted-for-the-first-time-in-drive-by-download-attacks.ars/4" alt="Please subscribe" /></a></div>
543 | </noscript>
544 | 
545 | <script type="text/javascript">
546 | try {
547 |   var ppanel = cnp.ad.create(cnp.ad.refreshable, false);
548 |   ppanel.addParameterString('kw=bottom;kw=1859s-great-auroral-stormthe-week-the-sun-touched-the-earth;kw=05;kw=2012;kw=news;kw=science;');
549 |   ppanel.addParameter({'sz':'300x250'});
550 |   ppanel.load();
551 | } catch(e) {}
552 | </script>
553 | </div>
554 |   <div id="jobs-ars"><a href="http://jobs.arstechnica.com/"><img src="http://static.arstechnica.com//public/v6/styles/dark/images/sidebar/monster-job-board-300x250b.jpg" alt="Ars Job Board Powered by Monster" /></a></div>
555 | 
556 | </div>
557 | <!--googleon: all-->
558 |       </div>
559 |       <div id="footer">
560 |         <div id="slogan">Serving the technologist for <span id="decades">1</span> &#x00d7; 10<sup>-1</sup> centuries</div>
561 |         <iframe src="http://static.arstechnica.net//public/v6/footer.html?1335816272" frameborder="0" scrolling="no" width="1000" height="350"></iframe>
562 |       </div>
563 |     </div> 
564 |     <script type="text/javascript">
565 |   var _gaq = _gaq || [];
566 |   _gaq.push(['_setAccount', 'UA-31997-1']);
567 |   _gaq.push(['_trackPageview']);
568 |   _gaq.push(['_trackPageLoadTime']);
569 |   (function() {
570 |     var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
571 |     ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
572 |     var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
573 |   })();
574 | 
575 | </script>
576 | 
577 | 
578 | <script type="text/javascript">
579 |   var page_class = 'individual',
580 |     site_root = "",
581 |     site_root_rel = '/',
582 |         discussion_url = "",
583 |     entry_author = {
584 |           "matthew lasar":true,
585 |       "matthew lasar":true,
586 |       "":true
587 |         },
588 |     entry_id = 55765,
589 |         fp_layout = 'normal',
590 |     syntaxhighlighter = "http://static.arstechnica.com/public/full/scripts/syntaxhighlighter.js",
591 |     new_comments = true,
592 |     disable_fb = 'false';
593 | </script>
594 |    
595 | 
596 |   <script src="http://static.arstechnica.net//public/v6/scripts/site.min.js?1335816272" type="text/javascript" charset="utf-8"></script>
597 | 
598 | <noscript>
599 | <img src="http://b.scorecardresearch.com/b?c1=2&c2=6035094&c3=&c4=&c5=&c6=&c15=&cv=1.3&cj=1" style="position:absolute; bottom: 0px; right:0px;"
600 | width="1" height="1" alt="" />
601 | </noscript>
602 | 
603 | <span style="display: none" id="ArsTechnicaNews" class="hslice">
604 |   <span style="display: none" class="entry-title">Ars Technica News</span>
605 |   <a style="display: none" href="http://www.ieaddons.com/en/ie8slice/Content.ashx?id=330" rel="entry-content"></a>
606 | </span>
607 | 
608 | <script type="text/javascript">
609 |   var s_account="conde-arstechnica";
610 |   var s_linkInternalFilters="javascript:,arstechnica.com";
611 |   var s_trackingServer="stats2.arstechnica.com";
612 |   var s_trackingServerSecure='sstats.arstechnica.com';
613 | </script>
614 | 
615 | <script type="text/javascript" src="http://static.arstechnica.net/public/shared/omniture/s_code.js"></script>
616 | <script type="text/javascript">
617 |   s.pageName = "http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars";
618 |   s.prop1 = "";
619 |   s.prop2 = "";
620 |   s.prop5 = "blog";
621 |   s.events = "";
622 |   
623 | s.prop6 = "science"
624 | s.prop7 = "science/news"
625 | s.prop8 = "science/news/2012"
626 | s.prop9 = "science/news/2012/05"
627 | s.prop3 = "Science";
628 | s.hier1 = "science,news,2012,05";
629 |   var s_code=s.t();
630 |   if(s_code) document.write(s_code);
631 | </script>
632 | 
633 | <script type="text/javascript"> if(navigator.appVersion.indexOf('MSIE')>=0)document.write(unescape('%3C')+'\!-'+'-')</script>
634 | <noscript><a href="http://www.omniture.com" title="Web Analytics"><img src="http://condenast.112.2o7.net/b/ss/condenet-dev/1/H.15.1--NS/0" height="1" width="1" border="0" alt="" /></a></noscript>
635 | 
636 | <!-- START Parse.ly Include: Standard -->
637 | <div id="parsely-root" style="display: none">
638 |   <a id="parsely-cfg" data-parsely-site="arstechnica.com"
639 |        href="http://parsely.com">Powered by the Parse.ly Publisher Platform (P3).</a>
640 | </div>
641 | <script>
642 | (function(s, p, d) {
643 |     var h=d.location.protocol, i=p+"-"+s,
644 |         e=d.getElementById(i), r=d.getElementById(p+"-root");
645 |     if (e) return;
646 |     e = d.createElement(s); e.id = i; e.async = true;
647 |     e.src = h+"//static.parsely.com/p.js"; r.appendChild(e);
648 | })("script", "parsely", document);
649 | </script>
650 | <!-- END Parse.ly Include -->
651 |             <script type='text/javascript'>
652 |         var _sf_async_config={};
653 |         /** CONFIGURATION START **/
654 |         _sf_async_config.uid = 27113;
655 |         _sf_async_config.domain = 'arstechnica.com';
656 |         _sf_async_config.sections = 'Nobel Intent';  //CHANGE THIS
657 |                 _sf_async_config.authors = 'Matthew Lasar';    //CHANGE THIS
658 |                 _sf_async_config.path = "http://arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars";
659 |         _sf_async_config.title = '1859\'s \"Great Auroral Storm\"&#8212;the week the Sun touched the earth';
660 |         /** CONFIGURATION END **/
661 |         (function(){
662 |           function loadChartbeat() {
663 |             window._sf_endpt=(new Date()).getTime();
664 |             var e = document.createElement('script');
665 |             e.setAttribute('language', 'javascript');
666 |             e.setAttribute('type', 'text/javascript');
667 |             e.setAttribute('src',
668 |                (('https:' == document.location.protocol) ? 'https://a248.e.akamai.net/chartbeat.download.akamai.com/102508/' : 'http://static.chartbeat.com/') +
669 |                'js/chartbeat.js');
670 |             document.body.appendChild(e);
671 |           }
672 |           var oldonload = window.onload;
673 |           window.onload = (typeof window.onload != 'function') ?
674 |              loadChartbeat : function() { oldonload(); loadChartbeat(); };
675 |         })();
676 |     </script>
677 |   </body>
678 | </html>
679 | 


--------------------------------------------------------------------------------
/tests/data/articles/zdrojak_automaticke_zabezpeceni.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="cs-CZ" prefix="og: http://ogp.me/ns#">
  3 | <head>
  4 | 	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
  5 | 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
  6 | 	<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
  7 | 	<title>Automatické zabezpečení | Zdroják</title>
  8 | 	<link rel="apple-touch-icon" href="http://www.zdrojak.cz/wp-content/themes/zdrojak/img/touch-icon-iphone.png"/>
  9 | 	<link rel="apple-touch-icon" sizes="72x72" href="http://www.zdrojak.cz/wp-content/themes/zdrojak/img/touch-icon-ipad.png"/>
 10 | 	<link rel="apple-touch-icon" sizes="114x114"
 11 | 	      href="http://www.zdrojak.cz/wp-content/themes/zdrojak/img/touch-icon-iphone-retina.png"/>
 12 | 	<link rel="apple-touch-icon" sizes="144x144"
 13 | 	      href="http://www.zdrojak.cz/wp-content/themes/zdrojak/img/touch-icon-ipad-retina.png"/>
 14 | 	<link rel="apple-touch-icon" sizes="512x512"
 15 | 	      href="http://www.zdrojak.cz/wp-content/themes/zdrojak/img/apple-touch-icon-itunes.png"/>
 16 | 	<meta name="apple-mobile-web-app-title" content="Zdrojak.cz"/>
 17 | 	<link rel="profile" href="http://gmpg.org/xfn/11"/>
 18 | 	<link rel="pingback" href="http://www.zdrojak.cz/xmlrpc.php"/>
 19 | 
 20 | <!-- This site is optimized with the Yoast WordPress SEO plugin v1.4.4 - http://yoast.com/wordpress/seo/ -->
 21 | <meta name="description" content="Nespoléhejte se na to, že do kódu nezapomenete na všechna místa připsat ošetření dat. Snažte se aplikaci navrhnout tak, aby se na nic zapomenout nedalo. Za cenu o něco složitějšího jádra bude veškerý kód, který ho používá, obvykle taky mnohem jednodušší."/>
 22 | <link rel="canonical" href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/" />
 23 | <meta property='og:locale' content='cs_CZ'/>
 24 | <meta property='og:title' content='Automatické zabezpečení - Zdroják'/>
 25 | <meta property='og:description' content='Nespoléhejte se na to, že do kódu nezapomenete na všechna místa připsat ošetření dat. Snažte se aplikaci navrhnout tak, aby se na nic zapomenout nedalo. Za cenu o něco složitějšího jádra bude veškerý kód, který ho používá, obvykle taky mnohem jednodušší.'/>
 26 | <meta property='og:url' content='http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/'/>
 27 | <meta property='og:site_name' content='Zdroják'/>
 28 | <meta property='og:type' content='article'/>
 29 | <meta property='og:image' content='http://www.zdrojak.cz/wp-content/uploads/2008/11/security-122617975844093.png'/>
 30 | <!-- / Yoast WordPress SEO plugin. -->
 31 | 
 32 | <link rel="alternate" type="application/rss+xml" title="Zdroják &raquo; RSS zdroj" href="http://www.zdrojak.cz/feed/" />
 33 | <link rel="alternate" type="application/rss+xml" title="Zdroják &raquo; RSS komentářů" href="http://www.zdrojak.cz/comments/feed/" />
 34 | <link rel="alternate" type="application/rss+xml" title="Zdroják &raquo; RSS komentářů pro Automatické zabezpečení" href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/feed/" />
 35 | <link rel='stylesheet' id='admin-bar-css'  href='http://www.zdrojak.cz/wp-includes/css/admin-bar.min.css?ver=3.5.1' type='text/css' media='all' />
 36 | <link rel='stylesheet' id='kindle-css-css'  href='http://www.zdrojak.cz/wp-content/plugins/omSocialButtons/kindle/kindle.css?ver=3.5.1' type='text/css' media='all' />
 37 | <link rel='stylesheet' id='twentytwelve-style-css'  href='http://www.zdrojak.cz/wp-content/themes/zdrojak/style.css?ver=5f54643' type='text/css' media='all' />
 38 | <script type='text/javascript' src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js?ver=3.5.1'></script>
 39 | <meta name="generator" content="WordPress 3.5.1" />
 40 | <!-- Google Plus -->
 41 | <script type="text/javascript">
 42 |     window.___gcfg = {lang: 'cs'};    (function () {
 43 |         var po = document.createElement('script');
 44 |         po.type = 'text/javascript';
 45 |         po.async = true;
 46 |         po.src = 'https://apis.google.com/js/plusone.js';
 47 |         var s = document.getElementsByTagName('script')[0];
 48 |         s.parentNode.insertBefore(po, s);
 49 |     })();
 50 | </script>
 51 | <!-- Google Plus -->
 52 | 	<style type="text/css">.recentcomments a{display:inline !important;padding:0 !important;margin:0 !important;}</style>
 53 | <style type="text/css" media="print">#wpadminbar { display:none; }</style>
 54 | <style type="text/css" id="custom-background-css">
 55 | body.custom-background { background-color: #e6e6e6; }
 56 | </style>
 57 | 	<link rel="shortcut icon" href="/favicon.ico"/>
 58 | 	<link rel="author" href="/humans.txt"/>
 59 | </head>
 60 | <body class="single single-post postid-3773 single-format-standard admin-bar no-customize-support custom-background">
 61 | <div id="page" class="hfeed site">
 62 | 
 63 | 
 64 | 	<div class="site-top">
 65 | 	<div class="wrapper">
 66 | </div>	<div id="main" class="wrapper">
 67 | 	<div id="primary" class="site-content">
 68 | 		<div id="content" role="main">
 69 | 
 70 | 			<article id="post-3773" class="post-3773 post type-post status-publish format-standard hentry category-ruzne tag-bezpecnost tag-ruzne full">
 71 | 		<header class="entry-header">
 72 | 		<h1 class="entry-title">Automatické zabezpečení</h1>
 73 | 	</header>
 74 | 
 75 | 	<div class="entry-content">
 76 | <p>Úroveň zabezpečení aplikace bych rozdělil do tří úrovní:</p>
 77 | <ol>
 78 | <li>Aplikace zabezpečená není, neošetřuje uživatelské vstupy ani své výstupy.</li>
 79 | <li>Aplikace se o zabezpečení snaží, ale takovým způsobem, že na ně lze zapomenout.</li>
 80 | <li>Aplikace se o zabezpečení stará sama, prakticky se nedá udělat chyba.</li>
 81 | </ol>
 82 | <p>Jak se tyto úrovně projevují v jednotlivých oblastech?</p>
 83 | <h2><a href="http://php.vrana.cz/cross-site-scripting.php">XSS</a></h2>
 84 | <p>Druhou úroveň představuje ruční ošetřování pomocí <a href="http://www.php.net/manual/en/function.htmlspecialchars.php"><kbd>htmlspecialchars</kbd></a>. Třetí úroveň zdánlivě reprezentuje automatické ošetřování v šablonách, např. v <a href="http://doc.nette.org/cs/templating"><strong>Nette Latte</strong></a>. Proč píšu zdánlivě? Problém je v tom, že ošetření se dá obvykle snadno zakázat, např. v Latte pomocí <code>{!$var}</code>. Viděl jsem šablony plné vykřičníků i na místech, kde být neměly. Autor to vysvětlil tak, že psaní <code>{$var}</code> někde způsobovalo problémy, které po přidání vykřičníku zmizely, tak je začal psát všude.</p>
 85 | <pre class="brush: php">&lt;?php
 86 | $safeHtml = $texy-&gt;process($content_texy);
 87 | $content = Html::el()-&gt;setHtml($safeHtml);
 88 | // v šabloně pak můžeme použít {$content}
 89 | ?&gt;</pre>
 90 | <p>Ideální by bylo, když by už samotná metoda <code>process()</code> vracela instanci  <code>Html</code>.</p>
 91 | <div class="social-buttons"><div class="wrapper">	<!-- Facebook -->
 92 | 	<div id="fb-root"></div>
 93 | 	<script>(function(d, s, id) {
 94 | 			var js, fjs = d.getElementsByTagName(s)[0];
 95 | 			if (d.getElementById(id)) return;
 96 | 			js = d.createElement(s); js.id = id;
 97 | 			js.src = "//connect.facebook.net/en_US/all.js#xfbml=1&appId=556615607691233";
 98 | 			fjs.parentNode.insertBefore(js, fjs);
 99 | 		}(document, 'script', 'facebook-jssdk'));</script>
100 | 	<!-- Facebook -->
101 | 
102 | <div class="facebook">
103 | 	<div class="fb-like"
104 | 		 data-href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/" data-width="450" data-layout="button_count"></div>
105 | </div><div class="twitter">
106 | 	<a href="https://twitter.com/share" class="twitter-share-button" data-via="zdrojak" data-url="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/" data-lang="en" data-text="Automatické zabezpečení"></a>
107 | 	<script>!function (d, s, id) {
108 | 			var js, fjs = d.getElementsByTagName(s)[0];
109 | 			if (!d.getElementById(id)) {
110 | 				js = d.createElement(s);
111 | 				js.id = id;
112 | 				js.src = "//platform.twitter.com/widgets.js";
113 | 				fjs.parentNode.insertBefore(js, fjs);
114 | 			}
115 | 		}(document, "script", "twitter-wjs");</script>
116 | </div><div class="google">
117 | 	<div class="g-plusone"
118 | 		 data-href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/" data-width="300" data-size="medium"></div>
119 | </div><div class="kindle">
120 | 	<div class="kindleWidget kindleLight">
121 | 		<img src="https://d1xnn692s7u6t6.cloudfront.net/black-15.png"/><span>Kindle</span>
122 | 	</div>
123 | </div>
124 | </div></div>	<a name="bottom"></a>
125 | 		</div>
126 | 
127 | </article>
128 | 
129 | 
130 | </div>
131 | <tr id="tr-comment-23965" class="comment byuser comment-author-okbob even thread-odd thread-alt depth-1 recent new">
132 | 	<td class="author"><img alt='' src='/wp-content/uploads/avatars/2005/01/okbob.png' class='avatar avatar-16 photo' height='16' width='16' />okbob</td>
133 | 	<th class="comment-title"><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23965">trochu jiný přístup</a></th>
134 | 	<td class="comments-date">
135 | 		<time datetime="2013-01-30T09:29:35+00:00">30.1.2013 v 07:29</time>
136 | 	</td>
137 | </tr><tr id="tr-comment-23966" class="comment odd alt depth-2 recent new">
138 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />Aleš Roubíček</td>
139 | 	<th class="comment-title"><span class="line"></span><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23966">Re: trochu jiný přístup</a></th>
140 | 	<td class="comments-date">
141 | 		<time datetime="2013-01-30T09:37:35+00:00">30.1.2013 v 07:37</time>
142 | 	</td>
143 | </tr><tr id="tr-comment-23967" class="comment even depth-3 recent new">
144 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />Futrál</td>
145 | 	<th class="comment-title"><span class="line"></span><span class="line"></span><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23967">Re: trochu jiný přístup</a></th>
146 | 	<td class="comments-date">
147 | 		<time datetime="2013-01-30T11:23:28+00:00">30.1.2013 v 09:23</time>
148 | 	</td>
149 | </tr></div>
150 | </div>
151 | <tr id="tr-comment-23968" class="comment odd alt depth-2 recent new">
152 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />Futrál</td>
153 | 	<th class="comment-title"><span class="line"></span><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23968">Re: trochu jiný přístup</a></th>
154 | 	<td class="comments-date">
155 | 		<time datetime="2013-01-30T11:24:59+00:00">30.1.2013 v 09:24</time>
156 | 	</td>
157 | </tr></div>
158 | </div>
159 | <tr id="tr-comment-23969" class="comment even thread-even depth-1 recent new">
160 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />Monty</td>
161 | 	<th class="comment-title"><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23969">Jaké ošetření sloupce?</a></th>
162 | 	<td class="comments-date">
163 | 		<time datetime="2013-01-30T14:30:19+00:00">30.1.2013 ve 12:30</time>
164 | 	</td>
165 | </tr><tr id="tr-comment-23971" class="comment byuser comment-author-jakub-vrana bypostauthor odd alt depth-2 recent new">
166 | 	<td class="author"><img alt='' src='/wp-content/uploads/avatars/2004/12/jakubvrana.jpg' class='avatar avatar-16 photo' height='16' width='16' />Jakub Vrána</td>
167 | 	<th class="comment-title"><span class="line"></span><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23971">Re: Jaké ošetření sloupce?</a></th>
168 | 	<td class="comments-date">
169 | 		<time datetime="2013-01-30T20:14:25+00:00">30.1.2013 ve 18:14</time>
170 | 	</td>
171 | </tr></div>
172 | </div>
173 | <tr id="tr-comment-23973" class="comment even thread-odd thread-alt depth-1 recent new">
174 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />bene</td>
175 | 	<th class="comment-title"><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23973">Re: Automatické zabezpečení</a></th>
176 | 	<td class="comments-date">
177 | 		<time datetime="2013-01-31T08:58:47+00:00">31.1.2013 v 06:58</time>
178 | 	</td>
179 | </tr></div>
180 | <tr id="tr-comment-23974" class="comment odd alt thread-even depth-1 recent new">
181 | 	<td class="author"><img alt='' src='http://0.gravatar.com/avatar/ad516503a11cd5ca435acc9bb6523536?s=16' class='avatar avatar-16 photo avatar-default' height='16' width='16' />5o</td>
182 | 	<th class="comment-title"><a href="http://www.zdrojak.cz/clanky/automaticke-zabezpeceni/?show=comments#comment-23974">ACL assertion</a></th>
183 | 	<td class="comments-date">
184 | 		<time datetime="2013-01-31T12:04:27+00:00">31.1.2013 v 10:04</time>
185 | 	</td>
186 | </tr></div>
187 | 			</table>
188 | 		</div>
189 | 
190 | 			<div class="comments-actions">
191 | 							<a class="btn add-comment" href="?show=comments#respond">Přidat komentář</a>
192 | 										<a class="btn btn-primary show-comments" href="?show=comments#comments">Zobrazit komentáře</a>
193 | 					</div>
194 | 	</div>				<nav class="nav-single no-print">
195 | 					<h3 class="assistive-text">Navigace pro příspěvky</h3>
196 |             <span class="nav-previous">
197 | 							<a href="http://www.zdrojak.cz/clanky/tvorba-moderniho-eshopu-kategorie-a-parametricke-hledani/" rel="next"><i class="icon-left"></i>Tvorba moderního eshopu: kategorie a parametrické hledání</a>            </span>
198 |             <span class="nav-next">
199 | 	             <a href="http://www.zdrojak.cz/clanky/bezpecny-sandboxovany-iframe/" rel="prev">Bezpečný sandboxovaný iframe<i class="icon-right"></i></a>             </span>
200 | 				</nav>
201 | 
202 | 				<div class="visible-print">Zdroj: http://www.zdrojak.cz/?p=3773</div>
203 | 
204 | 
205 | 		</div>
206 | 
207 | 	</div>
208 | 
209 | 
210 | 			<div id="secondary" class="widget-area" role="complementary">
211 | 						<aside id="recent_category_posts-0" class="with-thumb widget widget_recent_entries">			<h3 class="widget-title">Mobilní vývoj</h3>							<div class="thumbnail">
212 | 					<img width="180" height="180" src="http://www.zdrojak.cz/wp-content/uploads/2013/02/develcz.jpg" class="attachment- wp-post-image" alt="Mobilní vývoj" title="Mobilní vývoj" />				</div>
213 | 						<ul>
214 | 									<li><a href="http://www.zdrojak.cz/clanky/reportaz-z-devel-cz-konference-2013/" title="Reportáž z Devel.cz konference 2013">Reportáž z Devel.cz konference 2013</a></li>
215 | 									<li><a href="http://www.zdrojak.cz/clanky/webapi-firefox-os/" title="Prohlédněte si možnosti WebAPI ve Firefox OS">Prohlédněte si možnosti WebAPI ve Firefox OS</a></li>
216 | 									<li><a href="http://www.zdrojak.cz/clanky/nova-vyvojarska-konzole-v-google-play/" title="Nová Vývojářská konzole v Google Play">Nová Vývojářská konzole v Google Play</a></li>
217 | 							</ul>
218 | 			</aside>					<aside id="recent-posts-0" class="widget widget_recent_entries">		<h3 class="widget-title">Nejnovější příspěvky</h3>		<ul>
219 | 					<li>
220 | 				<a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/" title="Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a>
221 | 						</li>
222 | 					<li>
223 | 				<a href="http://www.zdrojak.cz/clanky/poslete-zdrojak-do-kindlu-novinky/" title="Pošlete Zdroják do Kindlu a další jarní novinky">Pošlete Zdroják do Kindlu a další jarní novinky</a>
224 | 						</li>
225 | 					<li>
226 | 				<a href="http://www.zdrojak.cz/clanky/formulare-v-html5-a-nove-atributy/" title="Formuláře v HTML5 a nové atributy">Formuláře v HTML5 a nové atributy</a>
227 | 						</li>
228 | 					<li>
229 | 				<a href="http://www.zdrojak.cz/clanky/modularni-webove-aplikace-s-minimem-rucni-prace/" title="Modulární webové aplikace s minimem ruční práce">Modulární webové aplikace s minimem ruční práce</a>
230 | 						</li>
231 | 					<li>
232 | 				<a href="http://www.zdrojak.cz/clanky/zapier-propojovani-api/" title="Zapier &#8211; dejte propojování API ten správný šmrnc">Zapier &#8211; dejte propojování API ten správný šmrnc</a>
233 | 						</li>
234 | 				</ul>
235 | 		</aside><aside id="related_posts-0" class="widget widget_related_posts"><h3 class="widget-title">Související články</h3><ul><li><a href="http://www.zdrojak.cz/clanky/do-hlubin-implementaci-javascriptu-1-dil-uvod/" rel="bookmark">Do hlubin implementací JavaScriptu: 1. díl &#8211; úvod</a></li><li><a href="http://www.zdrojak.cz/clanky/uvodni-analyza-pro-moderni-e-shop/" rel="bookmark">Úvodní analýza pro moderní e-shop</a></li><li><a href="http://www.zdrojak.cz/clanky/django-databazovy-model/" rel="bookmark">Django: Databázový model</a></li><li><a href="http://www.zdrojak.cz/clanky/dojo-toolkit-pokrocile-techniky/" rel="bookmark">Dojo Toolkit: pokročilé techniky</a></li><li><a href="http://www.zdrojak.cz/clanky/uvod-do-architektury-mvc/" rel="bookmark">Úvod do architektury MVC</a></li></ul></aside><aside id="recent-comments-0" class="widget widget_recent_comments"><h3 class="widget-title">Nejnovější komentáře</h3><ul id="recentcomments"><li class="recentcomments">Martin Kozák u <a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/?show=comments#comment-24239">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a></li><li class="recentcomments">maras u <a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/?show=comments#comment-24238">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a></li><li class="recentcomments">Ladislav Thon u <a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/?show=comments#comment-24237">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a></li><li class="recentcomments">satai u <a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/?show=comments#comment-24236">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a></li><li class="recentcomments">Ladislav Thon u <a href="http://www.zdrojak.cz/clanky/mloc-js-staticke-typovani/?show=comments#comment-24235">Typické! O maďarské konferenci mloc.js a statickém typování při vývoji webových aplikací</a></li></ul></aside>		</div><!-- #secondary -->
236 | 	</div>
237 | <div id="bottom" class="site-bottom no-print">
238 | 	<div class="wrapper">
239 | 		<aside class="follow">
240 | 	<h3>Sledujte</h3>
241 | 
242 | 	<div class="content">
243 | 		<ul class="menu">
244 | 			<li>
245 | 				<a href="/feed" class="rss">
246 | 					<i class="icon-rss"></i>
247 | 					<span class="assistive-text">RSS</span>
248 | 				</a>
249 | 			</li>
250 | 			<li>
251 | 				<a href="/mail" class="mail" title="Zpravodaj">
252 | 					<i class="icon-mail"></i>
253 | 					<span class="assistive-text">Zpravodaj</span>
254 | 				</a>
255 | 			</li>
256 | 			<li>
257 | 				<a target="_blank" href="https://twitter.com/zdrojak" title="@zdrojak" class="twitter">
258 | 					<i class="icon-twitter"></i>
259 | 					<span class="assistive-text">Twitter @zdrojak</span>
260 | 				</a>
261 | 			</li>
262 | 			<li>
263 | 				<a target="_blank" href="https://plus.google.com/101725826130888424314/posts" class="googleplus">
264 | 					<i class="icon-gplus"></i>
265 | 					<span class="assistive-text">Google + Zdroják</span>
266 | 				</a>
267 | 			</li>
268 | 		</ul>
269 | 	</div>
270 | </aside>	</div>
271 | </div>
272 | 
273 | 
274 | </div>
275 | 
276 | <script type='text/javascript' src='http://www.zdrojak.cz/wp-includes/js/admin-bar.min.js?ver=3.5.1'></script>
277 | <script type='text/javascript'>
278 | /* <![CDATA[ */
279 | (function k(){window.$SendToKindle&&window.$SendToKindle.Widget?$SendToKindle.Widget.init({"title":".entry-title","published":".entry-date","content":".post","exclude":".no-kindle"}):setTimeout(k,500);})();
280 | /* ]]> */
281 | </script>
282 | <script type='text/javascript' src='http://d1xnn692s7u6t6.cloudfront.net/widget.js'></script>
283 | <script type='text/javascript' src='http://www.zdrojak.cz/wp-content/themes/zdrojak/js/zdrojak.min.js?ver=5f54643'></script>
284 | 
285 | 	<div id="webstats">
286 | 		<script src="http://c1.navrcholu.cz/code?site=72;t=t1x1" type="text/javascript"></script>
287 | 		<noscript>
288 | 			<div>
289 | 				<img src="http://c1.navrcholu.cz/hit?site=72;t=t1x1;ref=;jss=0" width="1" height="1" alt=""/>
290 | 			</div>
291 | 		</noscript>
292 | 
293 | 		<script type="text/javascript">
294 | 			var _gaq = _gaq || [];
295 | 			_gaq.push(['_setAccount', 'UA-30960355-1']);
296 | 			_gaq.push(['_trackPageview']);
297 | 
298 | 			(function () {
299 | 				var ga = document.createElement('script');
300 | 				ga.type = 'text/javascript';
301 | 				ga.async = true;
302 | 				ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
303 | 				var s = document.getElementsByTagName('script')[0];
304 | 				s.parentNode.insertBefore(ga, s);
305 | 			})();
306 | 		</script>
307 | 	</div>
308 | <!-- d0e88e1c2777b950889c170b9b92adf8 -->
309 | </body>
310 | </html>
311 | 


--------------------------------------------------------------------------------
/tests/data/snippets/annotated_1.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 | <head>
 3 | 	<meta http-equiv="charset" content="utf-8"/>
 4 | 	<title>This is title of document</title>
 5 | </head>
 6 | <body>
 7 | 	<div>Inline text is not so good, but it's here.</div>
 8 | 	<div class="article">
 9 | 		<div class="wrapper">
10 | 			<p>
11 | 				Paragraph is more <em>better</em>.
12 | 				This text is very <strong>pretty</strong> 'cause she's girl.
13 | 			</p>
14 | 			<p>
15 | 				This is not <big>crap</big> so <dfn title="Make me readable">readability</dfn> me :)
16 | 			</p>
17 | 		</div>
18 | 	</div>
19 | 	<div>And some next not so <b>good</b> text.</div>
20 | </body>
21 | </html>
22 | 


--------------------------------------------------------------------------------
/tests/data/snippets/document_absolute_url.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |     <head>
 3 |         <title>Min Document Title</title>
 4 |     </head>
 5 |     <body>
 6 |         <h1>Min Document</h1>
 7 |         <a href="/about.hml">About Us</a>
 8 |         <a href="http://blog.mitechie.com/test.hml">About Us</a>
 9 |         <a href="http://amazon.com/test.hml">Amazon</a>
10 |     </body>
11 | </html>
12 | 


--------------------------------------------------------------------------------
/tests/data/snippets/document_min.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |     <head>
 3 |         <title>Min Document Title</title>
 4 |     </head>
 5 |     <body>
 6 |         <h1>Min Document</h1>
 7 |         <p>Testing content</p>
 8 |         <br /><br />
 9 |         <div>More content.</div>
10 |         <br><br>
11 |         <div>Additional content.</div>
12 |         <div>Final content.</div>
13 |     </body>
14 | </html>
15 | 


--------------------------------------------------------------------------------
/tests/data/snippets/document_no_body.html:
--------------------------------------------------------------------------------
1 | <html>
2 |     <head>
3 |         <title>Bad Document Title</title>
4 |     </head>
5 |     <h1>Bad Document</h1>
6 | </html>
7 | 


--------------------------------------------------------------------------------
/tests/data/snippets/document_only_content.html:
--------------------------------------------------------------------------------
1 | <h1>Bad Document</h1>
2 | <p>Some bad content in a document without proper html</p>
3 | 


--------------------------------------------------------------------------------
/tests/data/snippets/document_scripts.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |     <head>
 3 |         <title>Min Document Title</title>
 4 |         <script src="something.js"></script>
 5 |         <script src="something.js" />
 6 |         <link rel="stylesheet" href="style.css" type="text/css">
 7 |         <style type="text/css">
 8 |             body {
 9 | 
10 |             }
11 |         </style>
12 |     </head>
13 |     <body>
14 |         <h1>Min Document</h1>
15 |         <a href="/about.hml">About Us</a>
16 |         <a href="http://blog.mitechie.com/test.hml">About Us</a>
17 |         <a href="http://amazon.com/test.hml">Amazon</a>
18 |         <div id="footer">
19 |             <script type="text/javascript">
20 |                 // please go away for readability
21 |             </script>
22 |         </div>
23 |     </body>
24 | </html>
25 | 


--------------------------------------------------------------------------------
/tests/data/snippets/h1_and_2_paragraphs.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 | <head>
 3 | 	<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
 4 | 	<title>Paragraphs</title>
 5 | </head>
 6 | <body>
 7 | 	<div>
 8 | 		<h1>Nadpis H1, ktorý chce byť prvý s textom ale predbehol ho "title"</h1>
 9 | 		<p>
10 | 			Toto je prvý odstavec a to je fajn.
11 | 		</p>
12 | 		<p>
13 | 			Tento text je tu aby vyplnil prázdne miesto v srdci súboru.
14 | 			Aj súbory majú predsa city.
15 | 		</p>
16 | 	</div>
17 | </body>
18 | </html>
19 | 


--------------------------------------------------------------------------------
/tests/data/snippets/test_readable_unlikely.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |     <head>
 3 |         <title>Min Document Title</title>
 4 |     </head>
 5 |     <body>
 6 |         <h1>Min Document</h1>
 7 |         <p>Testing content</p>
 8 | 
 9 |         <!-- This is all stuff that should disappear -->
10 |         <div class="comment">Gone</div>
11 |         <div id="disqus">Gone</div>
12 |         <p id="foot">Gone</div>
13 |         <p id="header">Gone</div>
14 |         <p class="header">Gone</div>
15 |         <div id="header">Gone</div>
16 |         <div id="header">Gone</div>
17 | 
18 |         <!-- These have bad and good terms so should stay -->
19 |         <div id="mainfoot">Gone</div>
20 |         <div id="harticleeader">Gone</div>
21 |         <div class="article header">Gone</div>
22 |         <div class="column header">Gone</div>
23 |         <a class="imgBlogpostPermalink">Gone</a>
24 | 
25 |         <!-- And this will stick around for final -->
26 |         <div>Final content.</div>
27 |     </body>
28 | </html>
29 | 


--------------------------------------------------------------------------------
/tests/test_annotated_text.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | from __future__ import (
  4 |     absolute_import,
  5 |     division,
  6 |     print_function,
  7 |     unicode_literals
  8 | )
  9 | 
 10 | from lxml.html import fragment_fromstring, document_fromstring
 11 | from breadability.readable import Article
 12 | from breadability.annotated_text import AnnotatedTextHandler
 13 | from .utils import load_snippet, load_article
 14 | 
 15 | 
 16 | def test_simple_document():
 17 |     dom = fragment_fromstring("<p>This is\n\tsimple\ttext.</p>")
 18 |     annotated_text = AnnotatedTextHandler.parse(dom)
 19 | 
 20 |     assert annotated_text == [
 21 |         (
 22 |             ("This is\nsimple text.", None),
 23 |         ),
 24 |     ]
 25 | 
 26 | 
 27 | def test_empty_paragraph():
 28 |     dom = fragment_fromstring("<div><p>Paragraph <p>\t  \n</div>")
 29 |     annotated_text = AnnotatedTextHandler.parse(dom)
 30 | 
 31 |     assert annotated_text == [
 32 |         (
 33 |             ("Paragraph", None),
 34 |         ),
 35 |     ]
 36 | 
 37 | 
 38 | def test_multiple_paragraphs():
 39 |     dom = fragment_fromstring("<div><p> 1 first<p> 2\tsecond <p>3\rthird   </div>")
 40 |     annotated_text = AnnotatedTextHandler.parse(dom)
 41 | 
 42 |     assert annotated_text == [
 43 |         (
 44 |             ("1 first", None),
 45 |         ),
 46 |         (
 47 |             ("2 second", None),
 48 |         ),
 49 |         (
 50 |             ("3\nthird", None),
 51 |         ),
 52 |     ]
 53 | 
 54 | 
 55 | def test_single_annotation():
 56 |     dom = fragment_fromstring("<div><p> text <em>emphasis</em> <p> last</div>")
 57 |     annotated_text = AnnotatedTextHandler.parse(dom)
 58 | 
 59 |     assert annotated_text == [
 60 |         (
 61 |             ("text", None),
 62 |             ("emphasis", ("em",)),
 63 |         ),
 64 |         (
 65 |             ("last", None),
 66 |         ),
 67 |     ]
 68 | 
 69 | 
 70 | def test_recursive_annotation():
 71 |     dom = fragment_fromstring("<div><p> text <em><i><em>emphasis</em></i></em> <p> last</div>")
 72 |     annotated_text = AnnotatedTextHandler.parse(dom)
 73 | 
 74 |     assert annotated_text == [
 75 |         (
 76 |             ("text", None),
 77 |             ("emphasis", ("em", "i")),
 78 |         ),
 79 |         (
 80 |             ("last", None),
 81 |         ),
 82 |     ]
 83 | 
 84 | 
 85 | def test_annotations_without_explicit_paragraph():
 86 |     dom = fragment_fromstring("<div>text <strong>emphasis</strong>\t<b>hmm</b> </div>")
 87 |     annotated_text = AnnotatedTextHandler.parse(dom)
 88 | 
 89 |     assert annotated_text == [
 90 |         (
 91 |             ("text", None),
 92 |             ("emphasis", ("strong",)),
 93 |             ("hmm", ("b",)),
 94 |         ),
 95 |     ]
 96 | 
 97 | 
 98 | def test_process_paragraph_with_chunked_text():
 99 |     handler = AnnotatedTextHandler()
100 |     paragraph = handler._process_paragraph([
101 |         (" 1", ("b", "del")),
102 |         (" 2", ("b", "del")),
103 |         (" 3", None),
104 |         (" 4", None),
105 |         (" 5", None),
106 |         (" 6", ("em",)),
107 |     ])
108 | 
109 |     assert paragraph == (
110 |         ("1 2", ("b", "del")),
111 |         ("3 4 5", None),
112 |         ("6", ("em",)),
113 |     )
114 | 
115 | 
116 | def test_include_heading():
117 |     dom = document_fromstring(load_snippet("h1_and_2_paragraphs.html"))
118 |     annotated_text = AnnotatedTextHandler.parse(dom.find("body"))
119 | 
120 |     assert annotated_text == [
121 |         (
122 |             ('Nadpis H1, ktorý chce byť prvý s textom ale predbehol ho "title"', ("h1",)),
123 |             ("Toto je prvý odstavec a to je fajn.", None),
124 |         ),
125 |         (
126 |             ("Tento text je tu aby vyplnil prázdne miesto v srdci súboru.\nAj súbory majú predsa city.", None),
127 |         ),
128 |     ]
129 | 
130 | 
131 | def test_real_article():
132 |     article = Article(load_article("zdrojak_automaticke_zabezpeceni.html"))
133 |     annotated_text = article.main_text
134 | 
135 |     assert annotated_text == [
136 |         (
137 |             ("Automatické zabezpečení", ("h1",)),
138 |             ("Úroveň zabezpečení aplikace bych rozdělil do tří úrovní:", None),
139 |         ),
140 |         (
141 |             ("Aplikace zabezpečená není, neošetřuje uživatelské vstupy ani své výstupy.", ("li", "ol")),
142 |             ("Aplikace se o zabezpečení snaží, ale takovým způsobem, že na ně lze zapomenout.", ("li", "ol")),
143 |             ("Aplikace se o zabezpečení stará sama, prakticky se nedá udělat chyba.", ("li", "ol")),
144 |         ),
145 |         (
146 |             ("Jak se tyto úrovně projevují v jednotlivých oblastech?", None),
147 |         ),
148 |         (
149 |             ("XSS", ("a", "h2")),
150 |             ("Druhou úroveň představuje ruční ošetřování pomocí", None),
151 |             ("htmlspecialchars", ("a", "kbd")),
152 |             (". Třetí úroveň zdánlivě reprezentuje automatické ošetřování v šablonách, např. v", None),
153 |             ("Nette Latte", ("a", "strong")),
154 |             (". Proč píšu zdánlivě? Problém je v tom, že ošetření se dá obvykle snadno zakázat, např. v Latte pomocí", None),
155 |             ("{!$var}", ("code",)),
156 |             (". Viděl jsem šablony plné vykřičníků i na místech, kde být neměly. Autor to vysvětlil tak, že psaní", None),
157 |             ("{$var}", ("code",)),
158 |             ("někde způsobovalo problémy, které po přidání vykřičníku zmizely, tak je začal psát všude.", None),
159 |         ),
160 |         (
161 |             ("<?php\n$safeHtml = $texy->process($content_texy);\n$content = Html::el()->setHtml($safeHtml);\n// v šabloně pak můžeme použít {$content}\n?>", ("pre", )),
162 |         ),
163 |         (
164 |             ("Ideální by bylo, když by už samotná metoda", None),
165 |             ("process()", ("code",)),
166 |             ("vracela instanci", None),
167 |             ("Html", ("code",)),
168 |             (".", None),
169 |         ),
170 |     ]
171 | 


--------------------------------------------------------------------------------
/tests/test_articles/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_antipope_org/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_antipope_org/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_antipope_org/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """Test the scoring and parsing of the Blog Post"""
 4 | 
 5 | from __future__ import absolute_import, division, print_function, unicode_literals
 6 | 
 7 | import os
 8 | 
 9 | import pytest
10 | 
11 | from breadability.readable import Article
12 | 
13 | 
14 | @pytest.fixture(scope="module")
15 | def article():
16 |     """Load up the article for us"""
17 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
18 |     with open(article_path) as file:
19 |         return file.read()
20 | 
21 | 
22 | def test_parses(article):
23 |     """Verify we can parse the document."""
24 |     doc = Article(article)
25 | 
26 |     assert 'id="readabilityBody"' in doc.readable
27 | 
28 | 
29 | def test_comments_cleaned(article):
30 |     """The div with the comments should be removed."""
31 |     doc = Article(article)
32 | 
33 |     assert 'class="comments"' not in doc.readable
34 | 
35 | 
36 | def test_beta_removed(article):
37 |     """The id=beta element should be removed
38 | 
39 |     It's link heavy and causing a lot of garbage content. This should be
40 |     removed.
41 | 
42 |     """
43 |     doc = Article(article)
44 | 
45 |     assert 'id="beta"' not in doc.readable
46 | 


--------------------------------------------------------------------------------
/tests/test_articles/test_businessinsider-com/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_businessinsider-com/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_businessinsider-com/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """Test the scoring and parsing of the Blog Post"""
 4 | 
 5 | from __future__ import absolute_import, division, print_function, unicode_literals
 6 | 
 7 | import os
 8 | 
 9 | import pytest
10 | 
11 | from breadability.readable import Article
12 | 
13 | 
14 | @pytest.fixture(scope="module")
15 | def article():
16 |     """Load up the article for us"""
17 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
18 |     with open(article_path) as file:
19 |         return file.read()
20 | 
21 | 
22 | def test_parses(article):
23 |     """Verify we can parse the document."""
24 |     doc = Article(article)
25 | 
26 |     assert 'id="readabilityBody"' in doc.readable
27 | 
28 | 
29 | def test_images_preserved(article):
30 |     """The div with the comments should be removed."""
31 |     doc = Article(article)
32 | 
33 |     assert 'bharath-kumar-a-co-founder-at-pugmarksme-suggests-working-on-a-sunday-late-night.jpg' in doc.readable
34 |     assert 'bryan-guido-hassin-a-university-professor-and-startup-junkie-uses-airplane-days.jpg' in doc.readable
35 | 


--------------------------------------------------------------------------------
/tests/test_articles/test_businessinsider_com/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_businessinsider_com/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_businessinsider_com/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """
 4 | Test the scoring and parsing of the article from URL below:
 5 | http://www.businessinsider.com/tech-ceos-favorite-productivity-hacks-2013-8
 6 | """
 7 | 
 8 | from __future__ import absolute_import, division, print_function, unicode_literals
 9 | 
10 | import os
11 | 
12 | import pytest
13 | 
14 | from breadability.readable import Article
15 | 
16 | 
17 | @pytest.fixture(scope="module")
18 | def article():
19 |     """Load up the article for us"""
20 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
21 |     with open(article_path, "rb") as file:
22 |         return Article(file.read(), "http://www.businessinsider.com/tech-ceos-favorite-productivity-hacks-2013-8")
23 | 
24 | 
25 | def test_parses(article):
26 |     """Verify we can parse the document."""
27 |     assert 'id="readabilityBody"' in article.readable
28 | 
29 | 
30 | def test_images_preserved(article):
31 |     """The div with the comments should be removed."""
32 |     assert 'bharath-kumar-a-co-founder-at-pugmarksme-suggests-working-on-a-sunday-late-night.jpg' in article.readable
33 |     assert 'bryan-guido-hassin-a-university-professor-and-startup-junkie-uses-airplane-days.jpg' in article.readable
34 | 


--------------------------------------------------------------------------------
/tests/test_articles/test_cz_zdrojak_tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_cz_zdrojak_tests/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_cz_zdrojak_tests/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """
 4 | Test the scoring and parsing of the article from URL below:
 5 | http://www.zdrojak.cz/clanky/jeste-k-testovani/
 6 | """
 7 | 
 8 | from __future__ import absolute_import, division, print_function, unicode_literals
 9 | 
10 | import os
11 | 
12 | import pytest
13 | 
14 | from breadability._compat import unicode
15 | from breadability.readable import Article
16 | 
17 | 
18 | @pytest.fixture(scope="module")
19 | def article():
20 |     """Load up the article for us"""
21 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
22 |     with open(article_path, "rb") as file:
23 |         return Article(file.read(), "http://www.zdrojak.cz/clanky/jeste-k-testovani/")
24 | 
25 | 
26 | def test_parses(article):
27 |     """Verify we can parse the document."""
28 |     assert 'id="readabilityBody"' in article.readable
29 | 
30 | 
31 | def test_content_exists(article):
32 |     """Verify that some content exists."""
33 |     assert isinstance(article.readable, unicode)
34 | 
35 |     text = "S automatizovaným testováním kódu (a ve zbytku článku budu mít na mysli právě to) jsem se setkal v několika firmách."
36 |     assert text in article.readable
37 | 
38 |     text = "Ke čtení naleznete mnoho různých materiálů, od teoretických po praktické ukázky."
39 |     assert text in article.readable
40 | 
41 | 
42 | def test_content_does_not_exist(article):
43 |     """Verify we cleaned out some content that shouldn't exist."""
44 |     assert "Pokud vás problematika zajímá, využijte možnosti navštívit školení" not in article.readable
45 | 


--------------------------------------------------------------------------------
/tests/test_articles/test_scripting_com/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_scripting_com/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_scripting_com/article.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html xmlns:scripting2="http://scripting2.com/namespace.html">
	<head>
		<title>Scripting News: Tech press misses Google/Amazon name grab</title>
		<scripting2:navigation prev="http://scripting.com/stories/2012/06/15/podcastOnThursday.html" next="http://scripting.com/stories/2012/06/16/vectorsOnParade.html" parent="" child="" />
		<link rel="alternate" type="application/rss+xml" title="RSS" href="http://scripting.com/rss.xml" />
		<link rel="alternate" type="application/rss+xml" title="RSS/link-blog" href="http://links.scripting.com/rss.xml" />
		<link rel="alternate" type="application/opml+xml" title="blogroll" href="http://scripting.com/misc/blogroll.opml" /> 
		<link rel="alternate" type="application/opml+xml" title="source" href="http://scripting.com/stories/2012/06/15/theTechPressIsOutToLunch.opml" /> 
		<link rel="stylesheet" href="http://scripting.com/stylesheet.css" type="text/css" />
		
	
		<link href="http://static.scripting.com/github/bootstrap2/css/bootstrap.css" rel="stylesheet">
		<link href="http://static.scripting.com/github/bootstrap2/css/prettify.css" rel="stylesheet">
		
		<script src="http://static.scripting.com/github/bootstrap2/js/jquery.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/prettify.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-transition.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-alert.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-modal.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-dropdown.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-scrollspy.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-tab.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-tooltip.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-popover.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-button.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-collapse.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-carousel.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/bootstrap-typeahead.js"></script>
		<script src="http://static.scripting.com/github/bootstrap2/js/application.js"></script>
		
		<script src="http://static.opml.org/scripts.js"></script>
		<link href="http://static.opml.org/styles.css" rel="stylesheet">

		<script src="http://static.opml.org/scripts.js"></script>
		<link href="http://static.opml.org/styles.css" rel="stylesheet">
		<meta name="generator" content="OPML Editor v0.75">
		<script language="javaScript">
	<!--
	function expandCollapse(id,showClassName){
		var imgId = 'img_'+ id;
		if(document.getElementById(id).className == 'hide'){
			document.getElementById(id).className=showClassName;
			document.getElementById(imgId).src='http://scripting.com/mktree/minus.gif';		
			}
		else{
			document.getElementById(id).className='hide';
			document.getElementById(imgId).src='http://scripting.com/mktree/plus.gif';
			}
		}
	-->
</script> 

		<style>
			table th { 
				border: none; 
				padding: 0;
				}
			table td { 
				border: none; 
				padding: 0;
				}
			table td + td {
				border: none; 
				}
			#disqus_thread {
				width: 700px;
				}
			</style>
		</head>
	<body>
		

<script>
	function toggleMenubar () {
		if (document.getElementById ("mainMenuBar").style.display == ''){
			setCookie ("flMenuVisible", "0", 1000)
			document.getElementById ("mainMenuBar").style.display = "none";
			document.getElementById ("wedge").src = 'http://scripting.com/images/2011/12/16/downWedge.gif';		
			}
		else {
			setCookie ("flMenuVisible", "1", 1000)
			document.getElementById("mainMenuBar").style.display = '';
			document.getElementById("wedge").src = 'http://scripting.com/images/2011/12/16/upWedge.gif';
			}
		}
	</script>
<style>
	.divMenubarToggle img {
		position: absolute;
		top: 5px;
		left: 1px;
		z-index:100000;
		}
	</style>
<div class="divMenubarToggle">
	<a title="Click here to show or hide the menubar." onclick="toggleMenubar()"><img src="http://scripting.com/images/2011/12/16/upWedge.gif" id="wedge" width="16" height="16" alt="Click here to show or hide the menubar."></a>
	</div>

<div class="divOpmlMenubar" id="mainMenuBar">
	<div class="topbar-wrapper" style="z-index: 5;">
		<div class="navbar" data-dropdown="dropdown" >
			<div class="navbar-inner">
				<div class="container">
					<a class="brand" href="/"><white>Scripting News</white></a>
					<div class="nav-collapse">
						<ul class="nav">
							<li class="divOpmlMenuItem"><a href="http://scripting.com/">Blog</a></li>
							<li class="divOpmlMenuItem"><a href="http://links.scripting.com/">Linkblog</a></li>
							<li class="divOpmlMenuItem"><a href="http://links.scripting.com/topLinks.html">Top-40</a></li>
							<li class="divOpmlMenuItem"><a href="http://photos.scripting.com/">Photos</a></li>
							<li class="divOpmlMenuItem"><a href="http://threads.scripting.com/">Threads</a></li>
							<li class="divOpmlMenuItem"><a href="http://daveriver.com/">River</a></li>
							<li class="divOpmlMenuItem"><a href="http://davewiner.com/">Dave</a></li>
							</ul>
						</div>
					</div>
				</div>
			</div>
		</div>
	</div>

		<div class="divOpmlWebpage">
			<div class="divOpmlWebpageBody">
				<div class="divScriptingNavigation">
					<p class="crumbTrail">

<a href="http://scripting.com/">Home</a>&nbsp;>&nbsp;
 2 | <a href="http://scripting.com/toc.html">Archive</a>&nbsp;>&nbsp;
 3 | <a href="http://scripting.com/toc.html#y2012">2012</a>&nbsp;>&nbsp;
 4 | <a href="http://scripting.com/2012/06.html">June</a>&nbsp;>&nbsp;
 5 | <a href="http://scripting.com/2012/06/15.html">15</a>
 6 | </p>
					<p class="nextPrev"><a href="http://scripting.com/stories/2012/06/15/podcastOnThursday.html">Previous</a> / <a href="http://scripting.com/stories/2012/06/16/vectorsOnParade.html">Next</a></p>
					 </div>
				<div class="divScriptingStoryBody">
					<div class="storyTitle">Tech press misses Google/Amazon name grab</div>
					<div class="storyByline">By <a href="http://davewiner.com/">Dave Winer</a> on Friday, June 15, 2012 at 5:52 PM.</div>
					<table cellspacing="0" cellpadding="0">
						<tr>
							<td valign="top">
								<div class="storyBody">
									
<div class="divOutlineBody">
 7 | 	<div class="divOutlineList" style="padding-left: 0;">
 8 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14575"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">Amazon and Google have made an audacious grab of namespace on the Internet. As far as I can see there's been no mention of this in the tech press.</span><a name="amazonAndGoogleHaveMadeAnAudaciousGrabOfNamespaceOnTheInternetAsFarAsICanSeeTheresBeenNoMentionOfThisInTheTechPress">&nbsp;</a><a href="#amazonAndGoogleHaveMadeAnAudaciousGrabOfNamespaceOnTheInternetAsFarAsICanSeeTheresBeenNoMentionOfThisInTheTechPress"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
 9 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14583"><a href="javascript:ec('anExampleGoogleDoesn25374','show','http://scripting.com/images/2011/11/29/blank.gif','http://scripting.com/images/2011/11/29/blank.gif');"><img class="expandIcon" id="img_anExampleGoogleDoesn25374" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"></a><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">An example. Google doesn't intend to share .blog and it will only be used to point to Blogger sites. If you have a Tumblr or WordPress blog, you can't have a .blog domain. Here is the <a href="http://gtldresult.icann.org/application-result/applicationstatus/applicationdetails/527">public listing</a> of Google's <a href="http://dropbox.scripting.com/dave/misc/googleBlogApplication.html">application</a>.</span><a name="anExampleGoogleDoesntIntendToShareBlogAndItWillOnlyBeUsedToPointToBloggerSitesIfYouHaveATumblrOrWordpressBlogYouCantHaveABlogDomainHereIsTheAHrefhttpgtldresulticannorgapplicationresultapplicationstatusapplicationdetails527publicListingaOfGooglesAHrefhttpdropboxscriptingcomdavemiscgoogleblogapplicationhtmlapplicationa">&nbsp;</a><a href="#anExampleGoogleDoesntIntendToShareBlogAndItWillOnlyBeUsedToPointToBloggerSitesIfYouHaveATumblrOrWordpressBlogYouCantHaveABlogDomainHereIsTheAHrefhttpgtldresulticannorgapplicationresultapplicationstatusapplicationdetails527publicListingaOfGooglesAHrefhttpdropboxscriptingcomdavemiscgoogleblogapplicationhtmlapplicationa"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
10 | 		<div class="show" id="anExampleGoogleDoesn25374" name="anExampleGoogleDoesn25374">
11 | 			<div class="divOutlineList" style="padding-left: 30px;">
12 | 				<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14593"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;"><i>The purpose of the proposed gTLD, .blog, is to provide a dedicated Internet space where Google can continue to innovate on its Blogger offerings.  The mission of the proposed gTLD is to provide a dedicated domain space in which users can publish blogs.  All registered domains in the .blog gTLD will automatically be delegated to Google DNS servers, which will in turn provide authoritative DNS responses pointing to the Google Blogger service.  The mission of the proposed gTLD is to simplify the Blogger user experience.  Users will be able to publish content on a unique .blog domain (e.g., myname.blog) which will serve as a short and memorable URL for a particular Blogger account. This mission will enhance consumer choice by providing new availability in the second-level domain space, creating new layers of organization on the Internet, improving the Google user experience, and signaling the kind of content available in the domain.</i></span><a name="ithePurposeOfTheProposedGtldBlogIsToProvideADedicatedInternetSpaceWhereGoogleCanContinueToInnovateOnItsBloggerOfferingsTheMissionOfTheProposedGtldIsToProvideADedicatedDomainSpaceInWhichUsersCanPublishBlogsAllRegisteredDomainsInTheBlogGtldWillAutomaticallyBeDelegatedToGoogleDnsServersWhichWillInTurnProvideAuthoritativeDnsResponsesPointingToTheGoogleBloggerServiceTheMissionOfTheProposedGtldIsToSimplifyTheBloggerUserExperienceUsersWillBeAbleToPublishContentOnAUniqueBlogDomainEgMynameblogWhichWillServeAsAShortAndMemorableUrlForAParticularBloggerAccountThisMissionWillEnhanceConsumerChoiceByProvidingNewAvailabilityInTheSecondlevelDomainSpaceCreatingNewLayersOfOrganizationOnTheInternetImprovingTheGoogleUserExperienceAndSignalingTheKindOfContentAvailableInTheDomaini">&nbsp;</a><a href="#ithePurposeOfTheProposedGtldBlogIsToProvideADedicatedInternetSpaceWhereGoogleCanContinueToInnovateOnItsBloggerOfferingsTheMissionOfTheProposedGtldIsToProvideADedicatedDomainSpaceInWhichUsersCanPublishBlogsAllRegisteredDomainsInTheBlogGtldWillAutomaticallyBeDelegatedToGoogleDnsServersWhichWillInTurnProvideAuthoritativeDnsResponsesPointingToTheGoogleBloggerServiceTheMissionOfTheProposedGtldIsToSimplifyTheBloggerUserExperienceUsersWillBeAbleToPublishContentOnAUniqueBlogDomainEgMynameblogWhichWillServeAsAShortAndMemorableUrlForAParticularBloggerAccountThisMissionWillEnhanceConsumerChoiceByProvidingNewAvailabilityInTheSecondlevelDomainSpaceCreatingNewLayersOfOrganizationOnTheInternetImprovingTheGoogleUserExperienceAndSignalingTheKindOfContentAvailableInTheDomaini"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
13 | 				</div>
14 | 			</div>
15 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14584"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">Amazon plans to do the same with .search. So if you have a search site and it's not Amazon's you can't be part of .search. </span><a name="amazonPlansToDoTheSameWithSearchSoIfYouHaveASearchSiteAndItsNotAmazonsYouCantBePartOfSearch">&nbsp;</a><a href="#amazonPlansToDoTheSameWithSearchSoIfYouHaveASearchSiteAndItsNotAmazonsYouCantBePartOfSearch"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
16 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14594"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">Google is going to be exclusive about .cloud. </span><a name="googleIsGoingToBeExclusiveAboutCloud">&nbsp;</a><a href="#googleIsGoingToBeExclusiveAboutCloud"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
17 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14586"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">There are lots more new proposed TLDs like this.</span><a name="thereAreLotsMoreNewProposedTldsLikeThis">&nbsp;</a><a href="#thereAreLotsMoreNewProposedTldsLikeThis"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
18 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14587"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">Seems like a huge story to me. A big surprise. Did you think this is how it would work? I sure didn't.</span><a name="seemsLikeAHugeStoryToMeABigSurpriseDidYouThinkThisIsHowItWouldWorkISureDidnt">&nbsp;</a><a href="#seemsLikeAHugeStoryToMeABigSurpriseDidYouThinkThisIsHowItWouldWorkISureDidnt"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
19 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14578"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;"><a href="https://twitter.com/davewiner/status/213644043740131330">I tweeted this</a>, followed by a pointer to a <a href="http://www.internetnews.me/2012/06/14/big-brands-trying-to-corner-generic-namespaces/">blog post</a> written by Michele Neylon, all before 8AM Eastern this morning. It's now 6PM, and there have been no reports about it in the tech press. It'll be interesting to see when (or if) this becomes a story. </span><a name="aHrefhttpstwittercomdavewinerstatus213644043740131330iTweetedThisaFollowedByAPointerToAAHrefhttpwwwinternetnewsme20120614bigbrandstryingtocornergenericnamespacesblogPostaWrittenByMicheleNeylonAllBefore8amEasternThisMorningItsNow6pmAndThereHaveBeenNoReportsAboutItInTheTechPressItllBeInterestingToSeeWhenOrIfThisBecomesAStory">&nbsp;</a><a href="#aHrefhttpstwittercomdavewinerstatus213644043740131330iTweetedThisaFollowedByAPointerToAAHrefhttpwwwinternetnewsme20120614bigbrandstryingtocornergenericnamespacesblogPostaWrittenByMicheleNeylonAllBefore8amEasternThisMorningItsNow6pmAndThereHaveBeenNoReportsAboutItInTheTechPressItllBeInterestingToSeeWhenOrIfThisBecomesAStory"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
20 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14579"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">Another angle on this, the ICANN people must have known about these applications long before they were made public. How could they continue this process, knowing that is how Google and Amazon interpreted the idea of new TLDs?</span><a name="anotherAngleOnThisTheIcannPeopleMustHaveKnownAboutTheseApplicationsLongBeforeTheyWereMadePublicHowCouldTheyContinueThisProcessKnowingThatIsHowGoogleAndAmazonInterpretedTheIdeaOfNewTlds">&nbsp;</a><a href="#anotherAngleOnThisTheIcannPeopleMustHaveKnownAboutTheseApplicationsLongBeforeTheyWereMadePublicHowCouldTheyContinueThisProcessKnowingThatIsHowGoogleAndAmazonInterpretedTheIdeaOfNewTlds"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
21 | 		<div class="divOutlineItem" style="padding-bottom: 10px;" id="i14576"><img class="expandIcon" src="http://scripting.com/images/2011/11/29/blank.gif" border="0"><span class="spanOutlineText" style="font-family: Georgia; font-size: 18px; line-height: 140%;">BTW, this also happened on Wednesday morning when we had a <a href="http://scripting.com/stories/2012/06/13/twittersLevelPlayingField.html">story here</a>, at 8AM, about a fundamental change in the way Twitter works. It used to have a 140-character limit, but that limit was lifted for Twitter's media partners. A press release ran later in the day. That's when the reports started appearing in the tech press. Even though the story was in their Twitter timelines, and here on Scripting News.</span><a name="btwThisAlsoHappenedOnWednesdayMorningWhenWeHadAAHrefhttpscriptingcomstories20120613twitterslevelplayingfieldhtmlstoryHereaAt8amAboutAFundamentalChangeInTheWayTwitterWorksItUsedToHaveA140characterLimitButThatLimitWasLiftedForTwittersMediaPartnersAPressReleaseRanLaterInTheDayThatsWhenTheReportsStartedAppearingInTheTechPressEvenThoughTheStoryWasInTheirTwitterTimelinesAndHereOnScriptingNews">&nbsp;</a><a href="#btwThisAlsoHappenedOnWednesdayMorningWhenWeHadAAHrefhttpscriptingcomstories20120613twitterslevelplayingfieldhtmlstoryHereaAt8amAboutAFundamentalChangeInTheWayTwitterWorksItUsedToHaveA140characterLimitButThatLimitWasLiftedForTwittersMediaPartnersAPressReleaseRanLaterInTheDayThatsWhenTheReportsStartedAppearingInTheTechPressEvenThoughTheStoryWasInTheirTwitterTimelinesAndHereOnScriptingNews"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a></div>
22 | 		</div>
23 | 	</div>
24 | 
									
									</div>
								</td>
							<td valign="top">
								<div class="scriptingRightSidebar">
									<div class="divTwitterFollowButton">
										<a href="http://scripting.com/rss.xml"><img src="http://www.scripting.com/images/xml.gif" width="36" height="14" border="0" style="float: right;" alt="RSS feed for Scripting News"></a>
										<a href="https://twitter.com/davewiner" class="twitter-follow-button" data-show-count="true" data-show-screen-name="false">Follow @davewiner</a>
										<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0];if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src="//platform.twitter.com/widgets.js";fjs.parentNode.insertBefore(js,fjs);}}(document,"script","twitter-wjs");</script>
										</div>
									<script type="text/javascript" src="http://static.reallysimple.org/users/dave/linkblogxml/miniLinkBlog.js"></script>
									<div class="divScriptingCommunityBadge">
										<center><a href="http://river.scripting.com/"><img src="http://scripting.com/images/2011/11/28/badge.gif" width="200" height="25" border="0" alt="This site contributes to the scripting.com community river."></a></center>
										</div>
									<p><script type="text/javascript" src="http://127.0.0.1:5337/scripting2/editor/controls?username=davewiner&url=http://scripting.com/stories/2012/06/15/theTechPressIsOutToLunch.html"></script></p>
									</div>
								</td>
							</tr>
						</table>
					<div class="scriptingFooter">
						<hr size=1 noshade>&copy; Copyright 1997-2012 Dave Winer. Last update: Friday, June 15, 2012 at 6:27 PM Eastern. Last build: 6/16/2012; 9:03:27 AM. "It's even worse than it appears." 
						<p><a href="http://scripting.com/rss.xml"><img src="http://www.scripting.com/images/xml.gif" width="36" height="14" border="0" alt="RSS feed for Scripting News"></a></p>
						<p><script type="text/javascript" src="http://127.0.0.1:5337/scripting2/editor/controls?username=davewiner&url=http://scripting.com/stories/2012/06/15/theTechPressIsOutToLunch.html"></script></p>
						<p><a href="http://scripting.com/stories/2012/06/15/podcastOnThursday.html">Previous</a> / <a href="http://scripting.com/stories/2012/06/16/vectorsOnParade.html">Next</a></p>
						<script language="JavaScript" type="text/javascript"><!--
							var imageUrl = "http://counters.scripting.com/counters/count.gif";
							var imageTag = "<img src=\"" + imageUrl + "?group=scripting2&referer=" + escape (document.referrer) + "\" height=\"1\" width=\"1\">";
							document.write (imageTag);
							--></script>
					</div>
				</div>
			</div>
		</body>
	</html>


--------------------------------------------------------------------------------
/tests/test_articles/test_scripting_com/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """Test the scoring and parsing of the Article"""
 4 | 
 5 | from __future__ import absolute_import, division, print_function, unicode_literals
 6 | 
 7 | import os
 8 | from operator import attrgetter
 9 | 
10 | import pytest
11 | 
12 | from breadability.readable import Article, check_siblings, prep_article
13 | 
14 | 
15 | @pytest.fixture(scope="module")
16 | def article():
17 |     """Load up the article for us"""
18 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
19 |     with open(article_path) as file:
20 |         return Article(file.read())
21 | 
22 | 
23 | def test_parses(article):
24 |     """Verify we can parse the document."""
25 |     assert 'id="readabilityBody"' in article.readable
26 | 
27 | 
28 | def test_content_exists(article):
29 |     """Verify that some content exists."""
30 |     assert 'Amazon and Google' in article.readable
31 |     assert not 'Linkblog updated' in article.readable
32 |     assert not '#anExampleGoogleDoesntIntendToShareBlogAndItWill' in article.readable
33 | 
34 | 
35 | @pytest.mark.skip("Test fails because of some weird hash.")
36 | def test_candidates(article):
37 |     """Verify we have candidates."""
38 |     # from lxml.etree import tounicode
39 |     found = False
40 |     wanted_hash = '04e46055'
41 | 
42 |     for node in article.candidates.values():
43 |         if node.hash_id == wanted_hash:
44 |             found = node
45 | 
46 |     assert found
47 | 
48 |     # we have the right node, it must be deleted for some reason if it's
49 |     # not still there when we need it to be.
50 |     # Make sure it's not in our to drop list.
51 |     for node in article._should_drop:
52 |         assert node != found.node
53 | 
54 |     by_score = sorted(
55 |         [c for c in article.candidates.values()],
56 |         key=attrgetter('content_score'), reverse=True)
57 |     assert by_score[0].node == found.node
58 | 
59 |     updated_winner = check_siblings(by_score[0], article.candidates)
60 |     updated_winner.node = prep_article(updated_winner.node)
61 | 
62 |     # This article hits up against the img > p conditional filtering
63 |     # because of the many .gif images in the content. We've removed that
64 |     # rule.
65 | 


--------------------------------------------------------------------------------
/tests/test_articles/test_sweetshark/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bookieio/breadability/2285531110262ba09708c8f2054d365f79915995/tests/test_articles/test_sweetshark/__init__.py


--------------------------------------------------------------------------------
/tests/test_articles/test_sweetshark/test_article.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | """
 4 | Test the scoring and parsing of the article from URL below:
 5 | http://sweetshark.livejournal.com/11564.html
 6 | """
 7 | 
 8 | from __future__ import absolute_import, division, print_function, unicode_literals
 9 | 
10 | import os
11 | 
12 | import pytest
13 | 
14 | from breadability.readable import Article
15 | 
16 | 
17 | @pytest.fixture(scope="module")
18 | def article():
19 |     """Load up the article for us"""
20 |     article_path = os.path.join(os.path.dirname(__file__), 'article.html')
21 |     with open(article_path, "rb") as file:
22 |         return Article(file.read(), "http://sweetshark.livejournal.com/11564.html")
23 | 
24 | 
25 | def test_parses(article):
26 |     """Verify we can parse the document."""
27 |     assert 'id="readabilityBody"' in article.readable
28 | 
29 | 
30 | def test_content_after_video(article):
31 |     """The div with the comments should be removed."""
32 |     assert 'Stay hungry, Stay foolish' in article.readable
33 | 


--------------------------------------------------------------------------------
/tests/test_orig_document.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | """Verify we can process html into a document to work off of."""
  4 | 
  5 | from __future__ import absolute_import
  6 | from __future__ import division, print_function, unicode_literals
  7 | 
  8 | from collections import defaultdict
  9 | from breadability._compat import (
 10 |     to_unicode,
 11 |     to_bytes,
 12 |     unicode,
 13 | )
 14 | 
 15 | from breadability.document import (
 16 |     convert_breaks_to_paragraphs,
 17 |     decode_html,
 18 |     OriginalDocument,
 19 | )
 20 | from .utils import load_snippet
 21 | 
 22 | 
 23 | def test_convert_br_tags_to_paragraphs():
 24 |     returned = convert_breaks_to_paragraphs(
 25 |         ("<div>HI<br><br>How are you?<br><br> \t \n  <br>"
 26 |          "Fine\n I guess</div>"))
 27 | 
 28 |     assert returned == "<div>HI</p><p>How are you?</p><p>Fine\n I guess</div>"
 29 | 
 30 | 
 31 | def test_convert_hr_tags_to_paragraphs():
 32 |     returned = convert_breaks_to_paragraphs(
 33 |         "<div>HI<br><br>How are you?<hr/> \t \n  <br>Fine\n I guess</div>")
 34 | 
 35 |     assert returned == "<div>HI</p><p>How are you?</p><p>Fine\n I guess</div>"
 36 | 
 37 | 
 38 | def test_readin_min_document():
 39 |     """Verify we can read in a min html document"""
 40 |     doc = OriginalDocument(load_snippet('document_min.html'))
 41 | 
 42 |     assert to_unicode(doc).startswith('<html>')
 43 |     assert doc.title == 'Min Document Title'
 44 | 
 45 | 
 46 | def test_readin_with_base_url():
 47 |     """Passing a url should update links to be absolute links"""
 48 |     doc = OriginalDocument(
 49 |         load_snippet('document_absolute_url.html'),
 50 |         url="http://blog.mitechie.com/test.html")
 51 | 
 52 |     assert to_unicode(doc).startswith('<html>')
 53 | 
 54 |     # find the links on the page and make sure each one starts with out
 55 |     # base url we told it to use.
 56 |     links = doc.links
 57 |     assert len(links) == 3
 58 |     # we should have two links that start with our blog url
 59 |     # and one link that starts with amazon
 60 |     link_counts = defaultdict(int)
 61 |     for link in links:
 62 |         if link.get('href').startswith('http://blog.mitechie.com'):
 63 |             link_counts['blog'] += 1
 64 |         else:
 65 |             link_counts['other'] += 1
 66 | 
 67 |     assert link_counts['blog'] == 2
 68 |     assert link_counts['other'] == 1
 69 | 
 70 | 
 71 | def test_no_br_allowed():
 72 |     """We convert all <br/> tags to <p> tags"""
 73 |     doc = OriginalDocument(load_snippet('document_min.html'))
 74 | 
 75 |     assert doc.dom.find('.//br') is None
 76 | 
 77 | 
 78 | def test_empty_title():
 79 |     """We convert all <br/> tags to <p> tags"""
 80 |     document = OriginalDocument(
 81 |         "<html><head><title></title></head><body></body></html>")
 82 | 
 83 |     assert document.title == ""
 84 | 
 85 | 
 86 | def test_title_only_with_tags():
 87 |     """We convert all <br/> tags to <p> tags"""
 88 |     document = OriginalDocument(
 89 |         "<html><head><title><em></em></title></head><body></body></html>")
 90 | 
 91 |     assert document.title == ""
 92 | 
 93 | 
 94 | def test_no_title():
 95 |     """We convert all <br/> tags to <p> tags"""
 96 |     document = OriginalDocument("<html><head></head><body></body></html>")
 97 | 
 98 |     assert document.title == ""
 99 | 
100 | 
101 | def test_encoding():
102 |     text = "ľščťžýáíéäúňôůě".encode("iso-8859-2")
103 |     html = decode_html(text)
104 | 
105 |     assert type(html) is unicode
106 | 
107 | 
108 | def test_encoding_short():
109 |     text = to_bytes("ľščťžýáíé")
110 |     html = decode_html(text)
111 | 
112 |     assert type(html) is unicode
113 |     assert html == "ľščťžýáíé"
114 | 


--------------------------------------------------------------------------------
/tests/test_readable.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | from __future__ import absolute_import, division, print_function, unicode_literals
  4 | 
  5 | import pytest
  6 | from lxml.etree import tounicode
  7 | from lxml.html import document_fromstring, fragment_fromstring
  8 | 
  9 | from breadability._compat import to_unicode
 10 | from breadability.readable import (Article, get_class_weight, get_link_density, is_bad_link,
 11 |                                    leaf_div_elements_into_paragraphs, score_candidates, )
 12 | from breadability.scoring import ScoredNode
 13 | from .utils import load_article, load_snippet
 14 | 
 15 | # TestReadableDocument
 16 | """Verify we can process html into a document to work off of."""
 17 | 
 18 | 
 19 | def test_load_doc():
 20 |     """We get back an element tree from our original doc"""
 21 |     doc = Article(load_snippet('document_min.html'))
 22 |     # We get back the document as a div tag currently by default.
 23 | 
 24 |     assert doc.readable_dom.tag == 'div'
 25 | 
 26 | 
 27 | def test_title_loads():
 28 |     """Verify we can fetch the title of the parsed article"""
 29 |     doc = Article(load_snippet('document_min.html'))
 30 | 
 31 |     assert doc._original_document.title == 'Min Document Title'
 32 | 
 33 | 
 34 | def test_doc_no_scripts_styles():
 35 |     """Step #1 remove all scripts from the document"""
 36 |     doc = Article(load_snippet('document_scripts.html'))
 37 |     readable = doc.readable_dom
 38 | 
 39 |     assert readable.findall(".//script") == []
 40 |     assert readable.findall(".//style") == []
 41 |     assert readable.findall(".//link") == []
 42 | 
 43 | 
 44 | def test_find_body_exists():
 45 |     """If the document has a body, we store that as the readable html
 46 | 
 47 |     No sense processing anything other than the body content.
 48 | 
 49 |     """
 50 |     doc = Article(load_snippet('document_min.html'))
 51 | 
 52 |     assert doc.readable_dom.tag == 'div'
 53 |     assert doc.readable_dom.get('id') == 'readabilityBody'
 54 | 
 55 | 
 56 | def test_body_doesnt_exist():
 57 |     """If we can't find a body, then we create one.
 58 | 
 59 |     We build our doc around the rest of the html we parsed.
 60 | 
 61 |     """
 62 |     doc = Article(load_snippet('document_no_body.html'))
 63 | 
 64 |     assert doc.readable_dom.tag == 'div'
 65 |     assert doc.readable_dom.get('id') == 'readabilityBody'
 66 | 
 67 | 
 68 | def test_bare_content():
 69 |     """If the document is just pure content, no html tags we should be ok
 70 | 
 71 |     We build our doc around the rest of the html we parsed.
 72 | 
 73 |     """
 74 |     doc = Article(load_snippet('document_only_content.html'))
 75 | 
 76 |     assert doc.readable_dom.tag == 'div'
 77 |     assert doc.readable_dom.get('id') == 'readabilityBody'
 78 | 
 79 | 
 80 | def test_no_content():
 81 |     """Without content we supply an empty unparsed doc."""
 82 |     doc = Article('')
 83 | 
 84 |     assert doc.readable_dom.tag == 'div'
 85 |     assert doc.readable_dom.get('id') == 'readabilityBody'
 86 |     assert doc.readable_dom.get('class') == 'parsing-error'
 87 | 
 88 | 
 89 | # Test out our cleaning processing we do.
 90 | 
 91 | 
 92 | def test_unlikely_hits():
 93 |     """Verify we wipe out things from our unlikely list."""
 94 |     doc = Article(load_snippet('test_readable_unlikely.html'))
 95 |     readable = doc.readable_dom
 96 |     must_not_appear = [
 97 |         'comment', 'community', 'disqus', 'extra', 'foot',
 98 |         'header', 'menu', 'remark', 'rss', 'shoutbox', 'sidebar',
 99 |         'sponsor', 'ad-break', 'agegate', 'pagination' '', 'pager',
100 |         'popup', 'tweet', 'twitter', 'imgBlogpostPermalink']
101 | 
102 |     want_to_appear = ['and', 'article', 'body', 'column', 'main', 'shadow']
103 | 
104 |     for i in must_not_appear:
105 |         # we cannot find any class or id with this value
106 |         by_class = readable.find_class(i)
107 | 
108 |         for test in by_class:
109 |             # if it's here it cannot have the must not class without the
110 |             # want to appear class
111 |             found = False
112 |             for cls in test.get('class').split():
113 |                 if cls in want_to_appear:
114 |                     found = True
115 |             assert found
116 | 
117 |         by_ids = readable.get_element_by_id(i, False)
118 |         if by_ids is not False:
119 |             found = False
120 |             for ids in test.get('id').split():
121 |                 if ids in want_to_appear:
122 |                     found = True
123 |             assert found
124 | 
125 | 
126 | def test_misused_divs_transform():
127 |     """Verify we replace leaf node divs with p's
128 | 
129 |     They should have the same content, just be a p vs a div
130 | 
131 |     """
132 |     test_html = "<html><body><div>simple</div></body></html>"
133 |     test_doc = document_fromstring(test_html)
134 |     assert tounicode(leaf_div_elements_into_paragraphs(test_doc)) == to_unicode(
135 |         "<html><body><p>simple</p></body></html>"
136 |     )
137 | 
138 |     test_html2 = ('<html><body><div>simple<a href="">link</a>'
139 |                   '</div></body></html>')
140 |     test_doc2 = document_fromstring(test_html2)
141 |     assert tounicode(leaf_div_elements_into_paragraphs(test_doc2)) == to_unicode(
142 |         '<html><body><p>simple<a href="">link</a></p></body></html>'
143 |     )
144 | 
145 | 
146 | def test_dont_transform_div_with_div():
147 |     """Verify that only child <div> element is replaced by <p>."""
148 |     dom = document_fromstring(
149 |         "<html><body><div>text<div>child</div>"
150 |         "aftertext</div></body></html>"
151 |     )
152 | 
153 |     assert tounicode(leaf_div_elements_into_paragraphs(dom)) == to_unicode(
154 |         "<html><body><div>text<p>child</p>"
155 |         "aftertext</div></body></html>"
156 |     )
157 | 
158 | 
159 | def test_bad_links():
160 |     """Some links should just not belong."""
161 |     bad_links = [
162 |         '<a name="amazonAndGoogleHaveMadeAnAudaciousGrabOfNamespaceOnTheInternetAsFarAsICanSeeTheresBeenNoMentionOfThisInTheTechPress">&nbsp;</a>',
163 |         '<a href="#amazonAndGoogleHaveMadeAnAudaciousGrabOfNamespaceOnTheInternetAsFarAsICanSeeTheresBeenNoMentionOfThisInTheTechPress"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a>',
164 |         '<a href="http://scripting.com/stories/2012/06/15/theTechPressIsOutToLunch.html#anExampleGoogleDoesntIntendToShareBlogAndItWillOnlyBeUsedToPointToBloggerSitesIfYouHaveATumblrOrWordpressBlogYouCantHaveABlogDomainHereIsTheAHrefhttpgtldresulticannorgapplicationresultapplicationstatusapplicationdetails527publicListingaOfGooglesAHrefhttpdropboxscriptingcomdavemiscgoogleblogapplicationhtmlapplicationa"><img src="http://scripting.com/images/2001/09/20/sharpPermaLink3.gif" class="imgBlogpostPermalink" width="6" height="9" border="0" alt="permalink"></a>'
165 |     ]
166 | 
167 |     for l in bad_links:
168 |         link = fragment_fromstring(l)
169 |         assert is_bad_link(link)
170 | 
171 | 
172 | # Candidate nodes are scoring containers we use.
173 | 
174 | 
175 | def test_candidate_scores():
176 |     """We should be getting back objects with some scores."""
177 |     fives = ['<div/>']
178 |     threes = ['<pre/>', '<td/>', '<blockquote/>']
179 |     neg_threes = ['<address/>', '<ol/>']
180 |     neg_fives = ['<h1/>', '<h2/>', '<h3/>', '<h4/>']
181 | 
182 |     for n in fives:
183 |         doc = fragment_fromstring(n)
184 |         assert ScoredNode(doc).content_score == 5
185 | 
186 |     for n in threes:
187 |         doc = fragment_fromstring(n)
188 |         assert ScoredNode(doc).content_score == 3
189 | 
190 |     for n in neg_threes:
191 |         doc = fragment_fromstring(n)
192 |         assert ScoredNode(doc).content_score == -3
193 | 
194 |     for n in neg_fives:
195 |         doc = fragment_fromstring(n)
196 |         assert ScoredNode(doc).content_score == -5
197 | 
198 | 
199 | def test_article_enables_candidate_access():
200 |     """Candidates are accessible after document processing."""
201 |     doc = Article(load_article('ars.001.html'))
202 | 
203 |     assert hasattr(doc, 'candidates')
204 | 
205 | 
206 | # Certain ids and classes get us bonus points.
207 | 
208 | 
209 | def test_positive_class():
210 |     """Some classes get us bonus points."""
211 |     node = fragment_fromstring('<p class="article">')
212 |     assert get_class_weight(node) == 25
213 | 
214 | 
215 | def test_positive_ids():
216 |     """Some ids get us bonus points."""
217 |     node = fragment_fromstring('<p id="content">')
218 |     assert get_class_weight(node) == 25
219 | 
220 | 
221 | def test_negative_class():
222 |     """Some classes get us negative points."""
223 |     node = fragment_fromstring('<p class="comment">')
224 |     assert get_class_weight(node) == -25
225 | 
226 | 
227 | def test_negative_ids():
228 |     """Some ids get us negative points."""
229 |     node = fragment_fromstring('<p id="media">')
230 |     assert get_class_weight(node) == -25
231 | 
232 | 
233 | # We take out list of potential nodes and score them up.
234 | 
235 | 
236 | def test_we_get_candidates():
237 |     """Processing candidates should get us a list of nodes to try out."""
238 |     doc = document_fromstring(load_article("ars.001.html"))
239 |     test_nodes = tuple(doc.iter("p", "td", "pre"))
240 |     candidates = score_candidates(test_nodes)
241 | 
242 |     # this might change as we tweak our algorithm, but if it does,
243 |     # it signifies we need to look at what we changed.
244 |     assert len(candidates.keys()) == 37
245 | 
246 |     # one of these should have a decent score
247 |     scores = sorted(c.content_score for c in candidates.values())
248 |     assert scores[-1] > 100
249 | 
250 | 
251 | def test_bonus_score_per_100_chars_in_p():
252 |     """Nodes get 1 point per 100 characters up to max. 3 points."""
253 |     def build_candidates(length):
254 |         html = "<p>%s</p>" % ("c" * length)
255 |         node = fragment_fromstring(html)
256 | 
257 |         return [node]
258 | 
259 |     test_nodes = build_candidates(50)
260 |     candidates = score_candidates(test_nodes)
261 |     pscore_50 = max(c.content_score for c in candidates.values())
262 | 
263 |     test_nodes = build_candidates(100)
264 |     candidates = score_candidates(test_nodes)
265 |     pscore_100 = max(c.content_score for c in candidates.values())
266 | 
267 |     test_nodes = build_candidates(300)
268 |     candidates = score_candidates(test_nodes)
269 |     pscore_300 = max(c.content_score for c in candidates.values())
270 | 
271 |     test_nodes = build_candidates(400)
272 |     candidates = score_candidates(test_nodes)
273 |     pscore_400 = max(c.content_score for c in candidates.values())
274 | 
275 |     assert pscore_50 + 0.5 == pscore_100
276 |     assert pscore_100 + 2.0 == pscore_300
277 |     assert pscore_300 == pscore_400
278 | 
279 | 
280 | # Link density will adjust out candidate scoresself.
281 | 
282 | 
283 | def test_link_density():
284 |     """Test that we get a link density"""
285 |     doc = document_fromstring(load_article('ars.001.html'))
286 |     for node in doc.iter('p', 'td', 'pre'):
287 |         density = get_link_density(node)
288 | 
289 |         # the density must be between 0, 1
290 |         assert density >= 0.0 and density <= 1.0
291 | 
292 | 
293 | # Siblings will be included if their content is related.
294 | 
295 | 
296 | @pytest.mark.skip("Not implemented yet.")
297 | def test_bad_siblings_not_counted():
298 |     raise NotImplementedError()
299 | 
300 | 
301 | @pytest.mark.skip("Not implemented yet.")
302 | def test_good_siblings_counted():
303 |     raise NotImplementedError()
304 | 
305 | 
306 | # TestMainText
307 | 
308 | def test_empty():
309 |     article = Article("")
310 |     annotated_text = article.main_text
311 | 
312 |     assert annotated_text == []
313 | 
314 | 
315 | def test_no_annotations():
316 |     article = Article("<div><p>This is text with no annotations</p></div>")
317 |     annotated_text = article.main_text
318 | 
319 |     assert annotated_text == [(("This is text with no annotations", None),)]
320 | 
321 | 
322 | def test_one_annotation():
323 |     article = Article("<div><p>This is text\r\twith <del>no</del> annotations</p></div>")
324 |     annotated_text = article.main_text
325 | 
326 |     assert annotated_text == [(
327 |         ("This is text\nwith", None),
328 |         ("no", ("del",)),
329 |         ("annotations", None),
330 |     )]
331 | 
332 | 
333 | def test_simple_snippet():
334 |     snippet = Article(load_snippet("annotated_1.html"))
335 |     annotated_text = snippet.main_text
336 | 
337 |     assert annotated_text == [
338 |         (
339 |             ("Paragraph is more", None),
340 |             ("better", ("em",)),
341 |             (".\nThis text is very", None),
342 |             ("pretty", ("strong",)),
343 |             ("'cause she's girl.", None),
344 |         ),
345 |         (
346 |             ("This is not", None),
347 |             ("crap", ("big",)),
348 |             ("so", None),
349 |             ("readability", ("dfn",)),
350 |             ("me :)", None),
351 |         )
352 |     ]
353 | 


--------------------------------------------------------------------------------
/tests/test_scoring.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf8 -*-
  2 | 
  3 | from __future__ import absolute_import, division, print_function, unicode_literals
  4 | 
  5 | import re
  6 | from operator import attrgetter
  7 | 
  8 | from lxml.html import document_fromstring, fragment_fromstring
  9 | 
 10 | from breadability.readable import Article, get_link_density, is_unlikely_node
 11 | from breadability.scoring import (ScoredNode, check_node_attributes, generate_hash_id, get_class_weight,
 12 |                                   score_candidates)
 13 | from .utils import load_snippet
 14 | 
 15 | 
 16 | def test_generate_hash():
 17 |     dom = fragment_fromstring("<div>ľščťžýáí</div>")
 18 |     generate_hash_id(dom)
 19 | 
 20 | 
 21 | def test_hash_from_id_on_exception():
 22 |     generate_hash_id(None)
 23 | 
 24 | 
 25 | def test_different_hashes():
 26 |     dom = fragment_fromstring("<div>ľščťžýáí</div>")
 27 |     hash_dom = generate_hash_id(dom)
 28 |     hash_none = generate_hash_id(None)
 29 | 
 30 |     assert hash_dom != hash_none
 31 | 
 32 | 
 33 | def test_equal_hashes():
 34 |     dom1 = fragment_fromstring("<div>ľščťžýáí</div>")
 35 |     dom2 = fragment_fromstring("<div>ľščťžýáí</div>")
 36 |     hash_dom1 = generate_hash_id(dom1)
 37 |     hash_dom2 = generate_hash_id(dom2)
 38 |     assert hash_dom1 == hash_dom2
 39 | 
 40 |     hash_none1 = generate_hash_id(None)
 41 |     hash_none2 = generate_hash_id(None)
 42 |     assert hash_none1 == hash_none2
 43 | 
 44 | 
 45 | # Verify a node has a class/id in the given set.
 46 | # The idea is that we have sets of known good/bad ids and classes and need
 47 | # to verify the given node does/doesn't have those classes/ids.
 48 | 
 49 | 
 50 | def test_has_class():
 51 |     """Verify that a node has a class in our set."""
 52 |     test_pattern = re.compile('test1|test2', re.I)
 53 |     test_node = fragment_fromstring('<div/>')
 54 |     test_node.set('class', 'test2 comment')
 55 | 
 56 |     assert check_node_attributes(test_pattern, test_node, 'class')
 57 | 
 58 | 
 59 | def test_has_id():
 60 |     """Verify that a node has an id in our set."""
 61 |     test_pattern = re.compile('test1|test2', re.I)
 62 |     test_node = fragment_fromstring('<div/>')
 63 |     test_node.set('id', 'test2')
 64 | 
 65 |     assert check_node_attributes(test_pattern, test_node, 'id')
 66 | 
 67 | 
 68 | def test_lacks_class():
 69 |     """Verify that a node does not have a class in our set."""
 70 |     test_pattern = re.compile('test1|test2', re.I)
 71 |     test_node = fragment_fromstring('<div/>')
 72 |     test_node.set('class', 'test4 comment')
 73 | 
 74 |     assert not check_node_attributes(test_pattern, test_node, 'class')
 75 | 
 76 | 
 77 | def test_lacks_id():
 78 |     """Verify that a node does not have an id in our set."""
 79 |     test_pattern = re.compile('test1|test2', re.I)
 80 |     test_node = fragment_fromstring('<div/>')
 81 |     test_node.set('id', 'test4')
 82 | 
 83 |     assert not check_node_attributes(test_pattern, test_node, 'id')
 84 | 
 85 | 
 86 | # Verify we calc our link density correctly.
 87 | 
 88 | 
 89 | def test_empty_node():
 90 |     """An empty node doesn't have much of a link density"""
 91 |     doc = Article("<div></div>")
 92 | 
 93 |     assert get_link_density(doc.readable_dom) == 0.0
 94 | 
 95 | 
 96 | def test_small_doc_no_links():
 97 |     doc = Article(load_snippet('document_min.html'))
 98 | 
 99 |     assert get_link_density(doc.readable_dom) == 0.0
100 | 
101 | 
102 | def test_several_links():
103 |     """This doc has a 3 links with the majority of content."""
104 |     doc = Article(load_snippet('document_absolute_url.html'))
105 | 
106 |     assert get_link_density(doc.readable_dom) == 22/37
107 | 
108 | 
109 | # Verify we score nodes correctly based on their class/id attributes.
110 | 
111 | 
112 | def test_no_matches_zero():
113 |     """If you don't have the attribute then you get a weight of 0"""
114 |     node = fragment_fromstring("<div></div>")
115 | 
116 |     assert get_class_weight(node) == 0
117 | 
118 | 
119 | def test_id_hits():
120 |     """If the id is in the list then it gets a weight"""
121 |     test_div = '<div id="post">Content</div>'
122 |     node = fragment_fromstring(test_div)
123 | 
124 |     assert get_class_weight(node) == 25
125 | 
126 |     test_div = '<div id="comments">Content</div>'
127 |     node = fragment_fromstring(test_div)
128 | 
129 |     assert get_class_weight(node) == -25
130 | 
131 | 
132 | def test_class_hits():
133 |     """If the class is in the list then it gets a weight"""
134 |     test_div = '<div class="something post">Content</div>'
135 |     node = fragment_fromstring(test_div)
136 |     assert get_class_weight(node) == 25
137 | 
138 |     test_div = '<div class="something comments">Content</div>'
139 |     node = fragment_fromstring(test_div)
140 |     assert get_class_weight(node) == -25
141 | 
142 | 
143 | def test_scores_collide():
144 |     """We might hit both positive and negative scores.
145 | 
146 |     Positive and negative scoring is done independently so it's possible
147 |     to hit both positive and negative scores and cancel each other out.
148 | 
149 |     """
150 |     test_div = '<div id="post" class="something comment">Content</div>'
151 |     node = fragment_fromstring(test_div)
152 |     assert get_class_weight(node) == 0
153 | 
154 |     test_div = '<div id="post" class="post comment">Content</div>'
155 |     node = fragment_fromstring(test_div)
156 |     assert get_class_weight(node) == 25
157 | 
158 | 
159 | def test_scores_only_once():
160 |     """Scoring is not cumulative within a class hit."""
161 |     test_div = '<div class="post main">Content</div>'
162 |     node = fragment_fromstring(test_div)
163 | 
164 |     assert get_class_weight(node) == 25
165 | 
166 | 
167 | # is_unlikely_node should help verify our node is good/bad.
168 | 
169 | 
170 | def test_body_is_always_likely():
171 |     """The body tag is always a likely node."""
172 |     test_div = '<body class="comment"><div>Content</div></body>'
173 |     node = fragment_fromstring(test_div)
174 | 
175 |     assert not is_unlikely_node(node)
176 | 
177 | 
178 | def test_is_unlikely():
179 |     """Keywords in the class/id will make us believe this is unlikely."""
180 |     test_div = '<div class="something comments">Content</div>'
181 |     node = fragment_fromstring(test_div)
182 |     assert is_unlikely_node(node)
183 | 
184 |     test_div = '<div id="comments">Content</div>'
185 |     node = fragment_fromstring(test_div)
186 |     assert is_unlikely_node(node)
187 | 
188 | 
189 | def test_not_unlikely():
190 |     """Suck it double negatives."""
191 |     test_div = '<div id="post">Content</div>'
192 |     node = fragment_fromstring(test_div)
193 |     assert not is_unlikely_node(node)
194 | 
195 |     test_div = '<div class="something post">Content</div>'
196 |     node = fragment_fromstring(test_div)
197 |     assert not is_unlikely_node(node)
198 | 
199 | 
200 | def test_maybe_hits():
201 |     """We've got some maybes that will overrule an unlikely node."""
202 |     test_div = '<div id="comments" class="article">Content</div>'
203 |     node = fragment_fromstring(test_div)
204 |     assert not is_unlikely_node(node)
205 | 
206 | 
207 | # ScoredNodes constructed have initial content_scores, etc.
208 | 
209 | 
210 | def test_hash_id():
211 |     """ScoredNodes have a hash_id based on their content
212 | 
213 |     Since this is based on the html there are chances for collisions, but
214 |     it helps us follow and identify nodes through the scoring process. Two
215 |     identical nodes would score the same, so meh all good.
216 | 
217 |     """
218 |     test_div = '<div id="comments" class="article">Content</div>'
219 |     node = fragment_fromstring(test_div)
220 |     snode = ScoredNode(node)
221 | 
222 |     assert snode.hash_id == 'ffa4c519'
223 | 
224 | 
225 | def test_div_content_score():
226 |     """A div starts out with a score of 5 and modifies from there"""
227 |     test_div = '<div id="" class="">Content</div>'
228 |     node = fragment_fromstring(test_div)
229 |     snode = ScoredNode(node)
230 |     assert snode.content_score == 5
231 | 
232 |     test_div = '<div id="article" class="">Content</div>'
233 |     node = fragment_fromstring(test_div)
234 |     snode = ScoredNode(node)
235 |     assert snode.content_score == 30
236 | 
237 |     test_div = '<div id="comments" class="">Content</div>'
238 |     node = fragment_fromstring(test_div)
239 |     snode = ScoredNode(node)
240 |     assert snode.content_score == -20
241 | 
242 | 
243 | def test_headings_score():
244 |     """Heading tags aren't likely candidates, hurt their scores."""
245 |     test_div = '<h2>Heading</h2>'
246 |     node = fragment_fromstring(test_div)
247 |     snode = ScoredNode(node)
248 | 
249 |     assert snode.content_score == -5
250 | 
251 | 
252 | def test_list_items():
253 |     """Heading tags aren't likely candidates, hurt their scores."""
254 |     test_div = '<li>list item</li>'
255 |     node = fragment_fromstring(test_div)
256 |     snode = ScoredNode(node)
257 |     assert snode.content_score == -3
258 | 
259 | 
260 | # The grand daddy of tests to make sure our scoring works
261 | # Now scoring details will change over time, so the most important thing is
262 | # to make sure candidates come out in the right order, not necessarily how
263 | # they scored. Make sure to keep this in mind while getting tests going.
264 | 
265 | 
266 | def test_simple_candidate_set():
267 |     """Tests a simple case of two candidate nodes"""
268 |     html = """
269 |         <html>
270 |         <body>
271 |             <div class="content">
272 |                 <p>This is a great amount of info</p>
273 |                 <p>And more content <a href="/index">Home</a>
274 |             </div>
275 |             <div class="footer">
276 |                 <p>This is a footer</p>
277 |                 <p>And more content <a href="/index">Home</a>
278 |             </div>
279 |         </body>
280 |         </html>
281 |     """
282 |     dom = document_fromstring(html)
283 |     div_nodes = dom.findall(".//div")
284 | 
285 |     candidates = score_candidates(div_nodes)
286 |     ordered = sorted(
287 |         (c for c in candidates.values()), reverse=True,
288 |         key=attrgetter("content_score"))
289 | 
290 |     assert ordered[0].node.tag == "div"
291 |     assert ordered[0].node.attrib["class"] == "content"
292 |     assert ordered[1].node.tag == "body"
293 |     assert ordered[2].node.tag == "html"
294 |     assert ordered[3].node.tag == "div"
295 |     assert ordered[3].node.attrib["class"] == "footer"
296 | 


--------------------------------------------------------------------------------
/tests/utils.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf8 -*-
 2 | 
 3 | from __future__ import absolute_import
 4 | from __future__ import division, print_function, unicode_literals
 5 | 
 6 | from os.path import abspath, dirname, join
 7 | 
 8 | 
 9 | TEST_DIR = abspath(dirname(__file__))
10 | 
11 | 
12 | def load_snippet(file_name):
13 |     """Helper to fetch in the content of a test snippet."""
14 |     file_path = join(TEST_DIR, "data/snippets", file_name)
15 |     with open(file_path, "rb") as file:
16 |         return file.read()
17 | 
18 | 
19 | def load_article(file_name):
20 |     """Helper to fetch in the content of a test article."""
21 |     file_path = join(TEST_DIR, "data/articles", file_name)
22 |     with open(file_path, "rb") as file:
23 |         return file.read()
24 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | envlist = py27,py34,py35,py36
3 | [testenv]
4 | deps=pytest
5 | commands = {posargs:pytest}
6 | 


--------------------------------------------------------------------------------