├── .gitignore ├── LICENSE ├── README.md ├── ocrtoolkit ├── __init__.py └── parser.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | *.swp 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Center for Responsive Politics 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | OCRToolkit 2 | ========== 3 | 4 | Tools for working with Optical Character Recognition output 5 | 6 | ### Installation 7 | 8 | pip install git+git://github.com/opensecrets/OCRToolkit.git 9 | 10 | 11 | ### Example 12 | 13 | 14 | #### Sample PDF Document 15 | 16 | ![Sample document](http://assets.opensecrets.org/github/sample_doc.png "Sample PDF: Periodic Financial Disclosure from the US House of Representatives") 17 | 18 | 19 | ```python 20 | import numpy as np 21 | import matplotlib.pyplot as plt 22 | from scipy.cluster.vq import kmeans 23 | import os 24 | from ocrtoolkit import parser 25 | ``` 26 | 27 | ```python 28 | # Get the OCRed characters in a certain area 29 | # of a form. Useful for running through and 30 | # seeing if a set of forms are fairly consistant. 31 | boundingBox = {'l':900, 't':400, 'r':1050, 'b':1000} 32 | parser.getCharacters('2000777.xml', boundingBox) 33 | ``` 34 | 35 | ```python 36 | # Run through a directory, pulling out data from xml 37 | # documents matching the regular expression: 38 | # Some Characters, an X, Something that looks like a date. 39 | # What ABBYY calls a "line" or continuous string of text, is 40 | # separated by a pipe for matching purposes. 41 | 42 | regex = r"([^\n\|]+)\|(x)\|(\d\d\/\d\d\/\d\d)" 43 | 44 | 45 | directory = '.' 46 | xLocs = [] 47 | allData = [] 48 | 49 | for fname in os.listdir(directory ): 50 | (locs, data) = parser.parseXML(fname, regex) 51 | 52 | # I want the locations of the X, which 53 | # should appear in position 1 in the locations 54 | # array. Locations correspond to parentheses in 55 | # the expression. The X is in the second group. 56 | xLocs += [l[1] for l in locs] 57 | allData += [[fname] + d for d in data] 58 | 59 | # Now I'm going to try to find where most Xes 60 | # occur by finding the densest parts of the histogram. 61 | centroids = kmeans(np.array(xLocs), 3)[0] 62 | 63 | # Plot the histogram and the centroids. 64 | plt.hist(xLocs, facecolor='g', bins=100) 65 | 66 | for l in centroids: 67 | axvline(x=l, color='r', zorder=0) 68 | 69 | plt.title('Locations of X marks') 70 | plt.axis([600,850, 0, 600]) 71 | plt.grid(False) 72 | plt.show() 73 | 74 | # The plot shows I should probably consider Xes between about 75 | # 680 pixels from the left and 720 pixels from the left 76 | # to be in the first column, then 720 to 760 in the center column 77 | # and more than 760 in the right column. 78 | 79 | # Set Xes to something more meaningful. 80 | for i, loc in enumerate(xLocs): 81 | if loc > 680 and loc <= 720: 82 | allData[i][2] = 'Purchase' 83 | elif loc > 720 and loc <= 760: 84 | allData[i][2] = 'Sale' 85 | elif loc > 760 and loc <= 800: 86 | allData[i][2] = 'Exchange' 87 | 88 | # Print out the data from our example doc. 89 | print [x for x in allData if x[0] == '2000777.xml'] 90 | ``` 91 | 92 | 93 | ![Histogram of X marks](http://assets.opensecrets.org/github/x_mark_hist.png "Histogram of X marks") 94 | 95 | 96 | 97 | 98 | ### Well-Known Text Conversion 99 | 100 | Parse as normal, returning all coordinates. This will return a bounding box for the each line with a match. (Regex ```([^\n])*``` will return the whole document.) 101 | 102 | ```python 103 | (locs, data) = parser.parseXML('someXML.xml', regex, allCoords=True) 104 | pprint([parser.toWellKnownText(x) for x in locs]) 105 | ``` 106 | 107 | Output [WKT](http://en.wikipedia.org/wiki/Well-known_text): 108 | ``` 109 | ['POLYGON ((106 78, 106 1518, 156 1518, 156 78))', 110 | 'POLYGON ((215 1611, 215 1979, 241 1979, 241 1611))', 111 | 'POLYGON ((270 1630, 270 2062, 312 2062, 312 1630))', 112 | 'POLYGON ((249 77, 249 1215, 308 1215, 308 77))', 113 | 'POLYGON ((391 122, 391 173, 411 173, 411 122))', 114 | 'POLYGON ((412 230, 412 1066, 443 1066, 443 230))', 115 | 'POLYGON ((435 1568, 435 2029, 465 2029, 465 1568))', 116 | 'POLYGON ((470 1568, 470 1731, 496 1731, 496 1568))', 117 | . 118 | . 119 | . 120 | 121 | ``` 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /ocrtoolkit/__init__.py: -------------------------------------------------------------------------------- 1 | from . import parser 2 | 3 | -------------------------------------------------------------------------------- /ocrtoolkit/parser.py: -------------------------------------------------------------------------------- 1 | import re 2 | from lxml import etree 3 | import os 4 | import xmltodict 5 | 6 | schema = '{http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml}' 7 | 8 | PAGE = schema + 'page' 9 | BLOCK = schema + 'block' 10 | LINE = schema + 'line' 11 | CHAR = schema + 'charParams' 12 | 13 | def toWellKnownText(coords): 14 | return 'POLYGON ((%s %s, %s %s, %s %s, %s %s))' % tuple([item for sublist in coords for item in sublist]) 15 | 16 | def inside(bb, a): 17 | 18 | if bb['l'] < a['l'] and bb['t'] < a['t'] and bb['r'] > a['r'] and bb['b'] > a['b']: 19 | return True 20 | else: 21 | return False 22 | 23 | 24 | def getCharacters(fname, bb): 25 | charsInside = '' 26 | 27 | f = open(fname, 'r') 28 | root = etree.XML(f.read()) 29 | return xmltodict.parse(f.read()) 30 | 31 | for p in root.iter(PAGE): 32 | for c in p.iter(CHAR): 33 | # Convert attributes to integers 34 | a = {k: int(v) for k, v in c.attrib.items()} 35 | 36 | if inside(bb, a): 37 | charsInside += c.text 38 | charsInside += '\n' 39 | return charsInside 40 | 41 | def parseXML(fname, regex, minLineLen=200, lineSeparator='|', allCoords=False): 42 | regExC = re.compile(regex, re.IGNORECASE) 43 | 44 | f = open(fname, 'r') 45 | root = etree.XML(f.read()) 46 | matches = [] 47 | lefts = [] 48 | locs = [] 49 | line = '' 50 | 51 | for p in root.iter(PAGE): 52 | for b in p.iter(BLOCK): 53 | 54 | lastLeft = -1 55 | for l in b.iter(LINE): 56 | for c in l.iter(CHAR): 57 | left = int(c.attrib['l']) 58 | 59 | if left + minLineLen < lastLeft: 60 | match = regExC.match(line) 61 | if match is not None: 62 | matches.append(list(match.groups())) 63 | # Array of leftmost locations for each match 64 | leftMosts = [lefts[match.start(i+1)] for i, m in enumerate(match.groups())] 65 | if allCoords: 66 | locs.append([(l.attrib['t'], l.attrib['l']), (l.attrib['t'], l.attrib['r']), 67 | (l.attrib['b'], l.attrib['r']), (l.attrib['b'], l.attrib['l'])]) 68 | else: 69 | locs.append(leftMosts) 70 | 71 | lefts = [] 72 | line = '' 73 | 74 | lastLeft = left 75 | 76 | lefts.append(left) 77 | line += c.text 78 | 79 | lefts.append(-1) 80 | line += lineSeparator 81 | 82 | return (locs, matches) 83 | 84 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | OCR Toolkit 3 | ----- 4 | 5 | The OCR Toolkit helps parse data and location information from 6 | ABBYY XML output. 7 | 8 | """ 9 | 10 | from setuptools import find_packages, setup 11 | 12 | 13 | setup( 14 | name='ocrtoolkit', 15 | version='0.1a', 16 | url='http://github.com/opensecrets/', 17 | license='MIT', 18 | author='Alex Byrnes', 19 | author_email='abyrnes@crp.org', 20 | description='Tools for parsing data from PDF OCR output.', 21 | long_description=__doc__, 22 | packages=find_packages(exclude=['tests*']), 23 | include_package_data=True, 24 | zip_safe=False, 25 | platforms='any', 26 | install_requires=['lxml>=2.3.2'], 27 | classifiers=[ 28 | 'Development Status :: 3 - Alpha', 29 | 'Intended Audience :: Developers', 30 | 'License :: OSI Approved :: MIT License', 31 | 'Operating System :: OS Independent', 32 | 'Programming Language :: Python', 33 | 'Topic :: Text Processing :: Markup :: XML', 34 | 'Topic :: Software Development :: Libraries :: Python Modules' 35 | ], 36 | ) 37 | 38 | --------------------------------------------------------------------------------