├── doc
    ├── .gitignore
    ├── Algorithm.pdf
    ├── Makefile
    └── Algorithm.tex
├── requirements.txt
├── README.md
├── .gitignore
├── download-oldweather.py
├── test-otr.py
├── filter-table-pages.py
├── Interactive.ipynb
├── TableRecognition.py
└── LICENSE


/doc/.gitignore:
--------------------------------------------------------------------------------
1 | *.svg
2 | *.pdf
3 | *.aux
4 | 


--------------------------------------------------------------------------------
/doc/Algorithm.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ulikoehler/OTR/HEAD/doc/Algorithm.pdf


--------------------------------------------------------------------------------
/doc/Makefile:
--------------------------------------------------------------------------------
1 | all:
2 | 	pdflatex Algorithm.tex
3 | 	pdf2svg Algorithm.pdf Algorithm.svg
4 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | git+https://github.com/ulikoehler/cv_algorithms.git
2 | git+https://github.com/ulikoehler/UliEngineering.git
3 | networkx
4 | scipy
5 | numpy
6 | toolz
7 | opencv-python
8 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # OTR
 2 | Optical table recognition - recognize tables in scan images using OpenCV.
 3 | 
 4 | OTR uses a raster-based method to recognize tables, even if they have e.g. dashed lines or the page is slightly skewed (such as when scanning a book). **OTR can not be used for tables without a visible raster!**
 5 | 
 6 | # Install
 7 | 
 8 | Install OpenCV for Python3 e.g. `sudo apt-get install python3-opencv` in Ubuntu 18.04.
 9 | 
10 | I recommend to install numpy & scipy from apt if you use a deb-based linux system to speed up the dependency install: `sudo apt-get install python3-scipy python3-numpy`
11 | 
12 | [cv_algorithms](https://github.com/ulikoehler/cv_algorithms) is one of the dependencies. See there for some of the algorithms used in OTR in a reusable form.
13 | 
14 | ```sh
15 | sudo pip3 install -r requirements.txt
16 | ```
17 | 
18 | # Run
19 | 
20 | Get a test image, e.g. google for images like "Old naval log table" and select one with a table. Can't share one here due to copyright but if you know a public domain one, please add it via a pull request.
21 | 
22 | ```sh
23 | python3 test-otr.py <image filename>
24 | ```
25 | 
26 | It's currently only a proof of concept. See [Algorithm.pdf](https://github.com/ulikoehler/OTR/blob/master/doc/Algorithm.pdf) for details on how it works.
27 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | *$py.class
 5 | 
 6 | # C extensions
 7 | *.so
 8 | 
 9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | 
27 | # PyInstaller
28 | #  Usually these files are written by a python script from a template
29 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 | 
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 | 
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 | 
48 | # Translations
49 | *.mo
50 | *.pot
51 | 
52 | # Django stuff:
53 | *.log
54 | local_settings.py
55 | 
56 | # Flask stuff:
57 | instance/
58 | .webassets-cache
59 | 
60 | # Scrapy stuff:
61 | .scrapy
62 | 
63 | # Sphinx documentation
64 | docs/_build/
65 | 
66 | # PyBuilder
67 | target/
68 | 
69 | # IPython Notebook
70 | .ipynb_checkpoints
71 | 
72 | # pyenv
73 | .python-version
74 | 
75 | # celery beat schedule file
76 | celerybeat-schedule
77 | 
78 | # dotenv
79 | .env
80 | 
81 | # virtualenv
82 | venv/
83 | ENV/
84 | 
85 | # Spyder project settings
86 | .spyderproject
87 | 
88 | # Rope project settings
89 | .ropeproject
90 | 
91 | # OTR
92 | Northwind
93 | *.png
94 | 


--------------------------------------------------------------------------------
/download-oldweather.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | """
 3 | Fetch oldweather images for OpenCV tests
 4 | """
 5 | import subprocess
 6 | import argparse
 7 | import os
 8 | 
 9 | if __name__ == "__main__":
10 |     parser = argparse.ArgumentParser()
11 |     parser.add_argument('name')
12 |     parser.add_argument('year')
13 |     args = parser.parse_args()
14 |     name = args.name
15 | 
16 |     if name == "Northwind":  # 1946 - 1948
17 |         url = "https://zooniverse-static.s3.amazonaws.com/old-weather-2015/Cold_Science/Coast_Guard/Northwind_WAG-282_or_WAGB-282_/Northwind-WAG-282-{0}-split/Northwind-WAG-282-{0}-{1:04d}-{2}.JPG"
18 |     elif name == "Northland":  # 1928 - 1930
19 |         url = "https://zooniverse-static.s3.amazonaws.com/old-weather-2015/The_Arctic_Frontier/Coast_Guard/Northland_WPG-49_/Northland-WPG-49-{0}-split/Northland-WPG-49-{0}-{1:04d}-{2}.JPG"
20 |     elif name == "Eastwind": # 1946 - 1948
21 |         url = "https://zooniverse-static.s3.amazonaws.com/old-weather-2015/Cold_Science/Coast_Guard/Eastwind_WAG-279_or_WAGB-279_/eastwind-wag-279-{0}-split/eastwind-wag-279-{0}-{1:04d}-{2}.JPG"
22 |     else:
23 |         raise ValueError("Invalid name: " + name)
24 |     urls = [url.format(args.year, i, 0) for i in range(1000)]
25 |     urls += [url.format(args.year, i, 1) for i in range(1000)]
26 | 
27 |     # Create directory if it does not exist
28 |     if not os.path.isdir(name):
29 |         os.mkdir(name)
30 |     # Use wget for parallel downloading
31 |     subprocess.call(["wget", "-P", name, "-c"] + urls)


--------------------------------------------------------------------------------
/test-otr.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | import cv2
 3 | import numpy as np
 4 | import TableRecognition
 5 | 
 6 | 
 7 | def runOTR(filename):
 8 |     img = cv2.imread(filename, flags=cv2.IMREAD_COLOR)
 9 |     if img is None:
10 |         raise ValueError("File {0} does not exist".format(filename))
11 |     imgGrey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
12 |     imgThresh = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY_INV)[1]
13 |     imgThreshInv = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY)[1]
14 | 
15 |     imgDil = cv2.dilate(imgThresh, np.ones((5, 5), np.uint8))
16 |     imgEro = cv2.erode(imgDil, np.ones((4, 4), np.uint8))
17 | 
18 |     contour_analyzer = TableRecognition.ContourAnalyzer(imgDil)
19 |     # 1st pass (black in algorithm diagram)
20 |     contour_analyzer.filter_contours(min_area=400)
21 |     contour_analyzer.build_graph()
22 |     contour_analyzer.remove_non_table_nodes()
23 |     contour_analyzer.compute_contour_bounding_boxes()
24 |     contour_analyzer.separate_supernode()
25 |     contour_analyzer.find_empty_cells(imgThreshInv)
26 | 
27 |     contour_analyzer.find_corner_clusters()
28 |     contour_analyzer.compute_cell_hulls()
29 |     contour_analyzer.find_fine_table_corners()
30 | 
31 |     # Add missing contours to contour list
32 |     missing_contours = contour_analyzer.compute_filtered_missing_cell_contours()
33 |     contour_analyzer.contours += missing_contours
34 | 
35 |     # 2nd pass (red in algorithm diagram)
36 |     contour_analyzer.compute_contour_bounding_boxes()
37 |     contour_analyzer.find_empty_cells(imgThreshInv)
38 | 
39 |     contour_analyzer.find_corner_clusters()
40 |     contour_analyzer.compute_cell_hulls()
41 |     contour_analyzer.find_fine_table_corners()
42 | 
43 |     # End of 2nd pass. Continue regularly
44 |     contour_analyzer.compute_table_coordinates(5.)
45 | 
46 |     contour_analyzer.draw_table_coord_cell_hulls(img, xscale=.8, yscale=.8)
47 |     return img
48 | 
49 | if __name__ == "__main__":
50 |     import argparse
51 |     parser = argparse.ArgumentParser()
52 |     parser.add_argument('infile', help='The image file to read')
53 |     parser.add_argument('-o', '--outfile', default="out.png", help='The output file.')
54 |     args = parser.parse_args()
55 | 
56 |     img = runOTR(args.infile)
57 | 
58 |     cv2.imwrite(args.outfile, img)
59 | 


--------------------------------------------------------------------------------
/filter-table-pages.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | """
 3 | Utility script that separated images that have a table from those who have none.
 4 | 
 5 | This script creates two directories, <indir>.tables and <indir>.notables
 6 | where both classes of images will be symlinked.
 7 | """
 8 | import cv2
 9 | import os
10 | import os.path
11 | import concurrent.futures
12 | import numpy as np
13 | import TableRecognition
14 | from ansicolor import red, green
15 | 
16 | 
17 | def hasTable(filename, min_fract_area=.2, min_cells=150):
18 |     img = cv2.imread(filename, flags=cv2.IMREAD_COLOR)
19 |     if img is None:
20 |         raise ValueError("File {0} does not exist".format(filename))
21 |     imgGrey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
22 |     imgThresh = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY_INV)[1]
23 |     imgThreshInv = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY)[1]
24 | 
25 |     imgDil = cv2.dilate(imgThresh, np.ones((5, 5), np.uint8))
26 |     imgEro = cv2.erode(imgDil, np.ones((4, 4), np.uint8))
27 | 
28 |     contour_analyzer = TableRecognition.ContourAnalyzer(imgDil)
29 |     # 1st pass (black in algorithm diagram)
30 |     contour_analyzer.filter_contours(min_area=400)
31 |     contour_analyzer.build_graph()
32 |     contour_analyzer.remove_non_table_nodes()
33 |     contour_analyzer.compute_contour_bounding_boxes()
34 |     contour_analyzer.separate_supernode()
35 | 
36 |     return contour_analyzer.does_page_have_valid_table(min_fract_area, min_cells)
37 | 
38 | if __name__ == "__main__":
39 |     import argparse
40 |     parser = argparse.ArgumentParser()
41 |     parser.add_argument('directory', help='The directory containing image files')
42 |     args = parser.parse_args()
43 | 
44 |     # Build list of files to check
45 |     tocheck = []
46 |     for child in os.listdir(args.directory):
47 |         fullpath = os.path.join(args.directory, child)
48 |         if os.path.isfile(fullpath):
49 |             tocheck.append(fullpath)
50 | 
51 |     # Perform parallel checking
52 |     executor = concurrent.futures.ProcessPoolExecutor()
53 |     results = executor.map(hasTable, tocheck)
54 | 
55 |     # Create output directories (with symlinks)
56 |     dirname = args.directory.strip("/")
57 |     tabledir = dirname + ".table"
58 |     notabledir = dirname + ".notable"
59 | 
60 |     if not os.path.isdir(tabledir):
61 |         os.mkdir(tabledir)
62 |     if not os.path.isdir(notabledir):
63 |         os.mkdir(notabledir)
64 | 
65 |     # Iterate over results
66 |     for filename, result in zip(tocheck, results):
67 |         basename = os.path.basename(filename)
68 |         if result:  # Table found
69 |             print(green("Found table in {0}".format(filename), bold=True))
70 |             # Create symlink
71 |             os.symlink(os.path.join("..", filename), os.path.join(tabledir, basename))
72 |         else:  # No table found
73 |             print(red("Did not find table in {0}".format(filename), bold=True))
74 |             # Create symlink
75 |             os.symlink(os.path.join("..", filename), os.path.join(notabledir, basename))
76 | 
77 | 


--------------------------------------------------------------------------------
/doc/Algorithm.tex:
--------------------------------------------------------------------------------
  1 | % tikzpic.tex
  2 | \documentclass[tikz, border=1mm]{standalone}
  3 | \usepackage{siunitx}
  4 | \begin{document}
  5 | \usetikzlibrary{shapes,arrows,shadows,calc,backgrounds,fit,positioning}
  6 | 
  7 | \tikzset{
  8 |   sshadow/.style={opacity=.25, shadow xshift=0.1, shadow yshift=-0.1},
  9 | }
 10 | 
 11 | \begin{tikzpicture}[auto,
 12 |     block/.style ={rectangle, draw=black, thick,
 13 |       text width=25em, text centered, rounded corners,
 14 |       minimum height=4em},
 15 |     shortblock/.style ={rectangle, draw=black, thick,
 16 |       text width=10em, text centered, rounded corners,
 17 |       minimum height=4em},
 18 |     nodegroup/.style ={rectangle, draw=black, thick, rounded corners,
 19 |       minimum height=4em, bottom color=black!55, top color=black!25},
 20 |     red/.style ={bottom color=red!55, top color=red!25, drop shadow={sshadow,color=black!80!white}},
 21 |     green/.style ={bottom color=green!55, top color=green!25, drop shadow={sshadow,color=black!80!white}},
 22 |     blue/.style ={bottom color=blue!55, top color=blue!25, drop shadow={sshadow,color=black!80!white}},
 23 |     yellow/.style ={bottom color=yellow!55, top color=yellow!25, drop shadow={sshadow,color=black!80!white}},
 24 |     purple/.style ={bottom color=purple!55, top color=purple!25, drop shadow={sshadow,color=black!80!white}},
 25 |     line/.style ={thick, -latex', shorten >=0pt},
 26 |     firstpass/.style={draw=black},
 27 |     secondpass/.style={draw=red}]
 28 | 
 29 | \matrix [column sep=10mm,row sep=8mm] {
 30 |   \node [block, red] (a) {Read image from file}; &\\
 31 |   \node [block, red] (b) {Convert to binary B/W using threshold}; &\\
 32 |   \node [block, red] (c) {Morphological preprocessing\\(Dilation \& erosion)}; &\\&\\
 33 | 
 34 |   \node [block, green] (d) {Recognize image contours \& contour hierarchy}; &\\
 35 |   \node [block, green] (e) {Prefilter contours by area \& minimum number of nodes}; &\\
 36 |   \node [block, green] (f) {Build tree from contour hierarchy}; &\\
 37 |   \node [block, green] (g) {Identify coarse table contour (contour with the largest outdegree)}; &\\
 38 |   \node [block, green] (h) {Remove all contours which are not direct subcontours of the coarse table contour};\\
 39 |   \node [block, green] (i) {Computing rotated bounding boxes \& areas for all contours}; &\\
 40 |   \node [block, green] (j) {Remove coarse table contour from contour list \& store separately};
 41 |       & \node [shortblock, yellow] (ra) {Insert missing cell contours into contour list}; \\&\\
 42 | 
 43 |   \node [block, blue] (k) {Find clusters of contour bounding box corners ("nodes") by pairwise euclidean distance \& threshold};
 44 |     & \node [block, yellow] (r) {Filter missing cell contours: Keep only contours with all nodes inside fine table corner-defined polygon}; \\
 45 |   \node [block, blue] (l) {Compute node position by averaging all merged corner coordinates};
 46 |     & \node [block, yellow] (q) {Find contours in binary mask ("missing cell contours")}; \\
 47 |   \node [block, blue] (m) {Compute cell hulls by replacing contour bounding box corners by their respective nodes};
 48 |     & \node [block, yellow] (p) {Draw each table cell with 110\,\% center-referred scale to binary mask in white (mask now contains table space that is not part of a cell)}; &\\
 49 |   \node [block, blue] (n) {Find fine table corners as 4 nodes with min/max X/Y combinations};
 50 |     & \node [block, yellow] (o) {Create white img-sized binary mask and draw polygon defined by all 4 fine table corners in black.}; &\\&\\
 51 | 
 52 |   \node [block, purple] (s) {Compute cell centers by using   cell hull center of gravity}; &\\
 53 |   \node [block, purple] (t) {Compute horizontal and vertical groups (hgroups \& vgroups) of nodes by pairwise distance \& threshold}; &\\
 54 |   \node [block, purple] (u) {Compute transitive closures of preliminary hgroups \& vgroups to obtain final hgroups/vgroups}; &\\
 55 |   \node [block, purple] (v) {Sort hgroups by mean X coordinate and sort vgroups by mean Y coordinate to obtain table coordinates for each cell}; &\\
 56 | };
 57 | 
 58 | % Group boxes & labels
 59 | \begin{scope}[on background layer]
 60 |   % Shift top of group box up & left
 61 |   \coordinate (a') at ($(a.north) + (-3mm,4mm)$);
 62 |   \coordinate (d') at ($(d.north) + (-3mm,4mm)$);
 63 |   \coordinate (k') at ($(k.north) + (-3mm,4mm)$);
 64 |   \coordinate (r') at ($(r.north) + (-3mm,4mm)$); % opqr is in rqpo order
 65 |   \coordinate (s') at ($(s.north) + (-3mm,4mm)$);
 66 |   % Actual group boxes
 67 |   \node[nodegroup, inner sep=4mm, fit=(a') (b) (c)] (g1) {};
 68 |   \node[nodegroup, inner sep=4mm, fit=(d') (e) (f) (g) (h) (i) (j)] (g2) {};
 69 |   \node[nodegroup, inner sep=4mm, fit=(k') (l) (m) (n)] (g3) {};
 70 |   \node[nodegroup, inner sep=4mm, fit=(r') (q) (p) (o)] (g4) {};
 71 |   \node[nodegroup, inner sep=4mm, fit=(s') (t) (u) (v)] (g5) {};
 72 |   % Labels
 73 |   \node[below right] at (g1.north west) {\large\textbf{Preprocessing}};
 74 |   \node[below right] at (g2.north west) {\large\textbf{Cell contour detection}};
 75 |   \node[below right] at (g3.north west) {\large\textbf{Corner clustering \& cell detection}};
 76 |   \node[below right] at (g4.north west) {\large\textbf{Missing cell detection}};
 77 |   \node[below right] at (g5.north west) {\large\textbf{Table coordinate computation}};
 78 | \end{scope}
 79 | 
 80 | % Arrows between nodes
 81 | \begin{scope}[every path/.style=line]
 82 |   % First pass
 83 |   \path[firstpass] (a) -- (b);
 84 |   \path[firstpass] (b) -- (c);
 85 |   \path[firstpass] (c) -- (d);
 86 |   \path[firstpass] (d) -- (e);
 87 |   \path[firstpass] (e) -- (f);
 88 |   \path[firstpass] (f) -- (g);
 89 |   \path[firstpass] (g) -- (h);
 90 |   \path[firstpass] (h) -- (i);
 91 |   \path[firstpass] (i) -- (j);
 92 |   \path[firstpass] (j) -- (k);
 93 |   \path[firstpass] (k) -- (l);
 94 |   \path[firstpass] (l) -- (m);
 95 |   \path[firstpass] (m) -- (n);
 96 |   \path[firstpass] (n) -- (o);
 97 |   \path[firstpass] (o) -- (p);
 98 |   \path[firstpass] (p) -- (q);
 99 |   \path[firstpass] (q) -- (r);
100 |   % Missing edge loop
101 |   \path[firstpass, rounded corners=5pt] (r) -- (ra);
102 |   \path[secondpass, rounded corners=5pt] (ra) |- (i);
103 |   % Second pass
104 |   \path[secondpass] ([xshift=5mm]i.south) -- ([xshift=5mm]j.north);
105 |   \path[secondpass] ([xshift=5mm]j.south) -- ([xshift=5mm]k.north);
106 |   \path[secondpass] ([xshift=5mm]k.south) -- ([xshift=5mm]l.north);
107 |   \path[secondpass] ([xshift=5mm]l.south) -- ([xshift=5mm]m.north);
108 |   \path[secondpass] ([xshift=5mm]m.south) -- ([xshift=5mm]n.north);
109 |   \path[secondpass] ([xshift=0mm]n.south) -- ([xshift=0mm]s.north);
110 |   \path[secondpass] ([xshift=0mm]s.south) -- ([xshift=0mm]t.north);
111 |   \path[secondpass] ([xshift=0mm]t.south) -- ([xshift=0mm]u.north);
112 |   \path[secondpass] ([xshift=0mm]u.south) -- ([xshift=0mm]v.north);
113 | \end{scope}
114 | \end{tikzpicture}
115 | 
116 | \end{document}


--------------------------------------------------------------------------------
/Interactive.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": null,
  6 |    "metadata": {
  7 |     "collapsed": false
  8 |    },
  9 |    "outputs": [],
 10 |    "source": [
 11 |     "%load_ext autoreload\n",
 12 |     "%autoreload 1\n",
 13 |     "%aimport TableRecognition\n",
 14 |     "import cv2\n",
 15 |     "from matplotlib import pyplot as plt\n",
 16 |     "import numpy as np\n",
 17 |     "import time as t\n",
 18 |     "import cv_algorithms\n",
 19 |     "import collections\n",
 20 |     "import operator\n",
 21 |     "import scipy.signal\n",
 22 |     "import scipy.spatial.distance\n",
 23 |     "import TableRecognition\n",
 24 |     "from UliEngineering.SignalProcessing.Selection import *\n",
 25 |     "print (\"OpenCV Version : %s \" % cv2.__version__)"
 26 |    ]
 27 |   },
 28 |   {
 29 |    "cell_type": "markdown",
 30 |    "metadata": {},
 31 |    "source": [
 32 |     "### Configuration"
 33 |    ]
 34 |   },
 35 |   {
 36 |    "cell_type": "code",
 37 |    "execution_count": null,
 38 |    "metadata": {
 39 |     "collapsed": true
 40 |    },
 41 |    "outputs": [],
 42 |    "source": [
 43 |     "# Run './download-oldweather.py Northwind 1947' to get the files\n",
 44 |     "filename = \"Northwind/Northwind-WAG-282-1947-0063-0.jpg\""
 45 |    ]
 46 |   },
 47 |   {
 48 |    "cell_type": "markdown",
 49 |    "metadata": {},
 50 |    "source": [
 51 |     "### Image preprocessing"
 52 |    ]
 53 |   },
 54 |   {
 55 |    "cell_type": "code",
 56 |    "execution_count": null,
 57 |    "metadata": {
 58 |     "collapsed": false
 59 |    },
 60 |    "outputs": [],
 61 |    "source": [
 62 |     "%matplotlib inline\n",
 63 |     "img = cv2.imread(filename, flags=cv2.IMREAD_COLOR)\n",
 64 |     "if img is None:\n",
 65 |     "    raise ValueError(\"File {0} does not exist\".format(filename))\n",
 66 |     "imgGrey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
 67 |     "imgThresh = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY_INV)[1]\n",
 68 |     "imgThreshInv = cv2.threshold(imgGrey, 150, 255, cv2.THRESH_BINARY)[1]\n",
 69 |     "\n",
 70 |     "imgDil = cv2.dilate(imgThresh, np.ones((5, 5), np.uint8))\n",
 71 |     "imgEro = cv2.erode(imgDil, np.ones((4, 4), np.uint8))\n",
 72 |     "\n",
 73 |     "plt.gcf().set_size_inches(20,18)\n",
 74 |     "plt.imshow(imgDil, cmap=\"Greys_r\")"
 75 |    ]
 76 |   },
 77 |   {
 78 |    "cell_type": "code",
 79 |    "execution_count": null,
 80 |    "metadata": {
 81 |     "collapsed": false
 82 |    },
 83 |    "outputs": [],
 84 |    "source": [
 85 |     "#\n",
 86 |     "ix = img.copy()\n",
 87 |     "contour_analyzer = TableRecognition.ContourAnalyzer(imgDil)\n",
 88 |     "contour_analyzer.filter_contours(min_area=400)\n",
 89 |     "contour_analyzer.build_graph()\n",
 90 |     "contour_analyzer.remove_non_table_nodes()\n",
 91 |     "contour_analyzer.compute_contour_bounding_boxes()\n",
 92 |     "contour_analyzer.separate_supernode()\n",
 93 |     "print(contour_analyzer.does_page_have_valid_table())\n",
 94 |     "contour_analyzer.find_empty_cells(imgThreshInv)\n",
 95 |     "contour_analyzer.visualize_contours(ix)\n",
 96 |     "\n",
 97 |     "plt.gcf().set_size_inches(20,18)\n",
 98 |     "plt.imshow(ix)\n",
 99 |     "\n",
100 |     "#cv2.imwrite(\"/ram/Northwind-1.png\", ix)"
101 |    ]
102 |   },
103 |   {
104 |    "cell_type": "markdown",
105 |    "metadata": {},
106 |    "source": [
107 |     "### Find bounding box corner clusters"
108 |    ]
109 |   },
110 |   {
111 |    "cell_type": "code",
112 |    "execution_count": null,
113 |    "metadata": {
114 |     "collapsed": false
115 |    },
116 |    "outputs": [],
117 |    "source": [
118 |     "contour_analyzer.find_corner_clusters()"
119 |    ]
120 |   },
121 |   {
122 |    "cell_type": "code",
123 |    "execution_count": null,
124 |    "metadata": {
125 |     "collapsed": false
126 |    },
127 |    "outputs": [],
128 |    "source": [
129 |     "%matplotlib inline\n",
130 |     "imgCopy = img.copy()\n",
131 |     "plt.gcf().set_size_inches(20,18)\n",
132 |     "contour_analyzer.visualize_corner_clusters(imgCopy)\n",
133 |     "plt.imshow(imgCopy)\n",
134 |     "plt.title(\"Corner color shows number of merged nodes\")"
135 |    ]
136 |   },
137 |   {
138 |    "cell_type": "markdown",
139 |    "metadata": {},
140 |    "source": [
141 |     "### Compute cells from contours and clusters"
142 |    ]
143 |   },
144 |   {
145 |    "cell_type": "code",
146 |    "execution_count": null,
147 |    "metadata": {
148 |     "collapsed": false
149 |    },
150 |    "outputs": [],
151 |    "source": [
152 |     "%matplotlib inline\n",
153 |     "contour_analyzer.compute_cell_hulls()"
154 |    ]
155 |   },
156 |   {
157 |    "cell_type": "markdown",
158 |    "metadata": {},
159 |    "source": [
160 |     "### Recompute cell boundaries based on nodes & compute table angle"
161 |    ]
162 |   },
163 |   {
164 |    "cell_type": "code",
165 |    "execution_count": null,
166 |    "metadata": {
167 |     "collapsed": false
168 |    },
169 |    "outputs": [],
170 |    "source": [
171 |     "#### %matplotlib inline\n",
172 |     "ix = img.copy()\n",
173 |     "plt.gcf().set_size_inches(20,18)\n",
174 |     "\n",
175 |     "contour_analyzer.draw_all_cell_hulls(ix, xscale=0.8, yscale=0.8)\n",
176 |     "plt.imshow(ix)\n",
177 |     "#cv2.imwrite(\"/ram/ATR.png\", ix)"
178 |    ]
179 |   },
180 |   {
181 |    "cell_type": "markdown",
182 |    "metadata": {},
183 |    "source": [
184 |     "### Find missing cells by masking"
185 |    ]
186 |   },
187 |   {
188 |    "cell_type": "code",
189 |    "execution_count": null,
190 |    "metadata": {
191 |     "collapsed": false
192 |    },
193 |    "outputs": [],
194 |    "source": [
195 |     "ix = img.copy()\n",
196 |     "\n",
197 |     "contour_analyzer.find_fine_table_corners()\n",
198 |     "missing_contours = contour_analyzer.compute_filtered_missing_cell_contours()\n",
199 |     "\n",
200 |     "icm = cv2.drawContours(ix, missing_contours, -1, (0, 255, 0), 3)\n",
201 |     "\n",
202 |     "plt.gcf().set_size_inches(20,18)\n",
203 |     "plt.imshow(ix, cmap=\"Greys_r\")"
204 |    ]
205 |   },
206 |   {
207 |    "cell_type": "markdown",
208 |    "metadata": {},
209 |    "source": [
210 |     "### Insert clusters into main contours & perform second run"
211 |    ]
212 |   },
213 |   {
214 |    "cell_type": "code",
215 |    "execution_count": null,
216 |    "metadata": {
217 |     "collapsed": false
218 |    },
219 |    "outputs": [],
220 |    "source": [
221 |     "contour_analyzer.contours += missing_contours\n",
222 |     "\n",
223 |     "contour_analyzer.compute_contour_bounding_boxes()\n",
224 |     "contour_analyzer.find_empty_cells(imgThreshInv)\n",
225 |     "\n",
226 |     "contour_analyzer.find_corner_clusters()\n",
227 |     "contour_analyzer.compute_cell_hulls()\n",
228 |     "contour_analyzer.find_fine_table_corners()"
229 |    ]
230 |   },
231 |   {
232 |    "cell_type": "markdown",
233 |    "metadata": {
234 |     "collapsed": true
235 |    },
236 |    "source": [
237 |     "### Find cluster centers and group into hgroups and vgroups"
238 |    ]
239 |   },
240 |   {
241 |    "cell_type": "code",
242 |    "execution_count": null,
243 |    "metadata": {
244 |     "collapsed": false
245 |    },
246 |    "outputs": [],
247 |    "source": [
248 |     "contour_analyzer.compute_table_coordinates(5.)"
249 |    ]
250 |   },
251 |   {
252 |    "cell_type": "code",
253 |    "execution_count": null,
254 |    "metadata": {
255 |     "collapsed": false
256 |    },
257 |    "outputs": [],
258 |    "source": [
259 |     "ix = img.copy()\n",
260 |     "\n",
261 |     "contour_analyzer.draw_table_coord_cell_hulls(ix, xscale=.9, yscale=.9)\n",
262 |     "\n",
263 |     "plt.gcf().set_size_inches(20,18)\n",
264 |     "plt.imshow(ix)\n",
265 |     "#cv2.imwrite(\"/ram/ATR2.png\", ix)"
266 |    ]
267 |   },
268 |   {
269 |    "cell_type": "markdown",
270 |    "metadata": {
271 |     "collapsed": true
272 |    },
273 |    "source": [
274 |     "### Extract table cell image"
275 |    ]
276 |   },
277 |   {
278 |    "cell_type": "code",
279 |    "execution_count": null,
280 |    "metadata": {
281 |     "collapsed": false
282 |    },
283 |    "outputs": [],
284 |    "source": [
285 |     "plt.gcf().set_size_inches(20,18)\n",
286 |     "\n",
287 |     "#ix = cv2.cvtColor(ix, cv2.COLOR_GRAY2BGR)\n",
288 |     "ix = img.copy()\n",
289 |     "ix = contour_analyzer.extract_cell_from_image(ix, (14,6))\n",
290 |     "\n",
291 |     "plt.imshow(ix, cmap=\"Greys_r\")\n",
292 |     "#cv2.imwrite(\"/ram/cell.png\", ix)"
293 |    ]
294 |   },
295 |   {
296 |    "cell_type": "code",
297 |    "execution_count": null,
298 |    "metadata": {
299 |     "collapsed": false
300 |    },
301 |    "outputs": [],
302 |    "source": [
303 |     "plt.gcf().set_size_inches(20,18)\n",
304 |     "\n",
305 |     "ix = img.copy()\n",
306 |     "ix = contour_analyzer.extract_cell_from_image(ix, (14,6), xscale=1, yscale=1, mark_color=None)\n",
307 |     "plt.imshow(ix, cmap=\"Greys_r\")\n",
308 |     "cv2.imwrite(\"/ram/airwetbulb.png\", ix)"
309 |    ]
310 |   },
311 |   {
312 |    "cell_type": "code",
313 |    "execution_count": null,
314 |    "metadata": {
315 |     "collapsed": true
316 |    },
317 |    "outputs": [],
318 |    "source": []
319 |   }
320 |  ],
321 |  "metadata": {
322 |   "kernelspec": {
323 |    "display_name": "Python 3",
324 |    "language": "python",
325 |    "name": "python3"
326 |   },
327 |   "language_info": {
328 |    "codemirror_mode": {
329 |     "name": "ipython",
330 |     "version": 3
331 |    },
332 |    "file_extension": ".py",
333 |    "mimetype": "text/x-python",
334 |    "name": "python",
335 |    "nbconvert_exporter": "python",
336 |    "pygments_lexer": "ipython3",
337 |    "version": "3.5.2"
338 |   }
339 |  },
340 |  "nbformat": 4,
341 |  "nbformat_minor": 0
342 | }
343 | 


--------------------------------------------------------------------------------
/TableRecognition.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | # -*- coding: utf-8 -*-
  3 | import cv2
  4 | import numpy as np
  5 | import cv_algorithms
  6 | import networkx as nx
  7 | import operator
  8 | import scipy.spatial.distance
  9 | from UliEngineering.SignalProcessing.Selection import multiselect
 10 | from UliEngineering.Utils.NumPy import invert_bijection
 11 | 
 12 | def transitive_closure(clusters):
 13 |     """
 14 |     Takes a map of <node number> => <list of associated nodes>
 15 |     and computes all transitive closures using a fast set-based algorithm
 16 | 
 17 |     Returns a list of sets (each set containing one transitive closure) and
 18 |     a node -> closure ID lookup table. The closure id can be used to look up
 19 |     the hull nodes as index to the list of closure
 20 |     """
 21 |     transitive_clusters = []
 22 |     cluster_lut = np.zeros(len(clusters), np.int) # Node ID -> cluster map
 23 |     already_assigned = np.zeros(len(clusters), np.bool)
 24 |     for i, cluster in clusters.items():
 25 |         if already_assigned[i]:
 26 |             continue
 27 |         # Build transitive list
 28 |         already_seen = {i} # Avoids infinite recursion, also this is the newly built transitive group
 29 |         todo = set(cluster)
 30 |         # Find all transitive elements in the current cluster
 31 |         while len(todo) > 0:
 32 |             j = todo.pop()
 33 |             jsim = clusters[j]
 34 |             # Try to avoid too much Python algorithms, use C core functions instead
 35 |             already_seen.add(j)
 36 |             already_seen |= set(jsim)
 37 |             todo |= set(jsim)
 38 |             todo -= already_seen
 39 |             # Set already seen array
 40 |             already_assigned[j] = True
 41 |             already_assigned[list(jsim)] = True
 42 |         # Now we have a list of all nodes in the sim group.
 43 |         # We can assign IDs (i.e. the index) and
 44 |         #  store it in a <endpoint ID> -> <cluster> LUT
 45 |         cluster_no = len(transitive_clusters) # Because we append once per loop
 46 |         transitive_clusters.append(already_seen)
 47 |         cluster_lut[list(already_seen)] = cluster_no
 48 |     return transitive_clusters, cluster_lut
 49 | 
 50 | 
 51 | 
 52 | def is_inside_table(polygon, reference):
 53 |     """
 54 |     Determine if a given polygon is fully located inside
 55 |     This works by checking if all polygon points are within (or on the edge of)
 56 |     the reference 
 57 | 
 58 |     returns True if and only if polygon is fully within or identical to the reference.
 59 |     """
 60 |     brect = cv2.boxPoints(cv2.minAreaRect(polygon))
 61 |     # All points need to be inside table corners
 62 |     for point in brect:
 63 |         if cv2.pointPolygonTest(reference, tuple(point), False) < 0:
 64 |             return False  # Not in contour
 65 |     return True
 66 | 
 67 | def angle_degrees(dx, dy):
 68 |     if dx == 0: return 180.
 69 |     return np.rad2deg(np.arctan2(dx, dy))
 70 | 
 71 | 
 72 | def _find_contours(*args, **kwargs):
 73 |     """
 74 |     Calls cv2.findContours(*args, **kwargs) and returns (contours, hierarchy)
 75 |     """
 76 |     tupl = cv2.findContours(*args, **kwargs)
 77 |     # Fix for #8
 78 |     if len(tupl) == 3:
 79 |         im2, contours, hierarchy = tupl
 80 |     elif len(tupl) == 2:
 81 |         contours, hierarchy = tupl
 82 |     return contours, hierarchy
 83 | 
 84 | class ContourAnalyzer(object):
 85 |     def __init__(self, img, **kwargs):
 86 |         contours, hierarchy = _find_contours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_TC89_KCOS, **kwargs)
 87 |         self.hierarchy = hierarchy
 88 |         self.contours = contours
 89 |         self.imgshape = img.shape
 90 | 
 91 |     @property
 92 |     def size(self):
 93 |         return len(self.contours)
 94 | 
 95 |     def build_graph(self):
 96 |         # Contour to graph
 97 |         self.g = nx.DiGraph()
 98 |         # Add each contour as node (by index)
 99 |         self.g.add_nodes_from(range(self.hierarchy.shape[1]))
100 |         # Build graph from hierarchy
101 |         for i in range(self.hierarchy.shape[1]):
102 |             # Skip already invalidated contours
103 |             if self.contours[i] is None: continue
104 |             nxt, prev, first_child, parent = self.hierarchy[0, i]
105 |             if parent != -1:
106 |                 self.g.add_edge(parent, i)
107 | 
108 |     def compute_cell_polygons(self):
109 |         """
110 |         Compute a list of cell polygons from the contours and the associated
111 |         corner clusters. Generates a list of cells, with a cell being a list of
112 |         cluster IDs. This list is stored in self.cell_polygons
113 |         """
114 |         cell_polygons = []
115 |         for i in range(len(self.contours)):
116 |             # Skip invalidated contours
117 |             if self.contours[i] is None: continue
118 |             bbox = self.contours_bbox[i]  # Guaranteed to have 4 nodes by design (it's a bounding box)!
119 |             # bbox == corner1, ..., corner4
120 |             bclust = multiselect(self.cluster_coords_to_node_id, bbox, convert=tuple) 
121 |             cell_polygons.append(bclust)
122 |         self.cell_polygons = cell_polygons
123 | 
124 | 
125 |     def compute_cell_hulls(self):
126 |         """
127 |         Run find_table_cell_polygons() and compute a rectangle enclosing the cell (for each cell).
128 |         For most (4-point) cells, this is equivalent to the original path, however this removes
129 |         small irregularities and extra points from larger, 5+-point cells (mostly merged cells)
130 |         """
131 |         self.compute_cell_polygons()
132 |         # cv2 convexHull / minAreaRect only work with integer coordinates.
133 |         self.cell_hulls = [
134 |             cv2.boxPoints(cv2.minAreaRect(np.rint(self.cluster_coords[path]).astype(int)))
135 |             for path in self.cell_polygons]
136 |         # Compute centers of cell hulls
137 |         self.cell_centers = np.zeros((len(self.cell_hulls), 2))
138 |         for i in range(len(self.cell_hulls)):
139 |             hull_points = self.cell_hulls[i]
140 |             self.cell_centers[i] = cv_algorithms.meanCenter(hull_points)
141 | 
142 |     def compute_table_coordinates(self, xthresh=8., ythresh=8.):
143 |         """
144 |         Sorts all clusters into spreadsheet-like x/y coordinates.
145 |         Set self.cell_table_coord of shape (0, n)
146 |         """
147 |         center_x = self.cell_centers[:, 0]
148 |         center_y = self.cell_centers[:, 1]
149 |         # Compute adjacency list
150 |         hgroup_adjlist, vgroup_adjlist = {}, {}
151 |         for i in range(center_x.shape[0]):
152 |             hgroup_adjlist[i] = np.nonzero(np.abs(center_x - center_x[i]) < xthresh)[0]
153 |             vgroup_adjlist[i] = np.nonzero(np.abs(center_y - center_y[i]) < ythresh)[0]
154 | 
155 |         # Compute transitive closures so we get ALL grouped cells for each group,
156 |         # not just the ones that are similar to the first node.
157 |         hgroups, hgroup_lut = transitive_closure(hgroup_adjlist)
158 |         vgroups, vgroup_lut = transitive_closure(vgroup_adjlist)
159 |         ####
160 |         # Reorder groups by x/y
161 |         ####
162 |         # Compute mean X/Y coords for each hgroup/vgroup
163 |         hgroup_mean_centers = np.zeros(len(hgroups))
164 |         vgroup_mean_centers = np.zeros(len(vgroups))
165 |         for i, st in enumerate(hgroups):
166 |             hgroup_mean_centers[i] = np.mean(self.cell_centers[list(st)][:, 0])
167 |         for i, st in enumerate(vgroups):
168 |             vgroup_mean_centers[i] = np.mean(self.cell_centers[list(st)][:, 1])
169 |         # Find the sorted ordering (like in a spreadsheet) for both x and y coords
170 |         hgroup_sort_order = np.argsort(hgroup_mean_centers)
171 |         vgroup_sort_order = np.argsort(vgroup_mean_centers)
172 |         # Output of argsort: Input => new index ; output => old index
173 |         # BUT WE NEED: Input oldplace, output newplace
174 |         hgroup_sort_order = invert_bijection(hgroup_sort_order)
175 |         vgroup_sort_order = invert_bijection(vgroup_sort_order)
176 |         # Reorder everything based on the new order
177 |         hgroups = multiselect(hgroups, hgroup_sort_order)
178 |         vgroups = multiselect(vgroups, vgroup_sort_order)
179 |         # Convert LUTs
180 |         hgroup_lut = hgroup_sort_order[hgroup_lut]
181 |         vgroup_lut = vgroup_sort_order[vgroup_lut]
182 | 
183 |         # Build a (n, (tx,ty)) table coordinate LUT for all nodes
184 |         self.cell_table_coord = np.dstack([hgroup_lut, vgroup_lut])[0]
185 | 
186 |         # Build index of table coordinates to node ID
187 |         self.table_coords_to_node = {}
188 |         for i, (x, y) in enumerate(self.cell_table_coord):
189 |             self.table_coords_to_node[(x, y)] = i
190 | 
191 | 
192 |     def filter_contours(self, min_area=250, min_nodes=4):
193 |         """
194 |         Remove contours:
195 |             - With less than a specific area (square px)
196 |             - With less than n nodes (usually 4)
197 |         """
198 |         for i in range(self.size):
199 |             # Skip already invalidated contours
200 |             if self.contours[i] is None: continue
201 |             # Compute key parameters
202 |             num_nodes = self.contours[i].shape[0]
203 |             area = cv2.contourArea(self.contours[i])
204 |             # Check if node shall be removed
205 |             if area < min_area or num_nodes < min_nodes:
206 |                 self.contours[i] = None
207 | 
208 |     def remove_non_table_nodes(self):
209 |         """
210 |         Identify the topmost table node ("supernode") (i.e. the node with the most direct children)
211 |         and delete every node in the graph (and invalidate any related contour)
212 |         that is not either:
213 |             - The supernode itself (i.e. the table outline) or
214 |             - A direct child of the supernode
215 |         This will remove stuff nested inside table cells and nodes outside the table
216 |         The nodes are not removed from the graph.
217 |         """
218 |         self.supernode_idx = max(dict(self.g.degree()).items(), key=operator.itemgetter(1))[0]
219 |         for i in range(len(self.contours)):
220 |             if self.contours[i] is None: continue
221 |             nxt, prev, first_child, parent = self.hierarchy[0, i]
222 |             # Remove node if it has a non-supernode node as parent
223 |             if parent != self.supernode_idx and i != self.supernode_idx:
224 |                 self.contours[i] = None
225 |     
226 |     def compute_contour_bounding_boxes(self):
227 |         """
228 |         Compute rotated min-area bounding boxes for every contour
229 |         """
230 |         self.contours_bbox = [None] * len(self.contours)
231 |         self.aspect_ratios = np.zeros(self.size) # of rotated bounding boxes
232 |         for i in range(len(self.contours)):
233 |             if self.contours[i] is None: continue
234 |             # Compute rotated bounding rectangle (4 points)
235 |             bbox = cv2.minAreaRect(self.contours[i])
236 |             # Compute aspect ratio
237 |             (x1, y1), (x2, y2), angle = bbox
238 |             self.aspect_ratios[i] = np.abs(x2 - x1) / np.abs(y2 - y1)
239 |             # Convert to 4-point rotated box, convert to int and set as new contour
240 |             self.contours_bbox[i] = np.rint(cv2.boxPoints(bbox)).astype(np.int)
241 | 
242 |     def separate_supernode(self):
243 |         """
244 |         Remove the supernode from the contours and save it separately.
245 |         This means that only table cells and artifacts should be left as contours
246 |         """
247 |         # Store separately
248 |         self.supernode = self.contours[self.supernode_idx]
249 |         self.supernode_bbox = self.contours_bbox[self.supernode_idx]
250 |         # Invalidate in normal storage
251 |         self.contours[self.supernode_idx] = None
252 |         self.contours_bbox[self.supernode_idx] = None
253 | 
254 |     def does_page_have_valid_table(self, min_fract_area=.2, min_cells=50):
255 |         """
256 |         Analyzes whether the image contains a table by evaluating the
257 |         coarse table outline and its children
258 |         """
259 |         try: # Some CV2 operations may fail e.g. if no correct supernode has been recognized
260 |             # Check fractional area of table compared to image
261 |             img_area = self.imgshape[0] * self.imgshape[1]
262 |             supernode_area = cv2.contourArea(self.supernode_bbox)
263 |             if supernode_area < img_area * min_fract_area:
264 |                 return False
265 |             # Check minimum number of cells (ncells = degree of coarse outline node)
266 |             ncells = self.g.degree(self.supernode_idx)
267 |             return ncells >= min_cells
268 |         except cv2.error:
269 |             return False
270 | 
271 |     def find_empty_cells(self, img, threshold=.998):
272 |         """
273 |         Find out which cells are empty by 
274 |         """
275 |         # Compute which cells are empty
276 |         self.is_contour_empty = np.zeros(self.size, np.bool) # of rotated bounding boxes
277 |         for i in range(len(self.contours)):
278 |             if self.contours[i] is None: continue
279 |             sect = cv_algorithms.extractPolygonMask(img, self.contours_bbox[i])
280 |             self.is_contour_empty[i] = cv_algorithms.fractionWhite(sect) > threshold
281 |     
282 |     def compute_contour_centers(self):
283 |         """
284 |         Compute cell centers for each contour bounding box using meanCenter()
285 |         """
286 |         self.contour_centers = np.full((len(self.contours), 2), -1.)  # -1: Contour invalid
287 |         for i in range(len(self.contours)):
288 |             if self.contours[i] is None: continue
289 |             self.contour_centers[i] = cv_algorithms.meanCenter(self.contours_bbox[i])
290 | 
291 |     def find_corner_clusters(self, distance_threshold=20.):
292 |         # Find all bounding box corners
293 |         corners = []
294 |         for i in range(len(self.contours)):
295 |             if self.contours[i] is None: continue
296 |             bbox = self.contours_bbox[i]
297 |             for coord in bbox:
298 |                 corners.append((coord[0], coord[1]))
299 | 
300 |         # Simpler algorithm, still superfast (<40 ms for 2k corners): Compute all distances using cdist
301 |         corners = np.asarray(corners)
302 |         distmat = scipy.spatial.distance.cdist(corners, corners, 'euclidean')
303 |         ignore = np.zeros(corners.shape[0], np.bool) # Set to true if we found a cluster for this node already
304 | 
305 |         # Find cluster in the distance matrix, i.e. node groups which are close together
306 |         cluster_coords = []  # For each cluster, a (x,y coordinate pair)
307 |         cluster_num_nodes = []  # For each cluster, the number of nodes it consists of
308 |         cluster_coords_to_node_id = {}  # (x,y) tuple => cluster ID
309 |         for i in range(corners.shape[0]):
310 |             if ignore[i]: continue
311 |             # Which nodes are close to this node, including itself
312 |             below_thresh = distmat[i, :] < distance_threshold # Rather set this large, we can correct non-convexity later
313 |             allnodes = np.nonzero(below_thresh)[0] # index list
314 |             # Get a new ID
315 |             clusterid = len(cluster_coords)
316 |             allcorners = corners[allnodes]
317 |             cluster_coords.append(tuple(cv_algorithms.meanCenter(allcorners)))
318 |             cluster_num_nodes.append(allnodes.size)
319 |             # Also create a map from each position to the current cluster ID
320 |             # This works only because these coordinates are discrete integer pixel indices
321 |             for coord in allcorners:
322 |                 cluster_coords_to_node_id[tuple(coord)] = clusterid
323 |             # Ignore all nodes in the cluster (i.e. don't assign them to a new cluster)
324 |             ignore[allnodes] = True
325 |         # Now that the size is known, we can convert to numpy arrays
326 |         self.cluster_coords = np.asarray(cluster_coords)
327 |         self.cluster_num_nodes = np.asarray(cluster_num_nodes)
328 |         self.cluster_coords_to_node_id = cluster_coords_to_node_id
329 | 
330 |     def find_fine_table_corners(self):
331 |         """
332 |         Find fine table corners. This function works on cluster coordinates.
333 |         It returns the ones with the most extreme coordinates, intermixing X/Y coordinates.
334 |         """
335 |         # The absolute min/max (top left / bot right corner) can be found without array modification3
336 |         minmax_unmodified = np.prod(self.cluster_coords, axis=1)
337 |         minx_miny = np.argmin(minmax_unmodified)
338 |         maxx_maxy = np.argmax(minmax_unmodified)
339 |         # Copy and modify the array
340 |         ccopy = self.cluster_coords.copy()
341 |         ccopy[:, 1] = np.reciprocal(ccopy[:, 1])  # multiply Y by -1, results in X/-Y
342 |         minx_maxy = np.argmin(np.prod(ccopy, axis=1))
343 | 
344 |         ccopy = self.cluster_coords.copy()
345 |         ccopy[:, 0] = np.reciprocal(ccopy[:, 0])  # multiply X/Y by -1, results in -X/Y
346 |         maxx_miny = np.argmin(np.prod(ccopy, axis=1))
347 | 
348 |         corners = np.rint(self.cluster_coords[[minx_miny, maxx_miny, maxx_maxy, minx_maxy, minx_miny]]).astype(np.int)
349 |         self.table_corners = corners
350 |         return corners
351 | 
352 |     def compute_missing_cells_mask(self, close_ksize=5):
353 |         """
354 |         Compute a binary img-scale mask,
355 |         """
356 |         # Create white binary img
357 |         icellmask = np.full((self.imgshape[0], self.imgshape[1]), 255, np.uint8)
358 | 
359 |         # Mask everything except table, as defined by corner nodes (not the larger super-node!)
360 |         cv2.fillConvexPoly(icellmask, self.table_corners, 0)
361 |         # Now draw all cell hulls without text, but don't downsize them()
362 |         self.draw_all_cell_hulls(icellmask, None, xscale=1.1, yscale=1.1)
363 | 
364 |         # Morphology ops with large kernel to remove small intercell speckles
365 |         # NOTE: CLOSE => remove black holes
366 |         icellmask = cv2.morphologyEx(icellmask, cv2.MORPH_CLOSE,
367 |                                      np.ones((close_ksize, close_ksize), np.uint8))
368 |         return icellmask
369 | 
370 |     def compute_missing_cell_contours(self, missing_cells_mask):
371 |         contx, _ = _find_contours(missing_cells_mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_TC89_KCOS)
372 |         return contx
373 | 
374 |     def compute_filtered_missing_cell_contours(self):
375 |         """
376 |         Filter the given missing
377 |         """
378 |         missing_cells_mask = self.compute_missing_cells_mask()
379 |         missing_cell_contours = self.compute_missing_cell_contours(missing_cells_mask)
380 |         table_corners = self.table_corners  # fine table corners
381 |         # Perform filtering
382 |         return [c for c in missing_cell_contours if is_inside_table(c, table_corners)]
383 | 
384 | 
385 |     def extract_cell_from_image(self, img, table_coords, xscale=3, yscale=3, mark_color=(255,0,0), mark_thickness=1):
386 |         """
387 |         Extract a section from the image that corresponds to the table cell.
388 | 
389 |         Parameters
390 |         ----------
391 |         img : array_like 2D
392 |             The return value is a subsection of this
393 |         table_coords : (x,y) table coordinates
394 |             The coordinates of the cell to extract
395 |         """
396 |         # Convert table coordinates to node ID
397 |         node_id = self.table_coords_to_node[table_coords]
398 |         # Get upright bounding rectangle
399 |         bounding_rect = cv2.boundingRect(self.cell_hulls[node_id])
400 |         # Expand rectangle
401 |         bounding_rect_expanded = cv_algorithms.expandRectangle(bounding_rect, xscale, yscale)
402 |         # Slice image
403 |         x, y, w, h = bounding_rect_expanded
404 |         img_sect = img[y:y + h, x:x + w]
405 |         # Mark original rectangle
406 |         # The coordinates need to be re-calculated
407 |         #  because img_sect has a different origin point than img.
408 |         if mark_color is not None:
409 |             x1, y1, w1, h1 = bounding_rect
410 |             x2, y2, w2, h2 = bounding_rect_expanded
411 |             cv2.rectangle(img_sect, (x1 - x2, y1 - y2),
412 |                           (x1 - x2 + w1, y1 - y2 + h1), color=(0, 0, 255),
413 |                           thickness=mark_thickness)
414 |         return img_sect
415 | 
416 | 
417 | 
418 |     def visualize_corner_clusters(self, img):
419 |         col_map = {0: (0, 0, 0),
420 |                    1: (0, 0, 200),
421 |                    2: (200, 0, 0),
422 |                    3: (200, 200, 0),
423 |                    4: (0, 200, 0),
424 |                    5: (200, 200, 200),
425 |                    -1: (255, 255, 255)}
426 |         for i, (x, y) in enumerate(self.cluster_coords):
427 |             n = self.cluster_num_nodes[i]
428 |             col = col_map[n] if n < 5 else col_map[-1]
429 |             cv2.circle(img, (int(x), int(y)), 8,
430 |                        color=col, thickness=-1) # filled
431 | 
432 | 
433 |     def visualize_contours(self, img, thickness=2, draw_empty=True):
434 |         """
435 |         Draw the contour-based table information
436 |             - Outline in red
437 |             - Cells in green
438 |         """
439 |         # Draw table outline in red
440 |         cv2.drawContours(img, [self.supernode_bbox], -1, (255, 0, 0), thickness)
441 |         # Draw table cells
442 |         cv2.drawContours(img, self.contours_bbox, -1, (0,255,0), thickness)
443 |         # Draw empty table cells (they have already been drawn as table cells, just draw over that)
444 |         if draw_empty:
445 |             cv2.drawContours(img, [c for i,c in enumerate(self.contours_bbox)
446 |                                    if self.is_contour_empty[i] and c is not None], -1, (0, 0, 255), thickness)
447 | 
448 |     def draw_cell_hull(self, img, i, text, xscale=1., yscale=1., textsize=1, fill_color=(255,255,0)):
449 |         """
450 |         Draw a single cell hull as a filled polygon
451 |         """
452 |         # Drawing functions need pure integer coordinates
453 |         thehull = np.rint(cv_algorithms.scaleByRefpoint(self.cell_hulls[i], xscale, yscale)).astype(np.int)
454 |         # Draw the polygon
455 |         cv2.fillConvexPoly(img, thehull, color=fill_color)
456 |         # Draw text
457 |         if text is not None:
458 |             thehull_center = np.mean(thehull, axis=0)
459 |             boxWidth, boxHeight = np.ptp(thehull, axis=0) # Max X/Y extent of the box
460 |             cv_algorithms.putTextAutoscale(img, text, thehull_center, cv2.FONT_HERSHEY_SIMPLEX,
461 |                                            boxWidth, boxHeight, color=(0,0,0), thickness=2)
462 |             
463 |     def draw_all_cell_hulls(self, img, text="{0}", **kwargs):
464 |         for i in range(len(self.cell_hulls)):
465 |             self.draw_cell_hull(img, i, text.format(i) if text is not None else None, **kwargs)
466 | 
467 |     def draw_table_coord_cell_hulls(self, img, **kwargs):
468 |         for i in range(len(self.cell_hulls)):
469 |             tcoordX, tcoordY = self.cell_table_coord[i]
470 |             text = "{0},{1}".format(tcoordX, tcoordY)
471 |             self.draw_cell_hull(img, i, text, **kwargs)
472 | 
473 |     def visualize_node_arrows(self, img, node_id, size=3, color=(255,0,255)):
474 |         l, r, t, b = self.adjacency[node_id]
475 |         srcX, srcY = self.node_coordinates[node_id]
476 |         for direction, r in enumerate((l,r,t,b)):
477 |             if r == -1: continue
478 |             targetX, targetY = self.node_coordinates[r]
479 |             # Use constant arrow head size. As arrowedLine() takes a fraction of the length, we need to reverse that
480 |             length = np.hypot(targetX - srcX, targetY - srcY)
481 |             arrowHeadSizeWant = 15  #px
482 |             arrowHeadSize = arrowHeadSizeWant / length
483 |             print("Drawing <{3}> arrow from #{0} to #{1}  of length {2}".format(
484 |                     r, node_id, length, {0: "left", 1: "right", 2:"top", 3:"bottom"}[direction]))
485 |             cv2.arrowedLine(img, (int(targetX), int(targetY)), (int(srcX), int(srcY)),
486 |                             color=color, thickness=3, tipLength=arrowHeadSize, line_type=cv2.LINE_AA)
487 | 
488 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 |                             Preamble
  9 | 
 10 |   The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 |   The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 |   When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 |   To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 |   For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 |   Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 |   For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 |   Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 |   Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 |   The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |                        TERMS AND CONDITIONS
 72 | 
 73 |   0. Definitions.
 74 | 
 75 |   "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 |   "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 |   "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 |   To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 |   A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 |   To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 |   To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 |   An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 |   1. Source Code.
113 | 
114 |   The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 |   A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 |   The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 |   The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 |   The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 |   The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 |   2. Basic Permissions.
155 | 
156 |   All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 |   You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 |   Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 |   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 |   No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 |   When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 |   4. Conveying Verbatim Copies.
196 | 
197 |   You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 |   You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 |   5. Conveying Modified Source Versions.
209 | 
210 |   You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 |     a) The work must carry prominent notices stating that you modified
215 |     it, and giving a relevant date.
216 | 
217 |     b) The work must carry prominent notices stating that it is
218 |     released under this License and any conditions added under section
219 |     7.  This requirement modifies the requirement in section 4 to
220 |     "keep intact all notices".
221 | 
222 |     c) You must license the entire work, as a whole, under this
223 |     License to anyone who comes into possession of a copy.  This
224 |     License will therefore apply, along with any applicable section 7
225 |     additional terms, to the whole of the work, and all its parts,
226 |     regardless of how they are packaged.  This License gives no
227 |     permission to license the work in any other way, but it does not
228 |     invalidate such permission if you have separately received it.
229 | 
230 |     d) If the work has interactive user interfaces, each must display
231 |     Appropriate Legal Notices; however, if the Program has interactive
232 |     interfaces that do not display Appropriate Legal Notices, your
233 |     work need not make them do so.
234 | 
235 |   A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 |   6. Conveying Non-Source Forms.
246 | 
247 |   You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 |     a) Convey the object code in, or embodied in, a physical product
253 |     (including a physical distribution medium), accompanied by the
254 |     Corresponding Source fixed on a durable physical medium
255 |     customarily used for software interchange.
256 | 
257 |     b) Convey the object code in, or embodied in, a physical product
258 |     (including a physical distribution medium), accompanied by a
259 |     written offer, valid for at least three years and valid for as
260 |     long as you offer spare parts or customer support for that product
261 |     model, to give anyone who possesses the object code either (1) a
262 |     copy of the Corresponding Source for all the software in the
263 |     product that is covered by this License, on a durable physical
264 |     medium customarily used for software interchange, for a price no
265 |     more than your reasonable cost of physically performing this
266 |     conveying of source, or (2) access to copy the
267 |     Corresponding Source from a network server at no charge.
268 | 
269 |     c) Convey individual copies of the object code with a copy of the
270 |     written offer to provide the Corresponding Source.  This
271 |     alternative is allowed only occasionally and noncommercially, and
272 |     only if you received the object code with such an offer, in accord
273 |     with subsection 6b.
274 | 
275 |     d) Convey the object code by offering access from a designated
276 |     place (gratis or for a charge), and offer equivalent access to the
277 |     Corresponding Source in the same way through the same place at no
278 |     further charge.  You need not require recipients to copy the
279 |     Corresponding Source along with the object code.  If the place to
280 |     copy the object code is a network server, the Corresponding Source
281 |     may be on a different server (operated by you or a third party)
282 |     that supports equivalent copying facilities, provided you maintain
283 |     clear directions next to the object code saying where to find the
284 |     Corresponding Source.  Regardless of what server hosts the
285 |     Corresponding Source, you remain obligated to ensure that it is
286 |     available for as long as needed to satisfy these requirements.
287 | 
288 |     e) Convey the object code using peer-to-peer transmission, provided
289 |     you inform other peers where the object code and Corresponding
290 |     Source of the work are being offered to the general public at no
291 |     charge under subsection 6d.
292 | 
293 |   A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 |   A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 |   "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 |   If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 |   The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 |   Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 |   7. Additional Terms.
344 | 
345 |   "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 |   When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 |   Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 |     a) Disclaiming warranty or limiting liability differently from the
366 |     terms of sections 15 and 16 of this License; or
367 | 
368 |     b) Requiring preservation of specified reasonable legal notices or
369 |     author attributions in that material or in the Appropriate Legal
370 |     Notices displayed by works containing it; or
371 | 
372 |     c) Prohibiting misrepresentation of the origin of that material, or
373 |     requiring that modified versions of such material be marked in
374 |     reasonable ways as different from the original version; or
375 | 
376 |     d) Limiting the use for publicity purposes of names of licensors or
377 |     authors of the material; or
378 | 
379 |     e) Declining to grant rights under trademark law for use of some
380 |     trade names, trademarks, or service marks; or
381 | 
382 |     f) Requiring indemnification of licensors and authors of that
383 |     material by anyone who conveys the material (or modified versions of
384 |     it) with contractual assumptions of liability to the recipient, for
385 |     any liability that these contractual assumptions directly impose on
386 |     those licensors and authors.
387 | 
388 |   All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 |   If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 |   Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 |   8. Termination.
408 | 
409 |   You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 |   However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 |   Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 |   Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 |   9. Acceptance Not Required for Having Copies.
436 | 
437 |   You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 |   10. Automatic Licensing of Downstream Recipients.
447 | 
448 |   Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 |   An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 |   You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 |   11. Patents.
472 | 
473 |   A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 |   A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 |   Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 |   In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 |   If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 |   If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 |   A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 |   Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 |   12. No Surrender of Others' Freedom.
541 | 
542 |   If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 |   13. Use with the GNU Affero General Public License.
553 | 
554 |   Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 |   14. Revised Versions of this License.
564 | 
565 |   The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 |   Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 |   If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 |   Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 |   15. Disclaimer of Warranty.
590 | 
591 |   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 |   16. Limitation of Liability.
601 | 
602 |   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 |   17. Interpretation of Sections 15 and 16.
613 | 
614 |   If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |                      END OF TERMS AND CONDITIONS
622 | 
623 |             How to Apply These Terms to Your New Programs
624 | 
625 |   If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 |   To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 |     {one line to give the program's name and a brief idea of what it does.}
635 |     Copyright (C) {year}  {name of author}
636 | 
637 |     This program is free software: you can redistribute it and/or modify
638 |     it under the terms of the GNU General Public License as published by
639 |     the Free Software Foundation, either version 3 of the License, or
640 |     (at your option) any later version.
641 | 
642 |     This program is distributed in the hope that it will be useful,
643 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
644 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 |     GNU General Public License for more details.
646 | 
647 |     You should have received a copy of the GNU General Public License
648 |     along with this program.  If not, see <http://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 |   If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 |     {project}  Copyright (C) {year}  {fullname}
656 |     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 |     This is free software, and you are welcome to redistribute it
658 |     under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 |   You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <http://www.gnu.org/licenses/>.
668 | 
669 |   The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <http://www.gnu.org/philosophy/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------