├── Dockerfile ├── LICENSE.txt ├── README.md ├── liblookup ├── README.md ├── __init__.py └── ntfile.py ├── lookup-service-sqlite.py ├── lookup-service.py ├── preprocess.py └── sqlite-init.py /Dockerfile: -------------------------------------------------------------------------------- 1 | ################################ 2 | ## YodaQA Label Service Image ## 3 | ################################ 4 | 5 | # Proposed image name: label-service 6 | 7 | # Inherit Debian image 8 | FROM debian:jessie 9 | 10 | # Update and install dependencies [cmp. https://docs.docker.com/engine/articles/dockerfile_best-practices/] 11 | RUN apt-get update && apt-get install -y \ 12 | curl \ 13 | git \ 14 | pypy 15 | 16 | # Set the locale 17 | ENV LANG C.UTF-8 18 | ENV LC_ALL C.UTF-8 19 | 20 | RUN git clone https://github.com/brmson/label-lookup.git 21 | # If we were to copy label-service files into image 22 | #ADD ./label-service /label-service 23 | 24 | RUN cd label-lookup 25 | RUN curl -O https://bootstrap.pypa.io/get-pip.py 26 | # If you run this on an actual system instead of a container: The following 3 commands need root privileges 27 | RUN pypy get-pip.py 28 | RUN mv /usr/local/bin/pip ./pypy_pip 29 | RUN ./pypy_pip install flask SPARQLWrapper 30 | 31 | # Same as "export TERM=dumb"; prevents error "Could not open terminal for stdout: $TERM not set" 32 | ENV TERM dumb 33 | 34 | # Define working directory 35 | WORKDIR /label-lookup 36 | 37 | # Expose port 38 | EXPOSE 5000 39 | EXPOSE 5001 40 | 41 | ########## 42 | # BEWARE ##################################################################################### 43 | # With SELinux you need to run chcon -Rt svirt_sandbox_file_t /home//docker/docker_shared/ 44 | ############################################################################################## 45 | 46 | # Can be built with: "docker build -t labels ." 47 | 48 | # docker run -it -v /home/fp/docker/data/labels/:/shared --entrypoint="pypy" -p 5000:5000 labels /label-lookup/lookup-service.py /shared/sorted_list.dat 49 | # docker run -it -v /home/fp/docker/data/labels/:/shared --entrypoint="pypy" -p 5001:5001 labels /label-lookup/lookup-service-sqlite.py /shared/labels.db 50 | # RUN pypy lookup-service.py /shared/sorted_list.dat is done in run command; need to map sorted_list.dat as volume (read-only) 51 | 52 | # Can be used as usual: curl 127.0.0.1:5000/search/AlbaniaPeople 53 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Lookup Services 2 | =============== 3 | 4 | Fuzzy lookup 5 | ------------ 6 | 7 | This is a pseudo-fuzzy search script for DBpedia label lookup inspired by the 8 | kitt.ai paper (http://aclweb.org/anthology/N/N15/N15-3014.pdf) by Xuchen Yao. 9 | 10 | Our motivation is to look up concepts and entities by their name even if it 11 | contains typos or omissions. This is designed as an external service to 12 | conserve memory, allow even remote operation and pair up nicely with SPARQL 13 | endpoints and such. 14 | 15 | Setup 16 | ----- 17 | 18 | Requirements: Couple of gigabytes of disk space for the dataset. 19 | The preprocessing takes about 6GB of RAM; the service runtime stabilizes 20 | on 4.5GB RAM usage. 21 | 22 | ### Data Preprocessing 23 | 24 | Input data (DBpedia-2014): 25 | * http://downloads.dbpedia.org/2014/en/labels_en.nt.bz2 26 | * http://downloads.dbpedia.org/2014/en/page_ids_en.nt.bz2 27 | * http://downloads.dbpedia.org/2014/en/redirects_transitive_en.nt.bz2 28 | (FIXME: Use DBpedia-2015-04 for the movies setting instead?) 29 | 30 | First, we preprocess the dataset to build a single sorted list sorted_list.dat 31 | to speed up next loadings: 32 | 33 | ./preprocess.py labels_en.nt page_ids_en.nt redirects_transitive_en.nt sorted_list.dat 34 | 35 | ### Python Setup 36 | 37 | Our Python executables use PyPy to speed things up (a lot). Just install 38 | it, or change the first line of the script to use Python instead). 39 | However, you will also need to install the Flask module within PyPy 40 | (already installed Python module won't do the trick). Easiest is to 41 | install PyPy-specific pip, then use it to install flask: 42 | 43 | curl -O https://bootstrap.pypa.io/get-pip.py 44 | pypy get-pip.py 45 | mv /usr/local/bin/pip ./pypy_pip 46 | ./pypy_pip install flask 47 | 48 | Usage 49 | ----- 50 | 51 | Just run the script: 52 | 53 | ./lookup-service.py sorted_list.txt 54 | 55 | Wait until a confirmation message shows up and then send requests to 56 | ``http://localhost:5000/search/``, 57 | for example ``http://localhost:5000/search/AlbaniaPeople``. 58 | It returns a json containing the label, canon label and edit distance. 59 | 60 | Alternatively, it is possible to run it in interactive mode, but you have 61 | to change the last line from ``web_init()`` to ``interactive()``. 62 | 63 | Sqlite Lookup 64 | ------------- 65 | This search script is based on 2 papers: 66 | http://nlp.stanford.edu/pubs/crosswikis.pdf 67 | http://ad-publications.informatik.uni-freiburg.de/CIKM_freebase_qa_BH_2015.pdf 68 | 69 | It uses a sqlite database of search strings, wiki URLs and 70 | the probability of the URL given the string. The dataset is located at 71 | 72 | http://www-nlp.stanford.edu/pubs/crosswikis-data.tar.bz2/dictionary.bz2 73 | 74 | The database will be created automatically. 75 | To initialize the database, run 76 | 77 | ./sqlite-init.py labels.db dictionary.bz2 78 | 79 | It will initialize the database and create an index. The resulting size is roughly 12GB. Without the index, the size is 6.6GB, but a query takes 14s. 80 | 81 | Then, start it like this: 82 | 83 | ./lookup-service-sqlite.py labels.db 84 | 85 | It uses the same API as the fuzzy label lookup and should work the same. 86 | To test it, send requests to ``http://localhost:5001/search/`` 87 | 88 | This API has an extra support for returning also the respective enwiki 89 | pageId by querying DBpedia behind the scenes: 90 | ``http://localhost:5001/search/?addPageId=1`` 91 | -------------------------------------------------------------------------------- /liblookup/README.md: -------------------------------------------------------------------------------- 1 | This module contains various routines for manipulating some DBpedia 2 | datasets in a reasonably fast and memory efficient manner. 3 | 4 | To save memory, when mentioning "URL" we actually mean only the path 5 | fragment after /resource/. 6 | -------------------------------------------------------------------------------- /liblookup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brmson/label-lookup/644b4a8d4f6a976cf308d832a9a7531504c2224a/liblookup/__init__.py -------------------------------------------------------------------------------- /liblookup/ntfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities for loading .nt RDF triple files. 3 | """ 4 | 5 | def load_literals(nt_filename, first=0): 6 | """ 7 | From an .nt file, produce a dict mapping between the first and last 8 | elements. 9 | 10 | We make some strong assumptions: 11 | * the first element is DBpedia resource 12 | * the middle element is always the same predicate (which we ignore) 13 | * the last element is a literal 14 | 15 | If the @first parameter is 0, the key in the dict is the name (URL); 16 | if it's 1, the key is the label. 17 | """ 18 | mapping = dict() 19 | with open(nt_filename, "r") as f: 20 | next(f) # a comment comes first 21 | for line in f: 22 | name = line[len('')] 23 | l_start_index = line.find('\"')+1 24 | l_end_index = line.find('\"', l_start_index) 25 | l = line[l_start_index:l_end_index] 26 | if first == 0: 27 | mapping[name] = l 28 | else: 29 | mapping[l] = name 30 | return mapping 31 | 32 | 33 | def load_resources(nt_filename): 34 | """ 35 | From an .nt file, produce a dict mapping between the first and last 36 | elements. 37 | 38 | We make some strong assumptions: 39 | * the first element is DBpedia resource 40 | * the middle element is always the same predicate (which we ignore) 41 | * the last element is DBpedia resource too 42 | """ 43 | mapping = dict() 44 | with open(nt_filename, "r") as f: 45 | next(f) # a comment comes first 46 | for line in f: 47 | field1_o = line.find(' ') + 1 48 | field2_o = line.find(' ', field1_o) + 1 49 | name0 = line[len('')] 50 | name2 = line[field2_o + len('', field2_o)] 51 | mapping[name0] = name2 52 | return mapping 53 | -------------------------------------------------------------------------------- /lookup-service-sqlite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Script for sqlite lookupfrom crossweb dataset 3 | # Usage: ./lookup-service-sqlite.py labels.db 4 | 5 | from SPARQLWrapper import SPARQLWrapper, JSON 6 | import sqlite3 7 | import sys 8 | 9 | from flask import * 10 | app = Flask(__name__) 11 | 12 | db = "labels.db" 13 | dbpurl = 'http://dbpedia.ailao.eu:3030/dbpedia/query' 14 | 15 | 16 | def queryWikipediaId(label): 17 | """ Convert the label to a pageId. """ 18 | if label is None: return None 19 | # first, check if this is a redirect and traverse it 20 | retVal = queryWikipediaIdRedirected(label) 21 | if retVal is not None: 22 | return retVal 23 | sparql = SPARQLWrapper(dbpurl) 24 | sparql.setReturnFormat(JSON) 25 | sparql_query = ''' 26 | PREFIX rdfs: 27 | SELECT DISTINCT ?pageID WHERE { 28 | ?pageID . 29 | } ''' 30 | sparql.setQuery(sparql_query) 31 | res = sparql.query().convert() 32 | retVal = [] 33 | for r in res['results']['bindings']: 34 | retVal.append(r['pageID']['value']) 35 | return retVal[0] if retVal else None 36 | 37 | 38 | def queryWikipediaIdRedirected(label): 39 | """ Convert the label to a pageId, traversing redirect.s """ 40 | if label is None: return None 41 | sparql = SPARQLWrapper(dbpurl) 42 | sparql.setReturnFormat(JSON) 43 | sparql_query = ''' 44 | PREFIX rdfs: 45 | SELECT DISTINCT ?pageID WHERE { 46 | ?tgt . 47 | ?tgt ?pageID . 48 | } ''' 49 | sparql.setQuery(sparql_query) 50 | res = sparql.query().convert() 51 | retVal = [] 52 | for r in res['results']['bindings']: 53 | retVal.append(r['pageID']['value']) 54 | return retVal[0] if retVal else None 55 | 56 | 57 | @app.route('/search/') # Also supports ?addPageId=1 58 | def web_search(name): 59 | print "searching " + name.encode('utf-8') 60 | global dataset 61 | result_list = search(name, addPageId=request.args.get('addPageId', False)) 62 | print "found:" 63 | print result_list[:3] 64 | return jsonify(results=result_list[:3]) 65 | 66 | 67 | def search(name, addPageId=False): 68 | connection = sqlite3.connect(db) 69 | connection.text_factory = str 70 | with connection: 71 | cursor = connection.cursor() 72 | cursor.execute("SELECT label, probability, url from labels as l JOIN urls as u on l.url_id = u.id where label=?", (name,)) 73 | sresult = cursor.fetchall() 74 | # TODO: canonLabel? change dist? 75 | result_list = [] 76 | for r in sresult: 77 | result = { 78 | 'matchedLabel': r[0], 79 | 'canonLabel': r[2], 80 | 'name': r[2], 81 | 'dist': 0, 82 | 'prob': r[1] 83 | } 84 | if addPageId: 85 | # We don't do this by default as YodaQA proper doesn't need 86 | # this. However, some other auxiliary stuff also uses this 87 | # service and typically always needs the pageId rather than 88 | # a label, to follow up in Freebase or a Solr collection. 89 | result['pageId'] = queryWikipediaId(result['name']) 90 | result_list.append(result) 91 | return result_list 92 | 93 | 94 | if __name__ == "__main__": 95 | db = sys.argv[1] 96 | app.run(port=5001, host='::', debug=False, use_reloader=False) 97 | -------------------------------------------------------------------------------- /lookup-service.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/pypy 2 | # Script for pseudo-fuzzy search in a list of labels 3 | # 4 | # Usage: ./lookup-service.py sorted_list.dat 5 | 6 | import sys 7 | import codecs 8 | 9 | from flask import * 10 | app = Flask(__name__) 11 | 12 | edit_threshold = 3 13 | neighbours_to_check = 2 # the checked amount is double, because we look n positions up and n positions down 14 | case_change_cost = 0.5 # edit distance const for any case change required 15 | interpunction_penalty = 0.2 16 | whitespace_penalty = 0.3 17 | apostrophe_with_s_penalty = 0.1 18 | 19 | 20 | def levenshtein(s, t): 21 | ''' From Wikipedia article; Iterative with two matrix rows. ''' 22 | # XXX this can be done better using numPy 23 | if s == t: return 0 24 | elif len(s) == 0: return len(t) 25 | elif len(t) == 0: return len(s) 26 | v0 = [None] * (len(t) + 1) 27 | v1 = [None] * (len(t) + 1) 28 | case_penalty = 0 29 | for i in range(len(v0)): 30 | v0[i] = i 31 | for i in range(len(s)): 32 | v1[0] = i + 1 33 | for j in range(len(t)): 34 | ins_cost = 1 35 | ins_cost2 = 1 36 | if s[i] == t[j]: 37 | cost = 0 38 | elif s[i].lower() == t[j].lower(): 39 | cost = 0 40 | # Tolerate case changes at the beginnings of words. 41 | if not (i == 0 or j == 0 or not s[i-1].isalnum() or not t[j-1].isalnum()): 42 | case_penalty = case_change_cost 43 | else: 44 | cost = 1 45 | ins_cost = get_interpunction_cost(t, j) 46 | ins_cost2 = get_interpunction_cost(s, i) 47 | v1[j + 1] = min(v1[j] + ins_cost, v0[j + 1] + ins_cost2, v0[j] + cost) 48 | for j in range(len(v0)): 49 | v0[j] = v1[j] 50 | return v1[len(t)] + case_penalty 51 | 52 | def get_interpunction_cost(s, i): 53 | ins_cost = 1 54 | if not(s[i].isalnum() or s[i].isspace()): # is interpunction character 55 | ins_cost = interpunction_penalty 56 | elif s[i].isspace(): 57 | ins_cost = whitespace_penalty 58 | elif i>0: 59 | if s[i] == 's' and s[i-1] == '\'': 60 | ins_cost = apostrophe_with_s_penalty 61 | return ins_cost 62 | 63 | # checks the edit distance of 2n neighbours and the target index 64 | def check_neighbours(name, labels, index, reversed): 65 | # check bounds of list 66 | global neighbours_to_check 67 | start = (index - neighbours_to_check) if (index - neighbours_to_check) > 0 else 0 68 | end = (index + neighbours_to_check) if (index + neighbours_to_check) < (len(labels) - 1) else (len(labels) - 1) 69 | res = set() 70 | global edit_threshold 71 | for x in range(start, end): 72 | label = labels[x][0] 73 | searched_name = name 74 | if reversed == True: 75 | label = label[::-1] 76 | searched_name = searched_name[::-1] 77 | if levenshtein(searched_name, label) <= edit_threshold: 78 | res.add(labels[x]) 79 | return res 80 | 81 | 82 | def binary_search(name, labels, reversed): 83 | lname = name.lower() 84 | lower_bound = 0 85 | upper_bound = len(labels) - 1 86 | middle = 0 87 | result = set() 88 | 89 | while lower_bound <= upper_bound: 90 | middle = (lower_bound+upper_bound) // 2 91 | result = result | (check_neighbours(name, labels, middle, reversed)) 92 | if labels[middle][0].lower() < lname: 93 | lower_bound = middle + 1 94 | elif labels[middle][0].lower() > lname: 95 | upper_bound = middle - 1 96 | else: 97 | break # full match 98 | return result 99 | 100 | 101 | class Dataset: 102 | """ 103 | Holding container for the mappings we query - from labels to URLs 104 | """ 105 | def __init__(self, labels, canon_label_map): 106 | self.labels = labels 107 | self.reversed_labels = map(lambda x: (x[0][::-1], x[1]), labels) 108 | self.reversed_labels.sort(key=lambda x: x[0].lower()) 109 | self.canon_label_map = canon_label_map 110 | 111 | @staticmethod 112 | def load_from_file(list_filename): 113 | print('loading labels') 114 | labels = [] 115 | canon_label_map = dict() 116 | with codecs.open(list_filename, "r", encoding="utf-8") as f: 117 | for line in f: 118 | label, url, pageID, isCanon = line.rstrip().split("\t") 119 | labels.append((label, url)) 120 | if bool(int(isCanon)): 121 | canon_label_map[url] = label 122 | print len(labels) 123 | 124 | return Dataset(labels, canon_label_map) 125 | 126 | def search(self, name): 127 | result = set() 128 | result = result | binary_search(name, self.labels, reversed=False) 129 | result = result | set([(r[0][::-1], r[1]) for r in binary_search(name[::-1], self.reversed_labels, reversed=True)]) 130 | result_list = [{ 131 | 'matchedLabel': r[0], 132 | 'canonLabel': self.canon_label_map[r[1]], 133 | 'name': r[1], 134 | 'dist': levenshtein(name, r[0]), 135 | 'prob': "0" 136 | } for r in result if r[1] in self.canon_label_map] 137 | result_list.sort(key=lambda x: x['dist']) 138 | return result_list 139 | 140 | 141 | @app.route('/search/') 142 | def web_search(name): 143 | print "searching " + name.encode("utf-8") 144 | global dataset 145 | result_list = dataset.search(name) 146 | print "found:" 147 | print result_list[:3] 148 | return jsonify(results=result_list[:3]) 149 | 150 | 151 | def web_init(list_filename): 152 | global dataset 153 | dataset = Dataset.load_from_file(list_filename) 154 | app.run(port=5000, host='::', debug=True, use_reloader=False) 155 | 156 | 157 | def interactive(list_filename): 158 | global dataset 159 | dataset = Dataset.load_from_file(list_filename) 160 | while (True): 161 | name = raw_input("lets look for: ") 162 | if (name == "exit"): 163 | break 164 | if (name == "setNeighbours"): 165 | global neighbours_to_check 166 | next_num = int(input("current neighbour count is "+str(neighbours_to_check)+", please input new number ")) 167 | neighbours_to_check = next_num 168 | continue 169 | if (name == "setThreshold"): 170 | global edit_threshold 171 | next_num = int(input("current edit threshold is "+str(edit_threshold)+", please input new number ")) 172 | edit_threshold = next_num 173 | continue 174 | sorted_list = dataset.search(name) 175 | print sorted_list[:3] 176 | return 177 | 178 | 179 | if __name__ == "__main__": 180 | list_filename = sys.argv[1] 181 | # To use a more interactive console mode, change web_init(...) to 182 | # interactive(...) 183 | web_init(list_filename) 184 | # interactive(list_filename) 185 | -------------------------------------------------------------------------------- /preprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/pypy 2 | # Pre-process the RDF datasets. 3 | # 4 | # Takes a pair of datasets 5 | # * (URL, has-a, label) triplets 6 | # * (URL, has-a, pageID) triplets 7 | # and generates a new dataset of (label, URL_subset, pageID) triplets 8 | # sorted by the label. 9 | # and generates a new dataset of (label, URL, pageID, isCanonical) tuples, 10 | # sorted by the label and the isCanonical flag indicating whether this is 11 | # *not* a redirect. 12 | # 13 | # Usage: ./preprocess.py labels_en.nt page_ids_en.nt redirects_transitive_en.nt sorted_list.dat 14 | 15 | import sys 16 | from liblookup import ntfile 17 | 18 | 19 | def save_to_file(labels, label_map, id_map, redirect_map, list_filename): 20 | with open(list_filename, "w") as f: 21 | for label in labels: 22 | url = label_map[label] 23 | try: 24 | url = redirect_map[url] 25 | isCanon = False 26 | except KeyError: 27 | isCanon = True 28 | try: 29 | pageID = id_map[url] 30 | except KeyError: 31 | # We may follow a redirect from general namespace to specific 32 | # one, e.g. Wikipedia:, but we don't have pageIDs for these. 33 | # This is junk anyway, skip it completely. 34 | continue 35 | f.write("%s\t%s\t%s\t%d\n" % (label, url, pageID, isCanon)) 36 | 37 | 38 | if __name__ == "__main__": 39 | labels_filename, pageIDs_filename, redirects_filename, list_filename = sys.argv[1:] 40 | 41 | print "loading labels" 42 | label_map = ntfile.load_literals(labels_filename, first=1) 43 | print "loading IDs" 44 | id_map = ntfile.load_literals(pageIDs_filename) 45 | print "loading redirects" 46 | redirect_map = ntfile.load_resources(redirects_filename) 47 | 48 | print "loading done, sorting" 49 | labels = label_map.keys() 50 | labels.sort(key=lambda x: x.lower()) 51 | 52 | print "saving to file" 53 | save_to_file(labels, label_map, id_map, redirect_map, list_filename) 54 | -------------------------------------------------------------------------------- /sqlite-init.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/pypy 2 | # Script for sqlite initialisation 3 | # Init: ./sqlite-init.py labels.db dictionary.bz2 4 | 5 | import sqlite3 6 | import bz2 7 | import sys 8 | 9 | 10 | def init_database(db, dict_filename): 11 | print("populating database") 12 | populate_database(db, dict_filename) 13 | print("creating index") 14 | create_index(db) 15 | print("done") 16 | 17 | 18 | def populate_database(db, dict_filename): 19 | connection = sqlite3.connect(db) 20 | input_file = bz2.BZ2File(dict_filename, 'r') 21 | with connection: 22 | cursor = connection.cursor() 23 | cursor.execute('''CREATE TABLE urls 24 | (id integer PRIMARY KEY , url text, UNIQUE(url))''') 25 | cursor.execute('''CREATE TABLE labels 26 | (id integer PRIMARY KEY , label text, probability real, url_id integer, FOREIGN KEY(url_id) REFERENCES urls(id))''') 27 | connection.text_factory = str 28 | counter = 0 29 | for line in input_file: 30 | result = line.split('\t') 31 | label = result[0] 32 | data = result[1].split(' ') 33 | probability = float(data[0]) 34 | url = data[1] 35 | if label != '' and probability > 0.05: 36 | if counter % 1000000 == 0: 37 | print str(counter) 38 | counter += 11 39 | cursor.execute('INSERT OR IGNORE INTO URLS(url) VALUES (?)', (url,)) 40 | cursor.execute('SELECT id from urls where url=?', (url,)) 41 | urlid = cursor.fetchone() 42 | urlid = urlid[0] 43 | cursor.execute('INSERT INTO labels(label, probability, url_id) VALUES (?, ?, ?)', (label, probability, urlid)) 44 | connection.commit() 45 | 46 | 47 | #the index takes another 6gb of memory, but lookup time decreases from 14s to 0.02s 48 | def create_index(db): 49 | connection = sqlite3.connect(db) 50 | cursor = connection.cursor() 51 | cursor.execute("create INDEX index_label on labels(label)") 52 | connection.commit() 53 | 54 | 55 | if __name__ == "__main__": 56 | db = sys.argv[1] 57 | dict_filename = sys.argv[2] 58 | init_database(db, dict_filename) 59 | --------------------------------------------------------------------------------