├── .gitignore ├── LICENSE ├── README.md ├── notes.md ├── pom.xml └── src ├── main └── java │ ├── model │ └── TripletRelation.java │ ├── neo4JUtils │ └── Neo4JDb.java │ ├── nlp │ ├── ParserExtractor.java │ ├── SentenceDetector.java │ └── TripletExtractor.java │ └── util │ ├── Consts.java │ └── Utils.java └── test └── java ├── graphs └── TestNeo4J.java └── nlp ├── BasicActions.java ├── CorefParse.java ├── CoreferenceResolution.java └── TripletExtraction.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.versionsBackup 5 | pom.xml.next 6 | release.properties 7 | 8 | .project 9 | .metadata 10 | bin/** 11 | tmp/** 12 | tmp/**/* 13 | *.tmp 14 | *.bak 15 | *.swp 16 | *~.nib 17 | local.properties 18 | .classpath 19 | .settings/ 20 | .loadpath 21 | 22 | #Models for OpenNLP 23 | src/main/resources/models 24 | src/main/resources/schema 25 | src/main/resources/out 26 | src/main/resources/index 27 | src/main/resources/dict 28 | src/main/resources/coref 29 | src/main/resources/books -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Knowledge Extraction 2 | ==================== 3 | 4 | From Natural Language Text to Graph Database. 5 | 6 | Extract information from big volumes of English language text, process it and store the results in a graph database for easy-to-do computation. Knowledge is represented as triplets of the form subject-action-object. 7 | 8 | ## Abstract 9 | Definition and analysis of a sample approach to the subject of knowledge extraction. 10 | The goal is to extract knowledge from large volumes of English language text, process it and finally store the results in a graph database for future usage. The basic and atomic element of knowledge is represented as a triplet of the form subject-predicate-object. We propose some approaches, solutions, and relative tools for each of the two major aspects (i.e. NLP and graph database storage/processing), while addressing the more specific scenario of open-ended ontology extraction. The experiments are specifically conducted in order to verify the feasibility of the solution with big amount of natural language text. Analysis and comparisons, in terms of performances and quality, of all the approaches encountered are then presented. 11 | 12 | ## License 13 | 14 | Released under version 2.0 of the [Apache License]. 15 | 16 | [Apache license]: http://www.apache.org/licenses/LICENSE-2.0 17 | -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | ## Project Notes 2 | ###TOOLS FOR NLP: 3 | * NLTK 4 | * OpenNLP, UIMA 5 | * CoreNLP 6 | * GATE, RapidMiner 7 | * MAHOUT 8 | 9 | ###RELATION EXTRACTION: 10 | * UIMA (DKPro-core) 11 | * Alchemy 12 | * OpenIE (ReVerb, Ollie) 13 | * RelEx 14 | * ??StandfordNLP 15 | 16 | 17 | ###GENERAL IDEAS 18 | - "person was here", or "person is that" 19 | - difference between generic "retrieval" and the specific one (ex: based on query) 20 | - extract relations, defining the strongest one (by mean of reinforcing its value for each 21 | founded same entry) or finding contradiction (with percentage of validity/strength) 22 | - filter sentences before processing based on some param (e.g. person) 23 | - "Pagerank": return most important entity of the text based on the number on relations to it 24 | - filter triplets by classification (Person, Place ect.) 25 | - construct knowledge base, level of confidence with the validity of something 26 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | edu.sagado 5 | knowledge-extraction 6 | 0.0.1-SNAPSHOT 7 | Knowledge Extraction 8 | From Natural Language Text to Graph Database 9 | 10 | 11 | 12 | junit 13 | junit 14 | 4.11 15 | test 16 | 17 | 18 | org.apache.opennlp 19 | opennlp-tools 20 | 1.5.3 21 | 22 | 23 | org.neo4j 24 | neo4j 25 | 2.1.4 26 | 27 | 28 | org.apache.commons 29 | commons-io 30 | 1.3.2 31 | 32 | 33 | com.sun.net.httpserver 34 | http 35 | 20070405 36 | 37 | 38 | edu.washington.cs.knowitall 39 | reverb-core 40 | 1.4.0 41 | 42 | 43 | org.hamcrest 44 | hamcrest-all 45 | 1.3 46 | test 47 | 48 | 49 | org.springframework 50 | spring-core 51 | 3.0.6.RELEASE 52 | 53 | 58 | 59 | -------------------------------------------------------------------------------- /src/main/java/model/TripletRelation.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | public class TripletRelation { 4 | private String arg1 = ""; // subject 5 | private String relation = ""; // predicate 6 | private String arg2 = ""; // object 7 | private double confidence; 8 | 9 | public TripletRelation() { 10 | } 11 | 12 | public TripletRelation(String arg1, String relation, String arg2) { 13 | setArg1(arg1); 14 | setRelation(relation); 15 | setArg2(arg2); 16 | } 17 | 18 | public String getArg1() { 19 | return arg1; 20 | } 21 | 22 | public void setArg1(String arg1) { 23 | this.arg1 = arg1; 24 | } 25 | 26 | public String getRelation() { 27 | return relation; 28 | } 29 | 30 | public void setRelation(String relation) { 31 | this.relation = relation; 32 | } 33 | 34 | public String getArg2() { 35 | return arg2; 36 | } 37 | 38 | public void setArg2(String arg2) { 39 | this.arg2 = arg2; 40 | } 41 | 42 | public double getConfidence() { 43 | return confidence; 44 | } 45 | 46 | public void setConfidence(double confidence) { 47 | this.confidence = confidence; 48 | } 49 | 50 | public boolean isComplete() { 51 | if (arg1.isEmpty() || relation.isEmpty() || arg2.isEmpty()) 52 | return false; 53 | return true; 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | return "arg1 = " + arg1 + "\n" + "relation = " + relation + "\n" 59 | + "arg2 = " + arg2 + "\n"; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/neo4JUtils/Neo4JDb.java: -------------------------------------------------------------------------------- 1 | package neo4JUtils; 2 | 3 | import java.io.FileWriter; 4 | import java.io.IOException; 5 | import java.util.Iterator; 6 | 7 | import model.TripletRelation; 8 | 9 | import org.apache.commons.lang.StringEscapeUtils; 10 | import org.neo4j.cypher.javacompat.ExecutionEngine; 11 | import org.neo4j.graphdb.DynamicLabel; 12 | import org.neo4j.graphdb.GraphDatabaseService; 13 | import org.neo4j.graphdb.Label; 14 | import org.neo4j.graphdb.Node; 15 | import org.neo4j.graphdb.Relationship; 16 | import org.neo4j.graphdb.RelationshipType; 17 | import org.neo4j.graphdb.ResourceIterator; 18 | import org.neo4j.graphdb.Transaction; 19 | import org.neo4j.graphdb.factory.GraphDatabaseFactory; 20 | import org.neo4j.graphdb.schema.Schema; 21 | import org.neo4j.helpers.collection.IteratorUtil; 22 | import org.neo4j.tooling.GlobalGraphOperations; 23 | 24 | public class Neo4JDb { 25 | private final static String PROP_NAME = "value"; 26 | private final static String PROP_CONFIDENCE = "conf"; 27 | private final static String PROP_OCCURRENCES = "occurrences"; 28 | private final static String LAB_SUBJECT = "subject"; 29 | private final static String LAB_PREDICATE = "predicate"; 30 | private final static String LAB_OBJECT = "object"; 31 | 32 | protected GraphDatabaseService graphDb; 33 | 34 | private static enum RelTypes implements RelationshipType { 35 | RELATES 36 | } 37 | 38 | public Neo4JDb(String dbUrl) { 39 | graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(dbUrl); 40 | registerShutdownHook(graphDb); 41 | } 42 | 43 | // TODO define relation for pronouns 44 | public void insertTriplet(TripletRelation triplet, boolean mergeSubject) { 45 | final Label label_sub = DynamicLabel.label(LAB_SUBJECT); 46 | final Label label_obj = DynamicLabel.label(LAB_OBJECT); 47 | 48 | Node arg1 = null; 49 | if (mergeSubject) { 50 | arg1 = getNode(null, PROP_NAME, triplet.getArg1() 51 | .toLowerCase()); 52 | } 53 | 54 | try (Transaction tx = graphDb.beginTx()) { 55 | if (arg1 == null) { 56 | arg1 = graphDb.createNode(label_sub); 57 | arg1.setProperty(PROP_OCCURRENCES, 1); 58 | arg1.setProperty(PROP_NAME, triplet.getArg1().toLowerCase()); 59 | } else { 60 | int occurences = (Integer) arg1.getProperty(PROP_OCCURRENCES) + 1; 61 | arg1.setProperty(PROP_OCCURRENCES, occurences); 62 | } 63 | Node arg2 = graphDb.createNode(label_obj); 64 | arg2.setProperty(PROP_OCCURRENCES, 1); 65 | arg2.setProperty(PROP_NAME, triplet.getArg2().toLowerCase()); 66 | 67 | Relationship relationship = arg1.createRelationshipTo(arg2, 68 | RelTypes.RELATES); 69 | relationship.setProperty(PROP_NAME, triplet.getRelation() 70 | .toLowerCase()); 71 | //confidence 72 | relationship.setProperty(PROP_CONFIDENCE, triplet.getConfidence()); 73 | tx.success(); 74 | } 75 | } 76 | 77 | // NOTE: here we return a node supposed to be unique by label, key and value 78 | // null if the nodeList is empty 79 | private Node getNode(Label label, String key, Object value) { 80 | Node node = null; 81 | try (Transaction tx = graphDb.beginTx()) { 82 | ResourceIterator nodes = null; 83 | if (label != null){ 84 | nodes = graphDb.findNodesByLabelAndProperty( 85 | label, key, value).iterator(); 86 | } 87 | else { 88 | String validValue = StringEscapeUtils.escapeJavaScript((String) value); 89 | ExecutionEngine engine = new ExecutionEngine(graphDb); 90 | nodes = engine.execute( 91 | "START n=node(*)" 92 | + " WHERE n." + key + "=\"" + validValue + "\"" 93 | + " RETURN n").columnAs("n"); 94 | 95 | } 96 | if (nodes.hasNext()) { 97 | node = nodes.next(); 98 | } 99 | nodes.close(); 100 | } 101 | return node; 102 | } 103 | 104 | public void createIndexes() { 105 | try (Transaction tx = graphDb.beginTx()) { 106 | Schema schema = graphDb.schema(); 107 | schema.indexFor(DynamicLabel.label(LAB_SUBJECT)) 108 | .on(PROP_NAME).create(); 109 | tx.success(); 110 | } 111 | } 112 | 113 | public GraphDatabaseService getDb() { 114 | return graphDb; 115 | } 116 | 117 | public void shutdown() { 118 | graphDb.shutdown(); 119 | } 120 | 121 | public void writeOutContent(String filename) { 122 | GlobalGraphOperations ops = GlobalGraphOperations.at(graphDb); 123 | ExecutionEngine engine = new ExecutionEngine(graphDb); 124 | 125 | try (FileWriter writer = new FileWriter(filename); 126 | Transaction tx = graphDb.beginTx()) { 127 | for (Node n : ops.getAllNodes()) { 128 | writer.write("[" + n.getId() + "," + n.getProperty(PROP_NAME) 129 | + ",["); 130 | Iterator connected = engine.execute( 131 | "START s=node(" + n.getId() 132 | + ") MATCH s-[r]->n RETURN n").columnAs("n"); 133 | for (Node e : IteratorUtil.asIterable(connected)) { 134 | Iterator rel = engine.execute( 135 | "START s=node(" + n.getId() + "), e=node(" 136 | + e.getId() 137 | + ") MATCH s-[r]->e RETURN r.value") 138 | .columnAs("r.value"); 139 | String relVal = rel.hasNext()? rel.next() : ""; 140 | writer.write("[" + e.getId() + "," 141 | + relVal + "],"); 142 | } 143 | writer.write("]]\n"); 144 | } 145 | tx.success(); 146 | } catch (IOException e1) { 147 | e1.printStackTrace(); 148 | } 149 | } 150 | 151 | public TripletRelation getRelation(long id){ 152 | TripletRelation t = new TripletRelation(); 153 | try (Transaction tx = graphDb.beginTx()) { 154 | Relationship rel = graphDb.getRelationshipById(id); 155 | t.setArg1(rel.getStartNode().getProperty(PROP_NAME).toString()); 156 | t.setRelation(rel.getProperty(PROP_NAME).toString()); 157 | t.setArg2(rel.getEndNode().getProperty(PROP_NAME).toString()); 158 | t.setConfidence(Double.valueOf(rel.getProperty(PROP_CONFIDENCE).toString())); 159 | } 160 | return t; 161 | } 162 | 163 | private static void registerShutdownHook(final GraphDatabaseService graphDb) { 164 | // Registers a shutdown hook for the Neo4j instance so that it 165 | // shuts down nicely when the VM exits (even if you "Ctrl-C" the 166 | // running application). 167 | Runtime.getRuntime().addShutdownHook(new Thread() { 168 | @Override 169 | public void run() { 170 | graphDb.shutdown(); 171 | } 172 | }); 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/main/java/nlp/ParserExtractor.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import model.TripletRelation; 9 | import opennlp.tools.cmdline.parser.ParserTool; 10 | import opennlp.tools.parser.Parse; 11 | import opennlp.tools.parser.Parser; 12 | import opennlp.tools.parser.ParserFactory; 13 | import opennlp.tools.parser.ParserModel; 14 | import opennlp.tools.util.Span; 15 | import util.Consts; 16 | 17 | public class ParserExtractor { 18 | private static final String LABEL_TOP = "TOP"; 19 | private static final String LABEL_SENTENCE = "S"; 20 | private static final String LABEL_NOUN_PHRASE = "NP"; 21 | private static final String LABEL_VERBAL_PHRASE = "VP"; 22 | 23 | private static final String LABEL_NAME_PREFIX = "NN"; 24 | private static final String LABEL_VERB_PREFIX = "VB"; 25 | 26 | private SentenceDetector sentenceDetector; 27 | private Parser parser; 28 | 29 | public ParserExtractor() { 30 | sentenceDetector = new SentenceDetector(Consts.EN_SENT_MODEL); 31 | 32 | try (InputStream modelIn = ParserExtractor.class.getClassLoader() 33 | .getResourceAsStream(Consts.EN_PARSER_MODEL);){ 34 | ParserModel model = new ParserModel(modelIn); 35 | parser = ParserFactory.create(model); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } 39 | }; 40 | 41 | public List extractRelationsFromText(String text){ 42 | List relations = new ArrayList(); 43 | List sentences = sentenceDetector.detectSentencesIn(text); 44 | 45 | for (String sentence : sentences) { 46 | relations.add(extractRelationFromSentence(sentence)); 47 | } 48 | 49 | return relations; 50 | } 51 | 52 | public TripletRelation extractRelationFromSentence(String sentence){ 53 | TripletRelation rel = new TripletRelation(); 54 | 55 | Parse p = parseSentence(sentence); 56 | if (p != null){ 57 | rel = new TripletRelation(ParserExtractor.getSubject(p), 58 | ParserExtractor.getPredicate(p), 59 | ParserExtractor.getObject(p) ); 60 | } 61 | else { 62 | System.out.println("no valid parse from parseSentence"); 63 | } 64 | 65 | return rel; 66 | } 67 | 68 | public Parse parseSentence(String sentence){ 69 | Parse topParses[] = ParserTool.parseLine(sentence, parser, 1); 70 | if (topParses.length == 0) 71 | return null; 72 | else 73 | return topParses[0]; 74 | } 75 | 76 | // TODO add possibility of multiple Ss and PP 77 | public static String getSubject(final Parse parse) { 78 | if (parse.getType().equals(LABEL_TOP)) { 79 | return getSubject(parse.getChildren()[0]); 80 | } 81 | 82 | if (parse.getType().equals(LABEL_SENTENCE)) { 83 | for (Parse child : parse.getChildren()) { 84 | if (child.getType().equals(LABEL_NOUN_PHRASE)) { 85 | return getSubject(child); 86 | } 87 | } 88 | } 89 | if (parse.getType().equals(LABEL_NOUN_PHRASE)) { 90 | return getFirstOccurenceForType(parse, LABEL_NAME_PREFIX); 91 | } 92 | 93 | return ""; 94 | } 95 | 96 | public static String getPredicate(final Parse parse) { 97 | if (parse.getType().equals(LABEL_TOP)) { 98 | return getPredicate(parse.getChildren()[0]); 99 | } 100 | 101 | if (parse.getType().equals(LABEL_SENTENCE)) { 102 | for (Parse child : parse.getChildren()) { 103 | if (child.getType().equals(LABEL_VERBAL_PHRASE)) { 104 | return getPredicate(child); 105 | } 106 | } 107 | return ""; 108 | } 109 | if (parse.getType().equals(LABEL_VERBAL_PHRASE)) { 110 | return getFirstOccurenceForType(parse, LABEL_VERB_PREFIX); 111 | } 112 | 113 | return ""; 114 | } 115 | 116 | public static String getObject(final Parse parse) { 117 | String object = ""; 118 | if (parse.getType().equals(LABEL_TOP)) { 119 | return getObject(parse.getChildren()[0]); 120 | } 121 | 122 | if (parse.getType().equals(LABEL_SENTENCE)) { 123 | for (Parse child : parse.getChildren()) { 124 | if (child.getType().equals(LABEL_VERBAL_PHRASE)) { 125 | object = getObject(child); 126 | if (!object.isEmpty()){ 127 | return object; 128 | } 129 | } 130 | } 131 | return object; 132 | } 133 | if (parse.getType().equals(LABEL_VERBAL_PHRASE)) { 134 | return getFirstOccurenceForType(parse, LABEL_NAME_PREFIX); 135 | } 136 | 137 | return object; 138 | } 139 | 140 | public static String getConstituent(final Parse parse, final String syntactic_cat, 141 | String lexical_cat) { 142 | String object = ""; 143 | if (parse.getType().equals(LABEL_TOP)) { 144 | return getConstituent(parse.getChildren()[0], syntactic_cat, lexical_cat); 145 | } 146 | 147 | if (parse.getType().equals(LABEL_SENTENCE)) { 148 | for (Parse child : parse.getChildren()) { 149 | if (child.getType().equals(syntactic_cat)) { 150 | object = getConstituent(child, syntactic_cat, lexical_cat); 151 | if (!object.isEmpty()){ 152 | return object; 153 | } 154 | } 155 | } 156 | return object; 157 | } 158 | if (parse.getType().equals(syntactic_cat)) { 159 | return getFirstOccurenceForType(parse, lexical_cat); 160 | } 161 | 162 | return object; 163 | } 164 | 165 | // public static String getObject(Parse parse){} 166 | 167 | private static String getFirstOccurenceForType(final Parse parse, 168 | final String typePrefix) { 169 | 170 | //TODO ADD PRP 171 | // For now we are only checking the prefix 172 | 173 | // check current 174 | if (parse.getType().length() > 1 175 | && parse.getType().substring(0, 2).equals(typePrefix)) { 176 | Span span = parse.getSpan(); 177 | String text = parse.getText().substring(span.getStart(), 178 | span.getEnd()); 179 | return text; 180 | } 181 | 182 | // check children (breadth) 183 | for (Parse child : parse.getChildren()) { 184 | if (child.getType().length() > 1 185 | && child.getType().substring(0, 2).equals(typePrefix)) { 186 | Span span = child.getSpan(); 187 | String text = child.getText().substring(span.getStart(), 188 | span.getEnd()); 189 | if (!text.isEmpty()) 190 | return text; 191 | } 192 | } 193 | 194 | // recursively check for children (deep) 195 | for (Parse child : parse.getChildren()) { 196 | String text = getFirstOccurenceForType(child, typePrefix); 197 | if (!text.isEmpty()) 198 | return text; 199 | } 200 | 201 | return ""; 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/main/java/nlp/SentenceDetector.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | import opennlp.tools.sentdetect.SentenceDetectorME; 9 | import opennlp.tools.sentdetect.SentenceModel; 10 | 11 | public class SentenceDetector { 12 | private SentenceDetectorME sentenceDetector; 13 | 14 | public SentenceDetector(String modelPath) { 15 | try (InputStream modelIn = SentenceDetector.class.getClassLoader() 16 | .getResourceAsStream(modelPath);){ 17 | SentenceModel model = new SentenceModel(modelIn); 18 | sentenceDetector = new SentenceDetectorME(model); 19 | } catch (IOException e) { 20 | e.printStackTrace(); 21 | } 22 | } 23 | 24 | public List detectSentencesIn(String text) { 25 | return Arrays.asList(sentenceDetector.sentDetect(text)); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/nlp/TripletExtractor.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import model.TripletRelation; 8 | import util.Consts; 9 | import edu.washington.cs.knowitall.extractor.ReVerbExtractor; 10 | import edu.washington.cs.knowitall.extractor.conf.ConfidenceFunction; 11 | import edu.washington.cs.knowitall.extractor.conf.ReVerbOpenNlpConfFunction; 12 | import edu.washington.cs.knowitall.nlp.ChunkedSentence; 13 | import edu.washington.cs.knowitall.nlp.OpenNlpSentenceChunker; 14 | import edu.washington.cs.knowitall.nlp.extraction.ChunkedBinaryExtraction; 15 | 16 | public class TripletExtractor { 17 | private OpenNlpSentenceChunker chunker; 18 | private SentenceDetector detector; 19 | private ReVerbExtractor reverb; 20 | 21 | public TripletExtractor() { 22 | try { 23 | chunker = new OpenNlpSentenceChunker(); 24 | } catch (IOException e) { 25 | e.printStackTrace(); 26 | } 27 | detector = new SentenceDetector(Consts.EN_SENT_MODEL); 28 | reverb = new ReVerbExtractor(); 29 | } 30 | 31 | public List extractRelationsFromText(String text){ 32 | List relations = new ArrayList(); 33 | List sentences = detector.detectSentencesIn(text); 34 | 35 | for (String sentence : sentences) { 36 | relations.add(extractRelationFromSentence(sentence)); 37 | } 38 | 39 | return relations; 40 | } 41 | 42 | public TripletRelation extractRelationFromSentence(String sentence){ 43 | TripletRelation rel = new TripletRelation(); 44 | ChunkedSentence chunkedSent = chunker.chunkSentence(sentence); 45 | ConfidenceFunction confFunc = null; 46 | try { 47 | confFunc = new ReVerbOpenNlpConfFunction(); 48 | } catch (Exception e) { 49 | e.printStackTrace(); 50 | } 51 | for (ChunkedBinaryExtraction extr : reverb.extract(chunkedSent)) { 52 | double conf = confFunc.getConf(extr); 53 | rel.setConfidence(conf); 54 | 55 | rel.setArg1(extr.getArgument1().getText()); 56 | rel.setRelation(extr.getRelation().getText()); 57 | rel.setArg2(extr.getArgument2().getText()); 58 | } 59 | 60 | return rel; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/util/Consts.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | public class Consts { 4 | private Consts(){} 5 | 6 | public static final String EN_SENT_MODEL = "models/en/en-sent.bin"; 7 | public static final String IT_SENT_MODEL = "models/it/it-sent.bin"; 8 | public static final String EN_TOKEN_MODEL = "models/en/en-token.bin"; 9 | public static final String EN_NER_MODEL = "models/en/en-ner-person.bin"; 10 | public static final String IT_POS_MODEL = "models/it/it-pos_perceptron.bin"; 11 | public static final String EN_POS_MODEL = "models/en/en-pos-maxent.bin"; 12 | public static final String EN_CHUNK_MODEL = "models/en/en-chunker.bin"; 13 | public static final String EN_PARSER_MODEL = "models/en/en-parser-chunking.bin"; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/util/Utils.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | import java.io.File; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | import model.TripletRelation; 9 | import opennlp.tools.parser.Parse; 10 | import opennlp.tools.parser.chunking.Parser; 11 | import opennlp.tools.util.Span; 12 | 13 | public class Utils { 14 | private Utils(){} 15 | 16 | public static void printParseTree(Parse p, int deep) { 17 | if (p.getType().length() > 1 && p.getType().substring(0, 2).equals(Parser.TOK_NODE)) 18 | return; 19 | 20 | char[] spaces = new char[deep*2]; 21 | Arrays.fill(spaces, ' '); 22 | Span span = p.getSpan(); 23 | System.out.println(new String(spaces) + p.getType() + " -- " + p.getText().substring(span.getStart(), 24 | span.getEnd())); 25 | for (Parse child : p.getChildren()) { 26 | printParseTree(child, new Integer(deep + 1)); 27 | } 28 | } 29 | 30 | public static void printList(List list) { 31 | for (T s : list){ 32 | System.out.println(s); 33 | System.out.println("--------------------"); 34 | } 35 | } 36 | 37 | public static long countValidTriplets(List tList) { 38 | return tList.stream().filter(rel -> rel.isComplete()).count(); 39 | } 40 | 41 | public static List getAllFilenames(String folderPath){ 42 | File folder = new File(folderPath); 43 | File[] listOfFiles = folder.listFiles(); 44 | List filenames = new ArrayList(); 45 | 46 | for (int i = 0; i < listOfFiles.length; i++) { 47 | if (listOfFiles[i].isFile()) { 48 | filenames.add(listOfFiles[i].getName()); 49 | //System.out.println(listOfFiles[i].getName()); 50 | } 51 | } 52 | 53 | return filenames; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/graphs/TestNeo4J.java: -------------------------------------------------------------------------------- 1 | package graphs; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.List; 6 | import java.util.Random; 7 | 8 | import model.TripletRelation; 9 | import neo4JUtils.Neo4JDb; 10 | import nlp.TripletExtractor; 11 | 12 | import org.apache.commons.io.IOUtils; 13 | import org.junit.After; 14 | import org.junit.AfterClass; 15 | import org.junit.Before; 16 | import org.junit.BeforeClass; 17 | import org.junit.Test; 18 | 19 | import util.Utils; 20 | 21 | public class TestNeo4J { 22 | private final String DB_URL = "C:/Users/Alex/Documents/Neo4j/rispTest"; 23 | private final String TEST_FILE = "books/myConv/sherryRisp.txt"; 24 | private final String TEST_OUT = "src/main/resources/out/test.txt"; 25 | private final String TEST_FOLDER = "src/main/resources/books/myConv/"; 26 | private final String TEST_FOLDER_NAME = "books/myConv/"; 27 | 28 | Neo4JDb db; 29 | 30 | @BeforeClass 31 | public void prepareTestDatabase() { 32 | db = new Neo4JDb(DB_URL); 33 | //TODO 34 | //db.createIndexes(); 35 | } 36 | 37 | @AfterClass 38 | public void destroyTestDatabase() { 39 | db.shutdown(); 40 | } 41 | 42 | @Test 43 | public void testPrint() { 44 | db.writeOutContent(TEST_OUT); 45 | } 46 | 47 | @Test 48 | public void SingleNeo4JTest() { 49 | neo4JTest(TEST_FILE); 50 | } 51 | 52 | @Test 53 | public void extractRandomRels() { 54 | int NUM_RELS = 30; 55 | int MAX_REL_ID = 400; 56 | Random rand = new Random(); 57 | TripletRelation rel; 58 | for (int i=0; i docs = Utils.getAllFilenames(TEST_FOLDER); 69 | long startTime = System.currentTimeMillis(); 70 | System.out.println("Start iterateFolderExtraction"); 71 | for (String doc : docs){ 72 | neo4JTest(TEST_FOLDER_NAME + doc); 73 | } 74 | long endTime = System.currentTimeMillis(); 75 | long duration = endTime - startTime; 76 | System.out.println("iterateFolderExtraction extraction ended in " + duration + "millis"); 77 | } 78 | 79 | private void neo4JTest(String fileName) { 80 | System.out.println("----------------"); 81 | System.out.println(fileName); 82 | 83 | String text = null; 84 | try (InputStream testArticle = TestNeo4J.class.getClassLoader() 85 | .getResourceAsStream(fileName);) { 86 | text = IOUtils.toString(testArticle, "UTF-8"); 87 | } catch (IOException e) { 88 | e.printStackTrace(); 89 | } 90 | 91 | System.out.println("Current text length: " + text.length()); 92 | 93 | TripletExtractor tExt = new TripletExtractor(); 94 | long startTime = System.currentTimeMillis(); 95 | System.out.println("Start extraction to Neo4J"); 96 | List rels = tExt.extractRelationsFromText(text); 97 | int numValid = 0; 98 | double confTot = 0.0; 99 | for (TripletRelation rel : rels) { 100 | if (rel.isComplete()){ 101 | db.insertTriplet(rel, false); 102 | numValid++; 103 | confTot += rel.getConfidence(); 104 | } 105 | } 106 | long endTime = System.currentTimeMillis(); 107 | long duration = endTime - startTime; 108 | System.out.println("Extraction ended in " + duration + "millis"); 109 | System.out.println(rels.size() + "triplets founded"); 110 | System.out.println(numValid + "valid triplets"); 111 | System.out.println("Average conf = " + confTot/numValid); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/nlp/BasicActions.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | import opennlp.tools.chunker.ChunkerME; 9 | import opennlp.tools.chunker.ChunkerModel; 10 | import opennlp.tools.namefind.NameFinderME; 11 | import opennlp.tools.namefind.TokenNameFinderModel; 12 | import opennlp.tools.postag.POSModel; 13 | import opennlp.tools.postag.POSTaggerME; 14 | import opennlp.tools.tokenize.Tokenizer; 15 | import opennlp.tools.tokenize.TokenizerME; 16 | import opennlp.tools.tokenize.TokenizerModel; 17 | import opennlp.tools.util.Span; 18 | 19 | import org.apache.commons.io.IOUtils; 20 | import org.junit.Test; 21 | 22 | import util.Consts; 23 | import util.Utils; 24 | 25 | public class BasicActions { 26 | private static final String TEST_TEXT = "books/myConv/myRisp.txt";; 27 | private static final String TEST_PHRASE = "Michael McGinn is the mayor of Seattle."; 28 | 29 | @Test 30 | public void testSentenceDetector(){ 31 | SentenceDetector detector = new SentenceDetector(Consts.EN_SENT_MODEL); 32 | try (InputStream testArticle = BasicActions.class.getClassLoader() 33 | .getResourceAsStream(TEST_TEXT);) { 34 | 35 | String text = IOUtils.toString(testArticle, "UTF-8"); 36 | List sentences = detector.detectSentencesIn(text); 37 | Utils.printList(sentences); 38 | } catch (IOException e) { 39 | e.printStackTrace(); 40 | } 41 | } 42 | 43 | public String[] testTokenizer(){ 44 | String[] tokens = {}; 45 | try (InputStream modelIn = BasicActions.class.getClassLoader() 46 | .getResourceAsStream(Consts.EN_TOKEN_MODEL);) { 47 | 48 | TokenizerModel tokenModel = new TokenizerModel(modelIn); 49 | Tokenizer tokenizer = new TokenizerME(tokenModel); 50 | tokens = tokenizer.tokenize(TEST_PHRASE); 51 | System.out.println(Arrays.toString(tokens)); 52 | } catch (IOException e) { 53 | e.printStackTrace(); 54 | } 55 | return tokens; 56 | } 57 | 58 | public String[] testTagger(){ 59 | String[] tags = {}; 60 | try (InputStream modelIn = BasicActions.class.getClassLoader(). 61 | getResourceAsStream(Consts.EN_POS_MODEL);){ 62 | 63 | POSModel posModel = new POSModel(modelIn); 64 | POSTaggerME tagger = new POSTaggerME(posModel); 65 | tags = tagger.tag(testTokenizer()); 66 | System.out.println(Arrays.toString(tags)); 67 | } catch (IOException e) { 68 | e.printStackTrace(); 69 | } 70 | return tags; 71 | } 72 | 73 | @Test 74 | public void testNameFinder(){ 75 | try (InputStream modelIn = BasicActions.class.getClassLoader() 76 | .getResourceAsStream(Consts.EN_NER_MODEL);){ 77 | 78 | TokenNameFinderModel model = new TokenNameFinderModel(modelIn); 79 | NameFinderME nameFinder = new NameFinderME(model); 80 | Span nameSpans[] = nameFinder.find(testTokenizer()); 81 | System.out.println(Arrays.toString(nameSpans)); 82 | 83 | } catch (IOException e) { 84 | e.printStackTrace(); 85 | } 86 | } 87 | 88 | @Test 89 | public void testChunker(){ 90 | try (InputStream modelIn = BasicActions.class.getClassLoader(). 91 | getResourceAsStream(Consts.EN_CHUNK_MODEL);){ 92 | 93 | String[] tokens = testTokenizer(); 94 | String[] tags = testTagger(); 95 | 96 | ChunkerModel chunkerModel = new ChunkerModel(modelIn); 97 | ChunkerME chunker = new ChunkerME(chunkerModel); 98 | String chunks[] = chunker.chunk(tokens, tags); 99 | System.out.println(Arrays.toString(chunks)); 100 | } catch (IOException e) { 101 | e.printStackTrace(); 102 | } 103 | } 104 | 105 | 106 | } -------------------------------------------------------------------------------- /src/test/java/nlp/CorefParse.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashMap; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import opennlp.tools.coref.DiscourseEntity; 10 | import opennlp.tools.coref.mention.DefaultParse; 11 | import opennlp.tools.coref.mention.MentionContext; 12 | import opennlp.tools.parser.Parse; 13 | import opennlp.tools.parser.chunking.Parser; 14 | import opennlp.tools.util.Span; 15 | 16 | public class CorefParse { 17 | private Map parseMap; 18 | private List parses; 19 | 20 | public CorefParse(List parses, DiscourseEntity[] entities) { 21 | this.parses = parses; 22 | parseMap = new HashMap(); 23 | for (int ei = 0, en = entities.length; ei < en; ei++) { 24 | if (entities[ei].getNumMentions() > 1) { 25 | for (Iterator mi = entities[ei].getMentions(); mi 26 | .hasNext();) { 27 | MentionContext mc = mi.next(); 28 | Parse mentionParse = ((DefaultParse) mc.getParse()) 29 | .getParse(); 30 | parseMap.put(mentionParse, ei + 1); 31 | // System.err.println("CorefParse: "+mc.getParse().hashCode()+" -> "+ 32 | // (ei+1)); 33 | } 34 | } 35 | } 36 | } 37 | 38 | public void show() { 39 | for (int pi = 0, pn = parses.size(); pi < pn; pi++) { 40 | Parse p = parses.get(pi); 41 | show(p); 42 | System.out.println(); 43 | } 44 | } 45 | 46 | public void print() { 47 | for (int pi = 0, pn = parses.size(); pi < pn; pi++) { 48 | Parse p = parses.get(pi); 49 | print(p, 0); 50 | System.out.println(); 51 | } 52 | } 53 | 54 | private void show(Parse p) { 55 | int start; 56 | start = p.getSpan().getStart(); 57 | if (!p.getType().equals(Parser.TOK_NODE)) { 58 | System.out.print("("); 59 | System.out.print(p.getType()); 60 | if (parseMap.containsKey(p)) { 61 | System.out.print("#" + parseMap.get(p)); 62 | } 63 | // System.out.print(p.hashCode()+"-"+parseMap.containsKey(p)); 64 | System.out.print(" "); 65 | } 66 | Parse[] children = p.getChildren(); 67 | for (int pi = 0, pn = children.length; pi < pn; pi++) { 68 | Parse c = children[pi]; 69 | Span s = c.getSpan(); 70 | if (start < s.getStart()) { 71 | System.out.print(p.getText().substring(start, s.getStart())); 72 | } 73 | show(c); 74 | start = s.getEnd(); 75 | } 76 | System.out.print(p.getText().substring(start, p.getSpan().getEnd())); 77 | if (!p.getType().equals(Parser.TOK_NODE)) { 78 | System.out.print(")"); 79 | } 80 | } 81 | 82 | private void print(Parse p, int deep) { 83 | if (p.getType().length() > 1 && p.getType().substring(0, 2).equals(Parser.TOK_NODE)) 84 | return; 85 | 86 | char[] spaces = new char[deep*2]; 87 | Arrays.fill(spaces, ' '); 88 | Span span = p.getSpan(); 89 | System.out.print(new String(spaces) + p.getType() + " -- " + p.getText().substring(span.getStart(), 90 | span.getEnd())); 91 | if (parseMap.containsKey(p)) { 92 | System.out.print("#" + parseMap.get(p)); 93 | } 94 | System.out.print("\n"); 95 | for (Parse child : p.getChildren()) { 96 | print(child, new Integer(deep + 1)); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/java/nlp/CoreferenceResolution.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import opennlp.tools.coref.DiscourseEntity; 12 | import opennlp.tools.coref.Linker; 13 | import opennlp.tools.coref.LinkerMode; 14 | import opennlp.tools.coref.TreebankLinker; 15 | import opennlp.tools.coref.mention.DefaultParse; 16 | import opennlp.tools.coref.mention.Mention; 17 | import opennlp.tools.parser.Parse; 18 | 19 | import org.apache.commons.io.IOUtils; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | import util.Consts; 24 | 25 | public class CoreferenceResolution { 26 | Linker linker = null; 27 | ParserExtractor parser = null; 28 | SentenceDetector detector = null; 29 | String text; 30 | 31 | @Before 32 | public void initLinker() throws IOException { 33 | try { 34 | linker = new TreebankLinker("src/main/resources/coref", 35 | LinkerMode.TEST); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } 39 | 40 | parser = new ParserExtractor(); 41 | detector = new SentenceDetector(Consts.EN_SENT_MODEL); 42 | 43 | InputStream testArticle = CoreferenceResolution.class.getClassLoader().getResourceAsStream( 44 | "story1.txt"); 45 | 46 | text = IOUtils.toString(testArticle, "UTF-8"); 47 | } 48 | 49 | @Test 50 | public void testOpenNLPResolution() throws IOException { 51 | assertNotNull(linker); 52 | assertNotNull(parser); 53 | assertNotNull(detector); 54 | 55 | List sentences = detector.detectSentencesIn(text); 56 | int sentenceNumber = 0; 57 | List document = new ArrayList(); 58 | List parses = new ArrayList(); 59 | for (String sentence : sentences) { 60 | Parse p = parser.parseSentence(sentence); 61 | if (p == null){ 62 | System.out.println("Null parse for: " + sentence); 63 | continue; 64 | } 65 | parses.add(p); 66 | Mention[] extents = linker.getMentionFinder().getMentions( 67 | new DefaultParse(p, sentenceNumber)); 68 | // construct new parses for mentions which don't have 69 | // constituents. 70 | for (int ei = 0, en = extents.length; ei < en; ei++) { 71 | // System.err.println("PennTreebankLiner.main: "+ei+" "+extents[ei]); 72 | 73 | if (extents[ei].getParse() == null) { 74 | // not sure how to get head index, but its not used at 75 | // this point. 76 | Parse snp = new Parse(p.getText(), extents[ei].getSpan(), 77 | "NML", 1.0, 0); 78 | p.insert(snp); 79 | extents[ei].setParse(new DefaultParse(snp, sentenceNumber)); 80 | } 81 | 82 | } 83 | document.addAll(Arrays.asList(extents)); 84 | sentenceNumber++; 85 | } 86 | 87 | DiscourseEntity[] entities = linker.getEntities(document 88 | .toArray(new Mention[document.size()])); 89 | new CorefParse(parses, entities).print(); 90 | sentenceNumber = 0; 91 | document.clear(); 92 | parses.clear(); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/nlp/TripletExtraction.java: -------------------------------------------------------------------------------- 1 | package nlp; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.List; 6 | 7 | import model.TripletRelation; 8 | 9 | import org.apache.commons.io.IOUtils; 10 | import org.junit.Before; 11 | import org.junit.BeforeClass; 12 | import org.junit.Test; 13 | 14 | import util.Utils; 15 | 16 | public class TripletExtraction { 17 | private final String TEST_FILE = "books/myConv/myRisp.txt"; 18 | private final String TEST_FOLDER = "src/main/resources/books/myConv/"; 19 | private final String TEST_FOLDER_NAME = "books/myConv/"; 20 | private final Boolean COMPARE = false; 21 | 22 | private ParserExtractor pExt; 23 | private TripletExtractor tExt; 24 | 25 | @BeforeClass 26 | public void initExtractors() throws IOException { 27 | pExt = new ParserExtractor(); 28 | tExt = new TripletExtractor(); 29 | } 30 | 31 | @Test 32 | public void singleExtraction() throws IOException{ 33 | experimentExtraction(TEST_FILE); 34 | } 35 | 36 | @Test 37 | public void iterateFolderExtraction() throws IOException{ 38 | List docs = Utils.getAllFilenames(TEST_FOLDER); 39 | long startTime = System.currentTimeMillis(); 40 | System.out.println("Start iterateFolderExtraction"); 41 | for (String doc : docs){ 42 | experimentExtraction(TEST_FOLDER_NAME + doc); 43 | } 44 | long endTime = System.currentTimeMillis(); 45 | long duration = endTime - startTime; 46 | System.out.println("iterateFolderExtraction extraction ended in " 47 | + duration + " millis"); 48 | } 49 | 50 | private void experimentExtraction(String filename){ 51 | String text = null; 52 | try (InputStream testArticle = TripletExtraction.class.getClassLoader() 53 | .getResourceAsStream(filename);) { 54 | text = IOUtils.toString(testArticle, "UTF-8"); 55 | } catch (IOException e) { 56 | e.printStackTrace(); 57 | } 58 | 59 | System.out.println("Current text article length: " + text.length()); 60 | List rels; 61 | 62 | //Using parser extractor 63 | if (COMPARE){ 64 | long startTime = System.currentTimeMillis(); 65 | System.out.println("Start extraction with OpenNLP"); 66 | rels = pExt.extractRelationsFromText(text); 67 | long endTime = System.currentTimeMillis(); 68 | long duration = endTime - startTime; 69 | System.out.println("Extraction ended in " + duration + " millis"); 70 | System.out.println(rels.size() + " triplets founded"); 71 | System.out.println(Utils.countValidTriplets(rels) + " valid triplets"); 72 | } 73 | 74 | //Using triplet extractor (Reverb) 75 | long startTime = System.currentTimeMillis(); 76 | System.out.println("----------------------------"); 77 | System.out.println("Start extraction with Reverb"); 78 | rels = tExt.extractRelationsFromText(text); 79 | long endTime = System.currentTimeMillis(); 80 | long duration = endTime - startTime; 81 | System.out.println("----------------------------"); 82 | Utils.printList(rels); 83 | System.out.println("----------------------------"); 84 | System.out.println("Extraction ended in " + duration + " millis"); 85 | System.out.println(rels.size() + " triplets founded"); 86 | System.out.println(Utils.countValidTriplets(rels) + " valid triplets \n"); 87 | } 88 | } 89 | --------------------------------------------------------------------------------