├── HeidelTime ├── __init__.py └── de.unihd.dbs.heideltime.standalone.jar ├── LICENSE ├── README.md ├── TreeTagger ├── COPYRIGHT ├── FILES ├── README ├── bin │ ├── separate-punctuation │ ├── train-tree-tagger │ └── tree-tagger ├── cmd │ ├── #tree-tagger-danish# │ ├── add-korean-sentence-markers.pl │ ├── chunker-read-lemma.perl │ ├── chunker-write-lemma.perl │ ├── filter-chunker-output-french.perl │ ├── filter-chunker-output-german.perl │ ├── filter-chunker-output.perl │ ├── filter-coordinate-output.perl │ ├── filter-german-tags │ ├── greek-mwls.pl │ ├── lookup.perl │ ├── mwl-lookup.perl │ ├── portuguese-post-tagging │ ├── portuguese-splitter.perl │ ├── reformat-korean-tagger-output.pl │ ├── split-romanian.perl │ ├── tagger-chunker-english │ ├── tagger-chunker-french │ ├── tagger-chunker-german │ ├── tagger-chunker-spanish │ ├── tokenize-korean.pl │ ├── tokenize.pl │ ├── tree-tagger-ancient-greek-beta │ ├── tree-tagger-ancient-greek-utf8 │ ├── tree-tagger-bulgarian │ ├── tree-tagger-catalan │ ├── tree-tagger-czech │ ├── tree-tagger-danish │ ├── tree-tagger-dutch │ ├── tree-tagger-english │ ├── tree-tagger-estonian │ ├── tree-tagger-finnish │ ├── tree-tagger-french │ ├── tree-tagger-galician │ ├── tree-tagger-german │ ├── tree-tagger-greek │ ├── tree-tagger-italian │ ├── tree-tagger-korean │ ├── tree-tagger-latin │ ├── tree-tagger-middle-high-german │ ├── tree-tagger-polish │ ├── tree-tagger-portuguese │ ├── tree-tagger-portuguese-finegrained │ ├── tree-tagger-portuguese2 │ ├── tree-tagger-romanian │ ├── tree-tagger-russian │ ├── tree-tagger-slovak │ ├── tree-tagger-slovenian │ ├── tree-tagger-spanish │ ├── tree-tagger-spanish-ancora │ ├── tree-tagger-swahili │ └── utf8-tokenize.perl ├── doc │ ├── nemlap94.ps │ └── sigdat95.ps ├── install-tagger.sh └── lib │ ├── bulgarian-abbreviations │ ├── bulgarian-mwls │ ├── catalan-abbreviations │ ├── czech-abbreviations-utf8 │ ├── dutch-abbreviations │ ├── english-abbreviations │ ├── english-utf8.par │ ├── estonian-abbreviations-utf8 │ ├── estonian-mwls-utf8 │ ├── finnish-abbreviations-utf8 │ ├── french-abbreviations │ ├── french-abbreviations-utf8 │ ├── french.par │ ├── galician-abbreviations-utf8 │ ├── galician-mwls │ ├── german-abbreviations │ ├── german-abbreviations-utf8 │ ├── german-lexicon-utf8.txt │ ├── german-lexicon.txt │ ├── greek-abbreviations-utf8 │ ├── italian-abbreviations │ ├── latin-abbreviations │ ├── latin-mwls │ ├── middle-high-german-abbreviations-utf8 │ ├── polish-abbreviations-utf8 │ ├── portuguese-abbreviations-utf8 │ ├── romanian-abbreviations │ ├── romanian-tokens │ ├── spanish-abbreviations │ ├── spanish-mwls │ ├── spanish-mwls-utf8 │ └── swahili-abbreviations ├── config.props └── requirements.txt /HeidelTime/__init__.py: -------------------------------------------------------------------------------- 1 | import jpype 2 | 3 | # start the JVM 4 | if not jpype.isJVMStarted() : 5 | jar = "HeidelTime/de.unihd.dbs.heideltime.standalone.jar" 6 | jpype.startJVM(jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=%s" % jar) 7 | 8 | # get the Java classes we want to use 9 | heideltime_resources = jpype.JPackage("de.unihd.dbs.uima.annotator.heideltime.resources") 10 | heideltime_standalone = jpype.JPackage("de.unihd.dbs.heideltime.standalone") 11 | 12 | # constants 13 | LANGUAGES = { 14 | 'english':heideltime_resources.Language.ENGLISH, 15 | 'german':heideltime_resources.Language.GERMAN, 16 | 'dutch':heideltime_resources.Language.DUTCH, 17 | 'italian':heideltime_resources.Language.ITALIAN, 18 | 'spanish':heideltime_resources.Language.SPANISH, 19 | 'arabic':heideltime_resources.Language.ARABIC, 20 | 'french':heideltime_resources.Language.FRENCH, 21 | 'chinese':heideltime_resources.Language.CHINESE, 22 | 'russian':heideltime_resources.Language.RUSSIAN, 23 | 'portuguese':heideltime_resources.Language.PORTUGUESE 24 | } 25 | 26 | DOCUMENTS = { 27 | 'narratives':heideltime_standalone.DocumentType.NARRATIVES, 28 | 'news':heideltime_standalone.DocumentType.NEWS, 29 | 'colloquial':heideltime_standalone.DocumentType.COLLOQUIAL, 30 | 'scientific':heideltime_standalone.DocumentType.SCIENTIFIC 31 | } 32 | 33 | OUTPUTS = { 34 | 'timeml':heideltime_standalone.OutputType.TIMEML, 35 | 'xmi':heideltime_standalone.OutputType.XMI 36 | } 37 | 38 | CONFIG = 'config.props' 39 | 40 | 41 | class HeidelTimeWrapper(): 42 | 43 | def __init__(self, lang, doc=None, output=None): 44 | self.language = LANGUAGES[lang] 45 | if (doc is None): 46 | self.doc_type = DOCUMENTS['news'] 47 | else: 48 | self.doc_type = DOCUMENTS[doc] 49 | if (output is None): 50 | self.output_type = OUTPUTS['timeml'] 51 | else: 52 | self.output_type = OUTPUTS[output] 53 | print(jpype.JPackage("de.unihd.dbs.uima.annotator.heideltime.resources").Language.ENGLISH) 54 | self.heideltime = heideltime_standalone.HeidelTimeStandalone(self.language, self.doc_type, self.output_type, CONFIG) 55 | 56 | def convert_date(slef, day, month, year): 57 | sdf = jpype.java.text.SimpleDateFormat('dd-M-yyyy hh:mm:ss') 58 | str_date = str(day)+'-'+str(month)+'-'+str(year)+' 00:00:00' 59 | return sdf.parse(str_date) 60 | 61 | def parse(self, text, date_ref=None): 62 | if (date_ref is None): 63 | document_creation_date = jpype.java.util.Date() 64 | else: 65 | # convert to Java.util.Date 66 | document_creation_date = self.convert_date(date_ref.day, date_ref.month, date_ref.year) 67 | return self.heideltime.process(text, document_creation_date) 68 | -------------------------------------------------------------------------------- /HeidelTime/de.unihd.dbs.heideltime.standalone.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/HeidelTime/de.unihd.dbs.heideltime.standalone.jar -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Python-HeidelTime 2 | 3 | This is a python wrapper for the multilingual temporal tagger HeidelTime. 4 | 5 | For more information about this temporal tagger, please visit : https://github.com/HeidelTime/heideltime 6 | 7 | Heideltime needs a Part Of Speech Tagger, by default it uses TreeTagger : http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/. 8 | This repo includes TreeTagger scripts for Mac and Linux. Windows users will have to download the appropriate scripts from the TreeTagger home page. 9 | 10 | :exclamation: Disclaimer: this repository is not maintained, you are free to use it and adapt it to your needs but you may not get support if needed. 11 | 12 | # Demo 13 | 14 | 1. Clone the current github repo: `git clone https://github.com/amineabdaoui/python-heideltime` 15 | 2. Install its requirements: `pip install -r requirements.txt` 16 | 3. Create a python file in the same directory with the following code: 17 | ```python 18 | import HeidelTime 19 | hw = HeidelTime.HeidelTimeWrapper('english') 20 | hw.parse('Neil Armstrong was born in 1930') 21 | ``` 22 | 23 | # Future Work 24 | - Encapsulate the returned tagged string in a python object 25 | - Find a way to install the appropriate POS tagger files according to the user OS. 26 | - Package all the necessary parts in the same python package and release it on PyPi. 27 | 28 | `Pull requests` are welcome to improve this wrapper (which is in an early stage). 29 | -------------------------------------------------------------------------------- /TreeTagger/COPYRIGHT: -------------------------------------------------------------------------------- 1 | 2 | ************************ 3 | * License Conditions * 4 | ************************ 5 | 6 | concerning the use and distribution of the program system 'TreeTagger'. 7 | 8 | The license is granted by 9 | Helmut Schmid, Markusstraße 8, 72760 Reutlingen, Germany 10 | Email schmid@cis.lmu.de 11 | 12 | 1. The user can freely use TreeTagger for evaluation, research, and 13 | teaching purposes. Any commercial usage is forbidden without a 14 | separate commercial license available from the licensor. 15 | 16 | 2. The user is not allowed to distribute or sell the system to third 17 | parties without written permission from the licensor. 18 | 19 | NO WARRANTY 20 | 21 | 3. BECAUSE THE SYSTEM IS LICENSED FREE OF CHARGE, WE PROVIDE 22 | ABSOLUTELY NO WARRANTY, TO THE EXTENT PERMITTED BY APPLICABLE STATE 23 | LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE LICENSOR PROVIDES THE 24 | SYSTEM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR 25 | IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 26 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK 27 | AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH THE USER. 28 | SHOULD THE SYSTEM PROVE DEFECTIVE, THE USER ASSUMES THE COST OF ALL 29 | NECESSARY SERVICING, REPAIR OR CORRECTION. 30 | 31 | 4. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW WILL THE LICENSOR BE 32 | LIABLE TO THE USER FOR DAMAGES, INCLUDING ANY LOST PROFITS, LOST 33 | MONIES, OR OTHER SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 34 | OUT OF THE USE OR INABILITY TO USE (INCLUDING BUT NOT LIMITED TO LOSS 35 | OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY THIRD 36 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAM) 37 | THE PROGRAM, EVEN IF THE USER HAS BEEN ADVISED OF THE POSSIBILITY OF 38 | SUCH DAMAGES, OR FOR ANY CLAIM BY ANY OTHER PARTY. 39 | 40 | 41 | The wording of this license agreement has been adapted from the 42 | license of the ALF system by Michael Hanus, Max-Planck-Institut 43 | Saarbruecken and the GnuEmacs General Public License (c) 1991 Free 44 | Software Foundation. 45 | -------------------------------------------------------------------------------- /TreeTagger/FILES: -------------------------------------------------------------------------------- 1 | 2 | This package contains the TreeTagger, a probabilistic part-of-speech 3 | tagger developed by Helmut Schmid. All rights are reserved by Helmut 4 | Schmid. 5 | 6 | Files contained in this package: 7 | 8 | - FILES this file 9 | - COPYRIGHT Copyright notice 10 | - README How to use the tagger 11 | - bin/train-tree-tagger training program 12 | - bin/tree-tagger tagger programm 13 | - bin/separate-punctuation program for tokenization (used by the shell scripts) 14 | - cmd/lookup.perl Perl script for pretagging 15 | - doc/nemlap94.ps paper describing the TreeTagger 16 | - doc/sigdat95.ps paper describing the TreeTagger 17 | 18 | This package can be downloaded at 19 | http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger 20 | 21 | Also available at this URL: 22 | - parameter files 23 | - shell scripts which convert text to the format required by the tagger 24 | - papers about the TreeTagger 25 | 26 | The shell script package should be unpacked in the same directory as the 27 | tagger package and the parameter files should be decompressed and moved 28 | to the lib subdirectory. 29 | -------------------------------------------------------------------------------- /TreeTagger/README: -------------------------------------------------------------------------------- 1 | 2 | /*****************************************************************************/ 3 | /* How to use the TreeTagger */ 4 | /*****************************************************************************/ 5 | 6 | 7 | The TreeTagger consists of two programs: train-tree-tagger is used to 8 | create a parameter file from a lexicon and a handtagged corpus. 9 | tree-tagger expects a parameter file and a text file as arguments and 10 | annotates the text with part-of-speech tags. The file formats are 11 | described below. By default, the programs are located in the ./bin 12 | sub-directory. 13 | 14 | If either of the programs is called without arguments, it will print 15 | information about its usage. 16 | 17 | 18 | Tagging 19 | ------- 20 | 21 | Tagging is done with the tree-tagger program. It requires at least one 22 | command line argument, the parameter file. If no input file is specified, 23 | input will be read from stdin. If neither an input file nor an output file 24 | is specified, the tagger will print to stdout. 25 | 26 | tree-tagger {-eps } 27 | {-base} {-proto} {-sgml} {-token} {-lemma} {-beam } 28 | 29 | Description of the command line arguments: 30 | 31 | * : Name of a parameter file which was created with the 32 | train-tree-tagger program. 33 | * : Name of the file which is to be tagged. Each token in this 34 | file must be on a separate line. Tokens may contain blanks. It is possible 35 | to override the lexical information contained in the parameter file of the 36 | tagger by specifying a list of possible tags after a token. This list has 37 | to be preceded by a tab character. The tags are optionally followed by a 38 | floating point value to specify the probability of the tag. Adding such 39 | tag information in the tagger's input is sometimes useful to ensure that 40 | certain text-specific expressions are tagged properly. 41 | Punctuation marks must be on separate lines as well. Clitics (like "'s", 42 | "'re", and "'d" in English or "-la" and "-t-elle" in French) should be 43 | separated if they were separated in the training data. (The French and 44 | English parameter files available by ftp, expect separation of clitics). 45 | Sample input file: 46 | He 47 | moved 48 | to 49 | New York City NP 1.0 50 | . 51 | * : Name of the file to which the tagger should write its output. 52 | 53 | Further optional command line arguments: 54 | 55 | * -token: tells the tagger to print the words also. 56 | * -lemma: tells the tagger to print the lemmas of the words also. 57 | * -sgml: tells the tagger to ignore tokens starting with '<' and ending 58 | with '>' (SGML tags). 59 | 60 | The options below are for advanced users. Read the papers on the TreeTagger 61 | to fully understand their meaning. 62 | 63 | * -proto: If this option is specified, the tagger creates a file named 64 | "lexicon-protocol.txt", which contains information about the degree of 65 | ambiguity and about the other possible tags of a word form. The part of 66 | the lexicon in which the word form has been found is also indicated. 'f' 67 | means fullform lexicon and 's' means affix lexicon. 'h' means that the 68 | word contains a hyphen and that the part of the word following the 69 | hyphen has been found in the fullform lexicon. 70 | * -eps : Value which is used to replace zero lexical frequencies. 71 | This is the case if a word/tag pair is contained in the lexicon but not 72 | in the training corpus. The default is 0.1. The choice of this parameter 73 | has some minor influence on tagging accuracy. 74 | * -beam : If the tagger is slow, this option can be used to speed it up. 75 | Good values for are in the range 0.001-0.00001. 76 | * -base: If this option is specified, only lexical information is used 77 | for tagging but no contextual information about the preceding tags. 78 | This option is only useful in order to obtain a baseline result 79 | to which to compare the actual tagger output. 80 | 81 | There is another tagger program called "tree-tagger-flush" which 82 | flushes the output after reading an empty line. It expects a parameter 83 | file as argument and reads from stdin and writes to stdout. No command 84 | line options are supported. This program might be useful for 85 | implementing wrappers. 86 | 87 | 88 | 89 | Training 90 | -------- 91 | 92 | Training is done with the *train-tree-tagger* program. It expects at least 93 | four command line arguments which are described below. 94 | 95 | train-tree-tagger 96 | {-cl } {-dtg } 97 | {-ecw } {-atg } {-st } 98 | 99 | Description of the command line arguments: 100 | 101 | * : name of a file which contains the fullform lexicon. Each line 102 | of the lexicon corresponds to one word form and contains the word form 103 | itself followed by a Tab character and a sequence of tag-lemma pairs. 104 | The tags and lemmata are separated by whitespace. 105 | Example: 106 | 107 | aback RB aback 108 | abacuses NNS abacus 109 | abandon VB abandon VBP abandon 110 | abandoned JJ abandoned VBD abandon VBN abandon 111 | abandoning VBG abandon 112 | 113 | Remark: The tagger doesn't need the lemmata actually. If you do not have 114 | the lemma information or if you do not plan to annotate corpora with 115 | lemmas, you can replace the lemma with a dummy value, e.g. "-". 116 | 117 | * : name of a file which contains a list of open class 118 | tags i.e. possible tags of unknown word forms separated by whitespace. 119 | The tagger will use this information when it encounters unknown words, 120 | i.e. words which are not contained in the lexicon. 121 | Example: (for Penn Treebank tagset) 122 | 123 | FW JJ JJR JJS NN NNS NP NPS RB RBR RBS VB VBD VBG VBN VBP VBZ 124 | 125 | * : name of a file which contains tagged training data. The data 126 | must be in one-word-per-line format. This means that each line contains 127 | one token and one tag in that order separated by a tabulator. 128 | Punctuation marks are considered as tokens and must have been tagged as well. 129 | Example: 130 | 131 | Pierre NP 132 | Vinken NP 133 | , , 134 | 61 CD 135 | years NNS 136 | 137 | * : name of the file in which the resulting tagger parameters 138 | are stored. 139 | 140 | The following parameters are optional. Read the papers on the TreeTagger to 141 | fully understand their meaning. 142 | 143 | * -st : the end-of-sentence part-of-speech tag, i.e. the tag which 144 | is assigned to sentence punctuation like ".", "!", "?". 145 | Default is "SENT". It is important to set this option properly, if your 146 | tag for sentence punctuation is not "SENT". 147 | * -cl : number of preceding words forming the statistical 148 | context. The default is 2 which corresponds to a trigram context. For 149 | small training corpora and/or large tagsets, it could be useful to reduce 150 | this parameter to 1. 151 | * -dtg : Threshold - If the information gain at a 152 | leaf node of the decision tree is below this threshold, the node is deleted. 153 | The default value is 0.7. 154 | * -ecw : weight of the equivalence class based probability 155 | estimates. The default is 0.15. 156 | * -atg Threshold - If the information gain at a leaf of an 157 | affix tree is below this threshold, it is deleted. The default is 1.2. 158 | 159 | The accuracy of the TreeTagger is usually slightly improved, if different 160 | settings of the above parameters are tested and the best combination is 161 | chosen. 162 | -------------------------------------------------------------------------------- /TreeTagger/bin/separate-punctuation: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/bin/separate-punctuation -------------------------------------------------------------------------------- /TreeTagger/bin/train-tree-tagger: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/bin/train-tree-tagger -------------------------------------------------------------------------------- /TreeTagger/bin/tree-tagger: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/bin/tree-tagger -------------------------------------------------------------------------------- /TreeTagger/cmd/#tree-tagger-danish#: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/danish-abbreviations-utf8 14 | PARFILE=${LIB}/danish-utf8.par 15 | 16 | $TOKENIZER -i -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/add-korean-sentence-markers.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | use utf8; 6 | use open ':utf8'; # all open() use UTF-8 7 | use open ':std'; # standard filehandles too 8 | 9 | my $sf_seen = 1; # SF tag encountered in previous segment 10 | my $sf_context = 1; # possible ordinal number encountered. 11 | # A following SF tag no longer signals a sentence boundary. 12 | 13 | while (<>) { 14 | if ($_ eq "\n" && $sf_seen) { 15 | print "\n"; 16 | $sf_seen = 0; 17 | } 18 | if ($_ eq "\n") { 19 | if (!$sf_seen) { 20 | print; 21 | $sf_seen = 1; 22 | } 23 | $_ = ''; 24 | } 25 | elsif ($_ eq "\n") { 26 | $sf_seen = 1; 27 | $_ = ''; 28 | } 29 | print; 30 | if ($_ eq "\n" && $sf_seen) { 31 | print "\n"; 32 | } 33 | $sf_seen = 1 if /\tSF\t/ and $sf_context; 34 | if (/^([0-9]+|[a-z]|[ivx]+)\t/) { 35 | $sf_context = 0 36 | } 37 | elsif (!/^<.*>\n$/) { 38 | $sf_context = 1 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /TreeTagger/cmd/chunker-read-lemma.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | ################################################################### 4 | ### ### 5 | ### File: chunker-read-lemma.perl ### 6 | ### Author: Dennis Spohr (spohrds@ims.uni-stuttgart.de) ### 7 | ### Purpose: Bypass for obtaining lemmas with french chunker ### 8 | ### (see also chunker-write-lemma.perl) ### 9 | ### Created: Tue Feb 13 2007 ### 10 | ### ### 11 | ################################################################### 12 | 13 | # wait for 5 seconds to ensure that a portion of the lemmas has been 14 | # written to the file before attempting to read from the file 15 | sleep(5); 16 | 17 | open(IN,") { 20 | 21 | # print chunker output 22 | chomp; 23 | print; 24 | 25 | # attach lemma 26 | $in = ; 27 | if (defined $in) { 28 | print "\t$in" if defined $in; 29 | } else { 30 | print "\n"; 31 | } 32 | 33 | } 34 | 35 | close(IN); 36 | 37 | # remove lemma file 38 | system("rm lemmas.txt"); 39 | -------------------------------------------------------------------------------- /TreeTagger/cmd/chunker-write-lemma.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | ################################################################### 4 | ### ### 5 | ### File: chunker-write-lemma.perl ### 6 | ### Author: Dennis Spohr (spohrds@ims.uni-stuttgart.de) ### 7 | ### Purpose: Bypass for obtaining lemmas with french chunker ### 8 | ### (see also chunker-read-lemma.perl) ### 9 | ### Created: Tue Feb 13 2007 ### 10 | ### ### 11 | ################################################################### 12 | 13 | open(OUT,">lemmas.txt"); 14 | 15 | # flush filehandle for unbuffered output 16 | my $ofh = select OUT; 17 | $| = 1; 18 | select $ofh; 19 | 20 | while (<>) { 21 | 22 | chomp; 23 | (@F)=split("\t"); 24 | 25 | # print token and tag/chunk to STDOUT 26 | # print lemma to lemma file 27 | if ($#F==0){ 28 | print OUT "\n"; 29 | print "$_\n"; 30 | } else { 31 | print OUT "$F[2]\n"; 32 | print "$F[0]-$F[1]\n"; 33 | } 34 | 35 | } 36 | 37 | close(OUT); 38 | -------------------------------------------------------------------------------- /TreeTagger/cmd/filter-chunker-output-french.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/cmd/filter-chunker-output-french.perl -------------------------------------------------------------------------------- /TreeTagger/cmd/filter-chunker-output-german.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use Getopt::Std; 4 | getopts('t'); 5 | 6 | print doc_start(); 7 | 8 | $n = 0; 9 | while (<>) { 10 | s/.-SBAR$/O/; 11 | 12 | if (/^(.*)-(.*)\t(.*)\/(.*)$/) { 13 | $token[$n] = $1; 14 | $tag[$n] = $2; 15 | $chunk[$n] = $4; 16 | if ($chunk[$n] =~ /^(.*)-(.*)$/) { 17 | $flag[$n] = $1; 18 | $chunk[$n] = $2; 19 | } 20 | else { 21 | undef $flag[$n]; 22 | undef $chunk[$n]; 23 | } 24 | print_sentence() if $token[$n] eq '.'; 25 | $n++; 26 | } 27 | 28 | else { 29 | $markup[$n] .= $_; 30 | } 31 | } 32 | 33 | print_sentence(); 34 | print doc_end(); 35 | 36 | 37 | sub print_sentence { 38 | my($i,$chunk); 39 | 40 | for( $i=0; $i<=$n; $i++ ) { 41 | 42 | if ($flag[$i] eq 'I' && $chunk ne $chunk[$i]) { 43 | $flag[$i] = 'B'; 44 | } 45 | 46 | if ($flag[$i] eq 'B') { 47 | if (defined $chunk) { 48 | $cetags[$i-1] = end_tag($chunk); 49 | } 50 | $chunk = $chunk[$i]; 51 | $cbtags[$i] .= start_tag($chunk[$i]); 52 | } 53 | 54 | # German chunker uses E-flags for PCs 55 | elsif ($flag[$i] eq 'E') { 56 | if ($chunk[$i] eq $chunk) { 57 | $cetags[$i] = end_tag($chunk); 58 | undef $chunk; 59 | } 60 | elsif ($chunk[$i] eq "PC" && $chunk eq "NC") { 61 | for( $k=$i-1; $k>=0; $k-- ) { 62 | if ($chunk[$k] eq "NC") { 63 | $chunk[$k] = "PC"; 64 | } 65 | if ($flag[$k] ne "I") { 66 | last; 67 | } 68 | } 69 | $cbtags[$k] = start_tag($chunk[$i]); 70 | $cetags[$i] = end_tag($chunk[$i]); 71 | undef $chunk; 72 | undef $inPC; 73 | } 74 | } 75 | 76 | elsif ($flag[$i] ne 'I' && defined $chunk) { 77 | $cetags[$i-1] = end_tag($chunk); 78 | undef $chunk; 79 | } 80 | } 81 | 82 | for( $i=0; $i<=$n; $i++ ) { 83 | print $markup[$i]; 84 | print $cbtags[$i]; 85 | print token_and_tag($token[$i],$tag[$i]) if defined $token[$i]; 86 | print $cetags[$i]; 87 | } 88 | 89 | undef @token; 90 | undef @tag; 91 | undef @chunk; 92 | undef @cbtags; 93 | undef @cetags; 94 | undef @flag; 95 | undef @markup; 96 | $n = 0; 97 | } 98 | 99 | sub doc_start { 100 | return '' unless defined $opt_t; 101 | return "\n\n"; 102 | } 103 | 104 | sub doc_end { 105 | return '' unless defined $opt_t; 106 | return "\n"; 107 | } 108 | 109 | sub start_tag { 110 | my $t=shift; 111 | return "<$t>\n" unless defined $opt_t; 112 | return " \n"; 113 | } 114 | 115 | sub end_tag { 116 | my $t=shift; 117 | return "\n" unless defined $opt_t; 118 | return " \n"; 119 | } 120 | 121 | sub token_and_tag { 122 | my ($token,$tag)=@_; 123 | return "$token\t$tag\n" unless defined $opt_t; 124 | return " \n"; 125 | } 126 | -------------------------------------------------------------------------------- /TreeTagger/cmd/filter-chunker-output.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use Getopt::Std; 4 | getopts('t'); 5 | 6 | print doc_start(); 7 | 8 | $n = 0; 9 | while (<>) { 10 | s/.-SBAR$/O/; 11 | 12 | if (/^(.*)-(.*)\t(.*)\/(.*)$/) { 13 | $token[$n] = $1; 14 | $tag[$n] = $2; 15 | $chunk[$n] = $4; 16 | if ($chunk[$n] =~ /^(.*)-(.*)$/) { 17 | $flag[$n] = $1; 18 | $chunk[$n] = $2; 19 | } 20 | else { 21 | undef $flag[$n]; 22 | undef $chunk[$n]; 23 | } 24 | print_sentence() if $token[$n] eq '.'; 25 | $n++; 26 | } 27 | 28 | else { 29 | $markup[$n] .= $_; 30 | } 31 | } 32 | 33 | print_sentence(); 34 | print doc_end(); 35 | 36 | 37 | sub print_sentence { 38 | my($i,$chunk); 39 | 40 | for( $i=0; $i<=$n; $i++ ) { 41 | 42 | if ($flag[$i] eq 'I' && $chunk ne $chunk[$i]) { 43 | $flag[$i] = 'B'; 44 | } 45 | 46 | if ($flag[$i] ne '' && $token[$i] eq '.') { 47 | delete $flag[$i]; 48 | $chunk[$i] = '0'; 49 | } 50 | 51 | if ($flag[$i] eq 'B') { 52 | if (defined $chunk) { 53 | if (($chunk eq 'PC' && $chunk[$i] eq 'NC') || 54 | ($chunk eq 'PP' && $chunk[$i] eq 'NP')) 55 | { 56 | $inPC = $chunk; 57 | } 58 | else { 59 | $cetags[$i-1] = end_tag($chunk); 60 | if (defined $inPC) { 61 | $cetags[$i-1] .= end_tag($inPC); 62 | undef $inPC; 63 | } 64 | } 65 | } 66 | $chunk = $chunk[$i]; 67 | $cbtags[$i] .= start_tag($chunk[$i]); 68 | } 69 | 70 | elsif ($flag[$i] ne 'I' && defined $chunk) { 71 | $cetags[$i-1] = end_tag($chunk); 72 | undef $chunk; 73 | if (defined $inPC) { 74 | $cetags[$i-1] .= end_tag($inPC); 75 | undef $inPC; 76 | } 77 | } 78 | } 79 | 80 | for( $i=0; $i<=$n; $i++ ) { 81 | print $markup[$i]; 82 | print $cbtags[$i]; 83 | print token_and_tag($token[$i],$tag[$i]) if defined $token[$i]; 84 | print $cetags[$i]; 85 | } 86 | 87 | undef @token; 88 | undef @tag; 89 | undef @chunk; 90 | undef @cbtags; 91 | undef @cetags; 92 | undef @flag; 93 | undef @markup; 94 | $n = 0; 95 | } 96 | 97 | sub doc_start { 98 | return '' unless defined $opt_t; 99 | return "\n\n"; 100 | } 101 | 102 | sub doc_end { 103 | return '' unless defined $opt_t; 104 | return "\n"; 105 | } 106 | 107 | sub start_tag { 108 | my $t=shift; 109 | return "<$t>\n" unless defined $opt_t; 110 | return " \n"; 111 | } 112 | 113 | sub end_tag { 114 | my $t=shift; 115 | return "\n" unless defined $opt_t; 116 | return " \n"; 117 | } 118 | 119 | sub token_and_tag { 120 | my ($token,$tag)=@_; 121 | return "$token\t$tag\n" unless defined $opt_t; 122 | return " \n"; 123 | } 124 | -------------------------------------------------------------------------------- /TreeTagger/cmd/filter-coordinate-output.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | ################################################################### 4 | ### ### 5 | ### File: filter-coordinate-output.perl ### 6 | ### Author: Dennis Spohr (spohrds@ims.uni-stuttgart.de) ### 7 | ### Purpose: Combine coordinated NPs into larger NP (also if ### 8 | ### embedded in PP(s)) ### 9 | ### Created: Tue Feb 13 2007 ### 10 | ### ### 11 | ################################################################### 12 | 13 | use strict; 14 | 15 | # queue for storing the content of the NPs 16 | my @queue = (); 17 | 18 | # current line 19 | my $line = ""; 20 | 21 | # (possible) closing PP tags 22 | my $pps = ""; 23 | 24 | # flush STDOUT for unbuffered output 25 | $| = 1; 26 | 27 | while($line = <>) { 28 | 29 | # first NP is encountered 30 | np:if ($line =~ /^/) { 31 | 32 | push(@queue,$line); 33 | 34 | # read until closing NP tag 35 | until ($line =~ /^<\/NP>/) { 36 | $line = <> or last; 37 | push(@queue,$line); 38 | } 39 | unless (defined $line) { 40 | print @queue; 41 | last; 42 | } 43 | 44 | $line = <>; 45 | $pps = ""; 46 | 47 | # read possible closing PP tags 48 | while ($line =~ /^<\/PP>/) { 49 | $pps .= $line; 50 | $line = <>; 51 | } 52 | 53 | # continue as long as ...... 54 | # are encountered 55 | my $continue = 1; 56 | 57 | # number of coordinated NPs 58 | my $coord_np = 0; 59 | 60 | # number of coordinations 61 | my $coord = 0; 62 | 63 | # has current line been pushed onto queue? 64 | my $pushed = 0; 65 | 66 | while ($continue) { 67 | 68 | # encountered COORD 69 | if ($line =~ /^/) { 70 | 71 | push(@queue,$line); 72 | $pushed = 1; 73 | $coord++; 74 | 75 | until ($line =~ /^<\/COORD>/) { 76 | $line = <>; 77 | push(@queue,$line); 78 | } 79 | 80 | $line = <>; 81 | $pushed = 0; 82 | 83 | # encountered NP (immediately after closing COORD tag) 84 | if ($line =~ /^/) { 85 | 86 | push(@queue,$line); 87 | $pushed = 1; 88 | $coord_np++; 89 | 90 | until ($line =~ /^<\/NP>/) { 91 | $line = <>; 92 | push(@queue,$line); 93 | } 94 | 95 | # encountered COORD but no NP afterwards: stop 96 | } else { 97 | $continue = 0; 98 | } 99 | 100 | # encountered no COORD: stop 101 | } else { 102 | $continue = 0; 103 | } 104 | 105 | } 106 | 107 | # found as many COORDs as NPs: put and 108 | # around content and append possible closing PP tags 109 | if ($coord > 0 && $coord_np == $coord) { 110 | 111 | unshift(@queue,"\n"); 112 | push(@queue,"\n"); 113 | push(@queue,$pps) unless $pps eq ""; 114 | 115 | # found at least one coordinated NP, but then one COORD 116 | # without NP, e.g. 117 | # ............ 118 | } elsif ($coord > 1) { 119 | 120 | unshift(@queue,"\n"); 121 | 122 | # append and possible closing PP tags before the final COORD, since 123 | # this isn't part of the coordinated NPs 124 | for (my $i = $#queue; $i >= 0; $i--) { 125 | 126 | if ($queue[$i] =~ /^/) { 127 | $queue[$i] = "\n".$pps."\n"; 128 | last; 129 | } 130 | 131 | } 132 | 133 | # found only one COORD and no NPs 134 | } elsif ($coord == 1) { 135 | 136 | # append possible closing PP tags before COORD 137 | for (my $i = $#queue; $i >= 0; $i--) { 138 | 139 | if ($queue[$i] =~ /^/) { 140 | $queue[$i] = $pps."\n"; 141 | last; 142 | } 143 | 144 | } 145 | 146 | # found no COORD at all (usual case for simple NPs) 147 | } else { 148 | push(@queue,$pps) unless $pps eq ""; 149 | } 150 | 151 | # if current line is again an NP (which may in turn be 152 | # followed by COORDs), go to the start and process the NP 153 | # (i.e. don't read from STDIN first, but process current line); 154 | # this is e.g. the following case: 155 | # ............ 156 | if ($line =~ /^/) { 157 | 158 | print @queue; 159 | undef @queue; 160 | goto np; 161 | 162 | } else { 163 | 164 | push(@queue,$line) unless $pushed; 165 | print @queue; 166 | undef @queue; 167 | 168 | } 169 | 170 | # no opening NP tag 171 | } else { 172 | print $line; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /TreeTagger/cmd/filter-german-tags: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Input format: word tag 4 | # lemma is optional, tag must be in the second position 5 | 6 | awk 'BEGIN{OFS=FS="\t";getline;word=$1;tag=$2;lemma=$3}\ 7 | ((tag~"V.FIN" || tag~"V.INF")\ 8 | && $2~"^[$][.,]"\ 9 | && (word~"[erlu]n$" && word!~"[^aeiou]e*ten$") && word!~".zu.....") \ 10 | {\ 11 | if (flag||zu)\ 12 | {\ 13 | if (tag=="VVFIN") tag="VVINF";\ 14 | else if (tag=="VAFIN") tag="VAINF";\ 15 | else if (tag=="VMFIN") tag="VMINF";\ 16 | }\ 17 | else\ 18 | {\ 19 | if (tag=="VVINF") tag="VVFIN";\ 20 | else if (tag=="VAINF") tag="VAFIN";\ 21 | else if (tag=="VMINF") tag="VMFIN";\ 22 | }\ 23 | }\ 24 | tag~"^V[VAM]FIN$"\ 25 | {\ 26 | flag=1;\ 27 | }\ 28 | tag~"^[$][.,]"\ 29 | {\ 30 | flag=0;\ 31 | }\ 32 | \ 33 | {if (tag=="PTKZU") zu=1; else zu=0;}\ 34 | {print word,tag,lemma; word=$1;tag=$2;lemma=$3;} 35 | END\ 36 | {print word,tag,lemma}' $* | 37 | sed 's/[ ]*$//' 38 | -------------------------------------------------------------------------------- /TreeTagger/cmd/greek-mwls.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | use utf8::all; 6 | 7 | # script recognizes numerical and date expressions in one-token-perl-line input 8 | 9 | my @buffer; 10 | my $state = 0; 11 | while (<>) { 12 | chomp; 13 | next if $_ eq ''; 14 | 15 | if (/^[1-9][0-9]?ης?$/) { 16 | push @buffer, $_; 17 | $state = 1; # 12ης 18 | } 19 | elsif (/^Ιανουαρίου|Φεβρουαρίου|[ΜM]αρτίου|Απριλίου|Μαϊου|Μαΐου|Ιουνίου|Ιούλιου|Ιουλίου|Αυγούστου|Σεπτεμβρίου|Οκτωβρίου|Νοεμβρίου|Δεκεμβρίου|Ιανουάριος?|Φεβρουάριος?|[ΜM]άρτιος?|Απρίλιος?|Μάιος?|Ιούνιος?|Ιούλιος?|Σεπτέμβριος?|Οκτώβριος?|Νοέμβριος?|Δεκέμβριος?|Ιαν.|Σεπτ./) { 20 | push @buffer, $_; 21 | $state = 2; # 12ης Μαρτίου 22 | } 23 | elsif ($state == 2 && /^(19|20)[0-9][0-9]$/) { 24 | push @buffer, $_; 25 | print join('_',@buffer),"\n"; 26 | @buffer = (); 27 | $state = 0; # 12ης Μαρτίου 2012 28 | } 29 | elsif ($state == 0 && /^[0-9]+$/) { 30 | push @buffer, $_; 31 | $state = 3; 32 | } 33 | elsif (($state == 0 || $state == 3) && /^[0-9][0-9][0-9]$/) { 34 | push @buffer, $_; 35 | $state = 3; 36 | } 37 | elsif ($state == 3 && /^[0-9]$/) { 38 | print join('_',@buffer),"\n"; 39 | @buffer = (); 40 | push @buffer, $_; 41 | $state = 3; 42 | } 43 | elsif (($state == 0 || $state == 6) && /^[0-9]([,.0-9-]*[0-9])?$/) { 44 | push @buffer, $_; 45 | $state = 5; # 12 or 12 - 46 | } 47 | elsif (($state == 3 || $state == 5) && $_ eq '-') { 48 | push @buffer, $_; 49 | $state = 6; # 12 - 50 | } 51 | elsif (($state == 3 || $state == 5) && $_ eq '%') { 52 | push @buffer, $_; 53 | print join('_',@buffer),"\n"; 54 | @buffer = (); 55 | $state = 0; # 12 % 56 | } 57 | elsif ($state == 6) { 58 | my $s = pop @buffer; 59 | print join('_',@buffer),"\n"; 60 | @buffer = (); 61 | print "$s\n$_\n"; 62 | $state = 0; 63 | } 64 | else { 65 | if (@buffer) { 66 | print join('_',@buffer),"\n"; 67 | @buffer = (); 68 | } 69 | print "$_\n"; 70 | $state = 0; 71 | } 72 | 73 | # print "state: $state\n"; 74 | } 75 | 76 | # DATE = /^([1-9][0-9]?(ης?)?\t)?(Ιανουαρίου|Φεβρουαρίου|Μαρτίου|Απριλίου|Ιουνίου|Ιούλιο|Ιουλίου|Αυγούστου|Σεπτεμβρίου|Οκτωβρίου|Νοεμβρίου|Δεκεμβρίου)(\t(19|20)[0-9][0-9]$/ or /^[0-9]+[.\/][0-9]+[.\/](19|20)?[0-9][0-9])?$/ 77 | # DIG = /^([0-9]+(_?[,._-]_?))*[0-9]+(_?%)?$/ 78 | -------------------------------------------------------------------------------- /TreeTagger/cmd/lookup.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Usage: lookup.perl * 4 | # Perl script to be used prior to tagging 5 | 6 | # It assigns sets of possible tags to selected word forms. 7 | # The tag information is read from the first argument file. 8 | # The format of this file is: 9 | # [{}]* 10 | # The word form which may contain blanks is followed by a tab character 11 | # and a sequence of tags separated by whitespace. The tags are optionally 12 | # followed by tag probability values in the range from 0.0 to 1.0. 13 | 14 | $LEXICON = shift; 15 | open(LEXICON); 16 | while () { 17 | if (s/^(.*?)\t//) { 18 | $word = $1; 19 | s/\s*\t\s*/\t/g; 20 | s/ +/ /g; 21 | $tag{$word} = $_; 22 | } 23 | } 24 | close(LEXICON); 25 | 26 | while (<>) { 27 | chop(); 28 | s/[ \t\n][ \t\n]*/ /go; 29 | s/^[ \t\n]*(.*[^ \t\n])[ \t\n]*$/$1/go; 30 | if (defined($tag{$_})) { 31 | print $_,"\t",$tag{$_}; 32 | } 33 | else { 34 | print "$_\n"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /TreeTagger/cmd/mwl-lookup.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use Getopt::Std; 4 | getopt('dhf:'); 5 | 6 | # This perl script recognizes multi word units in the input stream 7 | # and puts them on one line. Input must have one-word-per-line format. 8 | # The multi word units are listed in the parameter file with POS tags. 9 | # Each line contains one multi word unit where the individual words 10 | # are separated by blanks followed by a tab character and the blank- 11 | # separated list of POS tags. 12 | # Author: Helmut Schmid, IMS, Uni Stuttgart 13 | 14 | if (!defined($opt_f) || defined($opt_h)) { 15 | $0 =~ s/.*\///; 16 | printf "\nUsage: $0 [-d del] -f mwl-file ...files...\n"; 17 | print "\nOptions:\n"; 18 | print "-d del : Use del as delimiter rather than a blank\n\n"; 19 | die 20 | } 21 | 22 | if (!open(FILE, $opt_f)) { 23 | die "\nCan't open mwl file: ",$opt_f,"\n"; 24 | } 25 | if (defined($opt_d)) { 26 | $del = $opt_d; 27 | } else { 28 | $del = " "; 29 | } 30 | 31 | $N=1; 32 | while () { 33 | chomp(); 34 | @G = split("\t"); 35 | @F = split(/\s+/,$G[0]); 36 | $state = 0; 37 | for($i=0; $i<=$#F; $i++) { 38 | if (!exists($arc{$state,$F[$i]})) { 39 | $arc{$state,$F[$i]} = $N++; 40 | } 41 | $state = $arc{$state,$F[$i]}; 42 | } 43 | $final{$state} = $G[1]; 44 | } 45 | close(FILE); 46 | 47 | 48 | $last = $match = $last_match = 0; 49 | $state = 0; 50 | 51 | for (;;) { 52 | if ($match == $last) { 53 | if (!($token[$last] = <>)) { 54 | if ($last_match > 0) { 55 | print $token[0]; 56 | for ($i=1; $i<=$last_match; $i++) { 57 | print $del,$token[$i]; 58 | } 59 | print "\n"; 60 | } else { 61 | $i=0; 62 | } 63 | for (; $i<$last; $i++) { 64 | print $token[$i],"\n"; 65 | } 66 | last; 67 | } 68 | chomp($token[$last++]); 69 | } 70 | if (($s = $arc{$state, $token[$match]}) || 71 | ($s = $arc{$state, lc($token[$match])}) || 72 | ($s = $arc{$state, ucfirst(lc($token[$match]))})) { 73 | if (exists($final{$s})) { 74 | $last_match = $match; 75 | $last_tag = $final{$s}; 76 | } 77 | $state = $s; 78 | $match++; 79 | } else { 80 | if ($last_match > 0) { 81 | print $token[0]; 82 | for($i=1; $i<=$last_match; $i++) { 83 | print $del,$token[$i]; 84 | } 85 | print "\t$last_tag\n"; 86 | } else { 87 | print $token[0],"\n"; 88 | } 89 | for($i=0,$k=$last_match+1; $k<$last; ) { 90 | $token[$i++] = $token[$k++]; 91 | } 92 | $last = $last - $last_match - 1; 93 | $last_match = $match = 0; 94 | $state = 0; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /TreeTagger/cmd/portuguese-post-tagging: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | binmode STDIN, ':utf8'; 4 | binmode STDOUT, ':utf8'; 5 | use utf8; 6 | 7 | $par = shift(@ARGV); 8 | 9 | if ($par eq "-yes") { 10 | $NP_default_tag = "NP00000" 11 | } 12 | if ($par eq "-no") { 13 | $NP_default_tag = "NP0" 14 | } 15 | 16 | while (<>) { 17 | chop($_); 18 | 19 | if ($_ eq "" || $_ =~ /^<[\w\W]*>$/ || $_ !~ /\t/ ) { 20 | next 21 | } 22 | else { 23 | 24 | ($token, $tag, $lemma) = split("\t", $_); 25 | 26 | # identificar um possível nome próprio (desconhecido ou composto) 27 | # e passar lemma a minúscula: 28 | if ($lemma =~ // && $token =~ /^[A-ZÁÉÍÓÚÂÊÎÔÛÀÈÌÒÙÑÇ]/) { 29 | $tag = $NP_default_tag; 30 | $lemma = lc($token); 31 | } 32 | 33 | print "$token\t$tag\t$lemma\n"; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /TreeTagger/cmd/portuguese-splitter.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | #Splitter de procliticos e contraçoes para o Portugues 4 | 5 | #author: Pablo Gamallo 6 | #date: 18/11/2011 7 | use utf8; 8 | use Encode; 9 | 10 | use locale; 11 | use POSIX; 12 | setlocale(LC_CTYPE,"pt_PT"); 13 | 14 | my $pron = "(me|te|mos|mas|mo|ma|tos|tas|to|ta|o|os|a|as|se|lhe|lhes|lho|lha|lhos|lhas|nos|vos|no-lo|no-los|no-la|no-las|vo-lo|vo-los|vo-la|vo-las|se-nos|se-vos|se-lhe|se-lhes|se-lho|se-lhos|se-lha|se-lhas)"; 15 | 16 | 17 | while ($line = <>) { 18 | $line = decode('UTF8', $line); 19 | chomp $line; 20 | 21 | ############separar ,...############ 22 | $line =~ s/\,\.\.\./\, \.\.\./g; 23 | 24 | ###############Verbos + procliticos###################### 25 | ##caso m-no -> m-o 26 | $line =~ s/m\-n([oa])\b/m\-$1/g; 27 | $line =~ s/m\-n([oa]s)\b/m\-$1/g; 28 | 29 | ##separar cliticos compostos no-lo, vo-lo, se-nos... 30 | $line =~ s/([\-\W\s])(no|vo)-(l[oa]|l[oa]s)([\W\s])/$1$2s $3$4/g; 31 | $line =~ s/([\-\W\s])(no|vo)-(l[oa]|l[oa]s)$/$1$2s $3/g; 32 | $line =~ s/([\-\W\s])(se)-(n[oa]s|lh[eoa]s)([\W\s])/$1$2 $3$4/g; 33 | $line =~ s/([\-\W\s])(se)-(n[oa]s|lh[eoa]s)$/$1$2 $3/g; 34 | 35 | ##separar procliticos de verbos 36 | $line =~ s/([^\-\s]+)\-$pron([\W\s])/$1 $2$3/g; 37 | $line =~ s/([^\-\s]+)\-$pron$/$1 $2/g; 38 | 39 | ##separar cliticos compostos: mo, mos, to, tos, lho, lhos.. 40 | $line =~ s/\bmo\b/me o/g; 41 | $line =~ s/\bmos\b/me os/g; 42 | $line =~ s/\bma\b/me a/g; 43 | # $line =~ s/\bmas\b/me as/g; AMBIGUO 44 | $line =~ s/\bto\b/te o/g; 45 | $line =~ s/\btos\b/te os/g; 46 | $line =~ s/\bta\b/te a/g; 47 | $line =~ s/\btas\b/te as/g; 48 | $line =~ s/\blhoo\b/lhe o/g; 49 | $line =~ s/\blhos\b/lhe os/g; 50 | $line =~ s/\blha\b/lhe a/g; 51 | $line =~ s/\blhas\b/lhe as/g; 52 | 53 | #$line =~ s/\bcho\b/che o/g; 54 | #$line =~ s/\bchos\b/che os/g; 55 | #$line =~ s/\bcha\b/che a/g; 56 | #$line =~ s/\bchas\b/che as/g; 57 | ########################################################## 58 | 59 | 60 | 61 | ###############separar contraçoes nao ambiguas########### 62 | 63 | #ao, à, aos, às 64 | $line =~ s/\bao([\s])/a o$1/g; 65 | $line =~ s/([\W\s])à([\s])/$1 a a$2/g; 66 | $line =~ s/\baos([\s])/a os$1/g; 67 | $line =~ s/([\W\s])às([\s])/$1 a as$2/g; 68 | 69 | #àquele(s), àquela(s). àquilo, aonde 70 | $line =~ s/([\W\s])àquele([\W\s])/$1 a aquel$2/g; 71 | $line =~ s/([\W\s])àqueles([\W\s])/$1 a aqueles$2/g; 72 | $line =~ s/([\W\s])àquela([\W\s])/$1 a aquela$2/g; 73 | $line =~ s/([\W\s])àquelas([\W\s])/$1 a aquelas$2/g; 74 | $line =~ s/([\W\s])àquilo([\W\s])/$1 a aquilo$2/g; 75 | 76 | $line =~ s/\baonde\b/a onde$1/g; 77 | 78 | #co(s), coa(s) /nao aparece no priberam nem no dico de freeling! 79 | # $line =~ s/\bco([\s])/com o$1/g; 80 | # $line =~ s/\bcoa([\s])/com a$1/g; 81 | # $line =~ s/\bcos([\s])/com os$1/g; 82 | # $line =~ s/\bcoas([\s])/com as$1/g; 83 | 84 | #do(s), da(s) 85 | $line =~ s/\bdo([\s])/de o$1/g; 86 | $line =~ s/\bdos([\s])/de os$1/g; 87 | $line =~ s/\bda([\s])/de a$1/g; 88 | $line =~ s/\bdas([\s])/de as$1/g; 89 | 90 | #dum(ns), duma(s) 91 | $line =~ s/\bdum([\s])/de um$1/g; 92 | $line =~ s/\bduns([\s])/de uns$1/g; 93 | $line =~ s/\bduma([\s])/de uma$1/g; 94 | $line =~ s/\bdumas([\s])/de umas$1/g; 95 | 96 | #dele(s), dela(s) 97 | $line =~ s/\bdele\b/de ele/g; 98 | $line =~ s/\bdeles\b/de eles/g; 99 | $line =~ s/\bdela\b/de ela/g; 100 | $line =~ s/\bdelas\b/de elas/g; 101 | 102 | #deste(s), desta(s), desse(s), dessa(s), daquele(s), daquela(s), disto, disso, daquilo 103 | #$line =~ s/([\W\s])deste([\W\s])/$1 de este$2/g; FORMA AMBIGUA 104 | # $line =~ s/\bdestes\b/de estes/g; FORMA AMBIGUA 105 | $line =~ s/\bdesta\b/de esta/g; 106 | $line =~ s/\bdestas\b/de estas/g; 107 | $line =~ s/\bdisto\b/de isto/g; 108 | #$line =~ s/\bdesse\b/de esse/g; FORMA AMBIGUA 109 | #$line =~ s/\bdesses\b/de esses/g; FORMA AMBIGUA 110 | $line =~ s/\bdessa\b/de essa/g; 111 | $line =~ s/\bdessas\b/de essas/g; 112 | $line =~ s/\bdisso\b/de isso/g; 113 | $line =~ s/\bdaquele\b/de aquele/g; 114 | $line =~ s/\bdaquela\b/de aquela/g; ##em galego deveria ser ambigua (adverbio) 115 | $line =~ s/\bdaqueles\b/de aqueles/g; 116 | $line =~ s/\bdaquelas\b/de aquelas/g; 117 | $line =~ s/\bdaquilo\b/de aquilo/g; 118 | 119 | #daqui, daí, ali, acolá, donde, doutro(s), doutra(s) 120 | $line =~ s/\bdaqui\b/de aqui/g; 121 | $line =~ s/\bdaí\b/de aí/g; ##em galego deveria ser ambigua (adverbio) 122 | $line =~ s/\bdacolá\b/de acolá/g; 123 | $line =~ s/\bdonde\b/de onde/g; 124 | $line =~ s/\bdoutro\b/de outro/g; 125 | $line =~ s/\bdoutros\b/de outros/g; 126 | 127 | $line =~ s/\bdoutra\b/de outra/g; 128 | $line =~ s/\bdoutras\b/de outras/g; 129 | 130 | #no(s), na(s) 131 | $line =~ s/\bno([\s])/em o$1/g; 132 | #$line =~ s/\bnos([\s])/em os$1/g; 133 | $line =~ s/\bna([\s])/em a$1/g; 134 | $line =~ s/\bnas([\s])/em as$1/g; 135 | 136 | #dum(ns), duma(s) 137 | $line =~ s/\bnum([\s])/em um$1/g; 138 | $line =~ s/\bnuns([\s])/em uns$1/g; 139 | $line =~ s/\bnuma([\s])/em uma$1/g; 140 | $line =~ s/\bnumas([\s])/em umas$1/g; 141 | 142 | #nele(s) 143 | # $line =~ s/\bnele\b/em ele/g; 144 | $line =~ s/\bneles\b/em eles/g; 145 | 146 | #neste(s), nesta(s), nesse(s), nessa(s), naquele(s), naquela(s), nisto, nisso, naquilo 147 | $line =~ s/\bneste\b/em este/g; 148 | $line =~ s/\bnestes\b/em estes/g; 149 | $line =~ s/\bnesta\b/em esta/g; 150 | $line =~ s/\bnestas\b/em estas/g; 151 | $line =~ s/\bnisto\b/em isto/g; 152 | $line =~ s/\bnesse\b/em esse/g; 153 | $line =~ s/\bnesses\b/em esses/g; 154 | $line =~ s/\bnessa\b/em essa/g; 155 | $line =~ s/\bnessas\b/em essas/g; 156 | $line =~ s/\bnisso\b/em isso/g; 157 | $line =~ s/\bnaquele\b/em aquele/g; 158 | $line =~ s/\bnaquela\b/em aquela/g; 159 | $line =~ s/\bnaqueles\b/em aqueles/g; 160 | $line =~ s/\bnaquelas\b/em aquelas/g; 161 | $line =~ s/\bnaquilo\b/em aquilo/g; 162 | 163 | #pelo(a), polo(s) TODOS AMBIGUOS! 164 | # $line =~ s/\bpelo([\s])/por o$1/g; 165 | # $line =~ s/\bpela([\s])/por a$1/g; 166 | # $line =~ s/\bpelos([\s])/por os$1/g; 167 | # $line =~ s/\bpelas([\s])/por as$1/g; 168 | 169 | # $line =~ s/\bpolo([\s])/por o$1/g; 170 | # $line =~ s/\bpola([\s])/por a$1/g; 171 | # $line =~ s/\bpolos([\s])/por os$1/g; 172 | # $line =~ s/\bpolas([\s])/por as$1/g; 173 | 174 | 175 | #dentre 176 | $line =~ s/\bdentre\b/de entre/g; 177 | 178 | #aqueloutro, essoutro, estoutro 179 | $line =~ s/\baqueloutro\b/aquele outro/g; 180 | $line =~ s/\baqueloutros\b/aqueles outros/g; 181 | $line =~ s/\baqueloutra\b/aquela outra/g; 182 | $line =~ s/\baqueloutras\b/aquelas outras/g; 183 | $line =~ s/\bessoutro\b/esse outro/g; 184 | $line =~ s/\bessoutros\b/esses outros/g; 185 | $line =~ s/\bessoutra\b/essa outra/g; 186 | $line =~ s/\bessoutras\b/esse outras/g; 187 | $line =~ s/\bestoutro\b/este outro/g; 188 | $line =~ s/\bestoutros\b/estes outros/g; 189 | $line =~ s/\bestoutra\b/esta outra/g; 190 | $line =~ s/\bestoutra\b/este outra/g; 191 | 192 | print encode('UTF8',$line),"\n"; 193 | } 194 | 195 | -------------------------------------------------------------------------------- /TreeTagger/cmd/reformat-korean-tagger-output.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | 6 | use utf8; 7 | use open ':utf8'; # all open() use UTF-8 8 | use open ':std'; # standard filehandles too 9 | 10 | while (<>) { 11 | chomp; 12 | if (/^<[^\t]*>$/) { 13 | print "$_\n"; 14 | } 15 | else { 16 | chomp; 17 | my($w,$t,$l) = split(/\t/); 18 | $w =~ s/"/\\"/g; 19 | print "\n"; 20 | my @tags = split(/_/,$t); 21 | my @morphs = split(/_/,$l); 22 | if ($#tags == $#morphs) { 23 | for( my $i=0; $i<=$#tags&&$i<=$#morphs; $i++ ) { 24 | print "$morphs[$i]\t$tags[$i]\n"; 25 | } 26 | } 27 | else { 28 | print "$w\t$t\n" 29 | } 30 | print "\n"; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /TreeTagger/cmd/split-romanian.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | use utf8::all; 6 | 7 | my $filename = shift or die "Error: missing argument file!\n"; 8 | open(FILE,$filename) or die "Error: unable to open file \"$filename\"!\n"; 9 | my %exception; 10 | while() { 11 | chomp; 12 | $exception{$_} = 1; 13 | } 14 | 15 | while (<>) { 16 | chomp; 17 | 18 | my $initial = ''; 19 | my $final = ''; 20 | 21 | if (s/^(şi)(-n)$/$1/i || s/^(mi)(-i)$/$1/i) { 22 | $final = "$2\n$final"; 23 | } 24 | 25 | while (!exists $exception{lc $_} && 26 | s/^(.+)(-(n[ţtcvfgdşs]\p{L}*|m[bp]\p{L}*))$/$1/) { 27 | $final = "$2\n$final"; 28 | } 29 | 30 | if (!exists $exception{lc $_} && 31 | s/^((\p{L}*ntr|şi|s|n|de|i|ţi|pe|te|mi|l|m|c|le|v|dac|ne|ş|parc|d|fii|dă)-)(.+)$/$3/i) { 32 | $initial = "$1\n"; 33 | } 34 | 35 | while (!exists $exception{lc $_} && 36 | s/^(.+)(-(i|l|t|m|şi|ţi|o|se|mi|n|ului|te|un|ul|ai|mă|am|le|s|a|sa|urilor|ilor|astea|vă|aşa|ăla|al|ar|aici|eşte|au|t\'|m\'|mea|ăştia|napoi|ţ\'|aş|alea|aţi|ăsta))$/$1/) { 37 | $final = "$2\n$final"; 38 | } 39 | 40 | while (!exists $exception{lc $_} && 41 | s/^(.+)(-(n[ţtcvfgdş]\p{L}*|m[bp]\p{L}*))$/$1/) { 42 | $final = "$2\n$final"; 43 | } 44 | 45 | print "$initial$_\n$final"; 46 | } 47 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tagger-chunker-english: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/english-abbreviations 14 | PARFILE=${LIB}/english-utf8.par 15 | 16 | PARFILE2=${LIB}/english-chunker-utf8.par 17 | FILTER=${CMD}/filter-chunker-output.perl 18 | 19 | $TOKENIZER -e -a $ABBR_LIST $* | 20 | # remove empty lines 21 | grep -v '^$' | 22 | # tagging 23 | $TAGGER $OPTIONS $PARFILE | 24 | perl -nae 'if ($#F==0){print}else{print "$F[0]-$F[1]\n"}' | 25 | $TAGGER $PARFILE2 -token -sgml -eps 0.00000001 -hyphen-heuristics -quiet | 26 | $FILTER | 27 | $TAGGER $OPTIONS $PARFILE | 28 | perl -pe 's/\tIN\/that/\tIN/;s/\tV[BDHV]/\tVB/' 29 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tagger-chunker-french: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # The training corpus had 17739 sentences, 524004 tokens for 4266 types taken from the French Treebank. Information on the phrasal tagsets can be found at http://www.llf.cnrs.fr/Gens/Abeille/French-Treebank-fr.php 4 | 5 | # Set these paths appropriately 6 | 7 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 8 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 9 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 10 | 11 | OPTIONS1="-token -lemma -sgml" 12 | OPTIONS2="-token -sgml -eps 0.00000001 -hyphen-heuristics -quiet" 13 | 14 | TAGGER=${BIN}/tree-tagger 15 | TOKENIZER=${CMD}/utf8-tokenize.perl 16 | ABBR_LIST=${LIB}/french-abbreviations-utf8 17 | PARFILE1=${LIB}/french-utf8.par 18 | PARFILE2=${LIB}/french-chunker-utf8.par 19 | FILTER=${CMD}/filter-chunker-output-french.perl 20 | FILTERCoordinate=${CMD}/filter-coordinate-output.perl 21 | WRITE_LEMMA=${CMD}/chunker-write-lemma.perl 22 | READ_LEMMA=${CMD}/chunker-read-lemma.perl 23 | 24 | # tagging with Stein's tagger 25 | $TOKENIZER -f -a $ABBR_LIST $* | 26 | 27 | # external lexicon lookup 28 | $CMD/lookup.perl $LIB/french-lexicon.txt | 29 | 30 | $TAGGER $OPTIONS1 $PARFILE1 | 31 | 32 | $WRITE_LEMMA | 33 | 34 | # chunking 35 | $TAGGER $PARFILE2 $OPTIONS2 | 36 | 37 | $READ_LEMMA | 38 | 39 | # filtering 40 | perl -nae '$F[1]=~s/-PP/-PC/; if (/^<.+>\s*$/) {print;} else {print "$F[0]\t$F[1]\t$F[2]\n"}' | 41 | 42 | $FILTER | 43 | 44 | perl -nae 's/<(\/)?PC>/<${1}PP>/; if (/^<.+>\s*$/) {print;} else {print "$F[0]\t$F[1]\t$F[2]\n"}' | 45 | $FILTERCoordinate | 46 | perl -nae 'if (!//) {print}' 47 | 48 | 49 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tagger-chunker-german: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | GERMANTAGGER=${CMD}/tree-tagger-german 10 | TAGGER=${BIN}/tree-tagger 11 | PARFILE=${LIB}/german-utf8.par 12 | PARFILE2=${LIB}/german-chunker-utf8.par 13 | FILTER=${CMD}/filter-chunker-output-german.perl 14 | 15 | $GERMANTAGGER $* | 16 | perl -nae 'if ($#F==0){print}else{print "$F[0]-$F[1]\n"}' | 17 | $TAGGER $PARFILE2 -token -sgml -eps 0.00000001 -hyphen-heuristics -quiet | 18 | $FILTER | 19 | $TAGGER -quiet -token -lemma -sgml $PARFILE 20 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tagger-chunker-spanish: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/spanish-abbreviations 15 | PARFILE=${LIB}/spanish-utf8.par 16 | MWLFILE=${LIB}/spanish-mwls-utf8 17 | 18 | PARFILE2=${LIB}/spanish-chunker-utf8.par 19 | FILTER=${CMD}/filter-chunker-output.perl 20 | 21 | $TOKENIZER -a $ABBR_LIST $* | 22 | # recognition of MWLs 23 | $MWL -f $MWLFILE | 24 | # tagging 25 | $TAGGER $OPTIONS $PARFILE | 26 | perl -ne 'my($w,$t,$l)=split(/\t/);if(defined $t){print "$w-$t\n"}else{print}' | 27 | $TAGGER $PARFILE2 -token -sgml -eps 0.00000001 -hyphen-heuristics -quiet | 28 | $FILTER | 29 | $TAGGER $OPTIONS $PARFILE 30 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tokenize-korean.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | use warnings; 4 | use strict; 5 | use utf8; 6 | use open ':utf8'; # all open() use UTF-8 7 | use open ':std'; # standard filehandles too 8 | 9 | while (<>) { 10 | s/[~∼～]/~/g; 11 | s/[·ㆍ]/·/g; 12 | s/['‘’˙`]/'/g; 13 | s/[“”"]/"/g; 14 | foreach my $word (split) { 15 | if ($word =~ /^<[^<>]+>$/) { 16 | print "$word\n"; 17 | } 18 | else { 19 | $word =~ s/([0-9]+([.,][0-9]+)?|[A-Za-z]+|[.?!…,\/.:;·ㆍㆍ'|"|‘|’|“|”|“|\＊|＇|\＿\＿|#$|\%|&()*+<=>@\ \-\[\\\]\^`{}~°±²¶¼½¿×ß÷ˇ˘˙˚|˝|ΔΦΩαβγδθλμνπφχψω―‘|’|“|”|†‡‥…′″※₁₂₃℃℉ℓ№ΩÅ⅓⅔⅛ Ⅰ\.|Ⅱ\.|Ⅲ\.|Ⅳ\.|Ⅴ\.|Ⅵ\.|Ⅶ\.|Ⅷ\.|Ⅸ\.|Ⅹ\.|ⅰ\.|ⅱ\.|ⅲ\.|ⅳ\.|ⅴ\.|ⅵ\.|ⅶ\.|ⅷ\.|ⅸ\.|ⅹ\.|ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅰⅱⅲⅳⅴⅵⅶⅷⅸⅹ←↑→↓↔↗↙⇒⇔∀∃∇∈√∞∠∥∧∨∩∪∫∴∵∼∽≠≡≥≪≫⊃⊆⊙⌒①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽⑾⒜⒝⒞ⓐⓑⓒⓓⓔⓛⓝⓧ─━│┬┼▒■□▣▦▨▲△▶▷▼▽◀◁◆◇◈○◎●◐◑★☆☎☞♀♂♠♡♣♤♥♧♨♩♪♬♭➀➌、。〃〈〉《》「」『』【】〓〔〕㉠㉡㉢㉣㉤㉥㉦㉧㉨㉩㉪㉫㉬㉭㉮㉯㉰㉱㉲㉳㉴㉵㉶㉷㉸㉹㉺㉻㎉㎎㎏㎐㎖㎗㎜㎝㎞㎡㎢㎥㎾㏄㏊＞～∥★■~∼\-])/ $1 /g; 20 | $word =~ s/ / /g; 21 | $word =~ s/^ //; 22 | $word =~ s/ $//; 23 | $word =~ tr/ /\n/; 24 | print "\n$word\n\n"; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tokenize.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | ######################################################################## 4 | # # 5 | # tokenization script for tagger preprocessing # 6 | # Author: Helmut Schmid, IMS, University of Stuttgart # 7 | # # 8 | # Description: # 9 | # - splits input text into tokens (one token per line) # 10 | # - cuts off punctuation, parentheses etc. # 11 | # - cuts of clitics like n't in English # 12 | # - disambiguates periods # 13 | # - preserves SGML markup # 14 | # - reads the whole file at once and should therefore not be called # 15 | # with very large files # 16 | # # 17 | ######################################################################## 18 | 19 | use Getopt::Std; 20 | getopts('hfeiza:w'); 21 | 22 | # Modify the following lines in order to adapt the tokenizer to other 23 | # types of text and/or languages 24 | 25 | # characters which have to be cut off at the beginning of a word 26 | my $PChar='[{¿¡\'\`\´"»«\202\204\206\207\213\221\222\223\224\225\226\227\233'; 27 | 28 | # characters which have to be cut off at the end of a word 29 | my $FChar=']}\'\`\´\",;:\!\?\%»«\202\204\205\206\207\211\213\221\222\223\224\225\226\227\233'; 30 | 31 | # character sequences which have to be cut off at the beginning of a word 32 | my $PClitic; 33 | 34 | # character sequences which have to be cut off at the end of a word 35 | my $FClitic; 36 | 37 | if (defined($opt_e)) { 38 | # English 39 | $FClitic = "'(s|re|ve|d|m|em|ll)|n't"; 40 | } 41 | if (defined($opt_i)) { 42 | # Italian 43 | $PClitic = "(?:d[ae]ll|nell|all|[ld]|sull|quest|un|senz|tutt|c|s)'"; 44 | } 45 | if (defined($opt_f)) { 46 | # French 47 | $PClitic = "(?:[dcjlmnst]|qu|jusqu|lorsqu|quoiqu|puisqu)'"; 48 | $FClitic = "-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mêmes?|-m'|-moi|-nous|-on|-toi|-tu|-t'|-vous|-en|-y|-ci|-là"; 49 | } 50 | if (defined($opt_z)) { 51 | # Galician 52 | $FClitic = '-la|-las|-lo|-los|-nos'; 53 | } 54 | 55 | 56 | ### NO MODIFICATIONS REQUIRED BEYOND THIS LINE ######################### 57 | 58 | if (defined($opt_h)) { 59 | die " 60 | Usage: tokenize.perl [ options ] ...files... 61 | 62 | Options: 63 | -e English text 64 | -f French text 65 | -i Italian text 66 | -a contains a list of words which are either abbreviations or 67 | words which should not be further split. 68 | -w replace whitespace by SGML tags 69 | "; 70 | } 71 | 72 | # Read the list of abbreviations and words 73 | if (defined($opt_a)) { 74 | die "file not found: $opt_a\n" unless (open(FILE, $opt_a)); 75 | while () { 76 | s/^[ \t\r\n]+//; 77 | s/[ \t\r\n]+$//; 78 | next if (/^\#/ || /^\s$/); # ignore comments 79 | $Token{$_} = 1; 80 | } 81 | close FILE; 82 | } 83 | 84 | 85 | ########################################################################### 86 | # read the file 87 | ########################################################################### 88 | 89 | while (<>) { 90 | 91 | # delete \r 92 | s/[\r\p{XPosixCntrl}]//g; 93 | 94 | # replace blanks within SGML Tags 95 | while (s/(<[^<> ]*)[ \t]([^<>]*>)/$1\377$2/g) { 96 | } 97 | 98 | # replace whitespace by SGML-Tags 99 | if (defined $opt_w) { 100 | s/\n//g; 101 | s/\t//g; 102 | s/ //g; 103 | } 104 | 105 | # restore SGML Tags 106 | tr/\377/ /; 107 | 108 | # put special characters around SGML Tags for tokenization 109 | s/(<[^<>]*>)/\377$1\377/g; 110 | s/(&[^; \t\n\r]*;)/\377$1\377/g; 111 | s/^\377//; 112 | s/\377$//; 113 | s/\377\377/\377/g; 114 | 115 | @S = split("\377"); 116 | for ($i=0; $i<=$#S; $i++) { 117 | $_ = $S[$i]; 118 | # skip lines with only SGML tags 119 | if (/^<.*>$/) { 120 | print $_,"\n"; 121 | } 122 | # normal text 123 | else { 124 | # put spaces at beginning and end 125 | $_ = ' '.$_.' '; 126 | # put spaces around punctuation 127 | s/(\.\.\.)/ ... /g; 128 | s/([;\!\?\/])([^ ])/$1 $2/g; 129 | s/(,)([^ 0-9.])/$1 $2/g; 130 | s/([a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ][.:])([A-ZÀ-Ý])/$1 $2/g; 131 | 132 | @F = split; 133 | for ($j=0; $j<=$#F; $j++) { 134 | my $suffix=""; 135 | $_ = $F[$j]; 136 | # cut off punctuation and brackets 137 | my $finished = 0; 138 | while (!$finished && !defined($Token{$_})) { 139 | 140 | # preceding parentheses 141 | if (s/^(\()([^\)]*)(.)$/$2$3/) { 142 | print "$1\n"; 143 | } 144 | 145 | # following preceding parentheses 146 | elsif (s/^([^(]+)(\))$/$1/) { 147 | $suffix = "$2\n$suffix"; 148 | } 149 | 150 | # other leading punctuation symbols 151 | elsif (s/^([$PChar])(.)/$2/) { 152 | print $1,"\n"; 153 | } 154 | 155 | # other following punctuation symbols 156 | elsif (s/(.)([$FChar])$/$1/) { 157 | $suffix = "$2\n$suffix"; 158 | } 159 | 160 | # cut off dot after punctuation etc. 161 | elsif (s/([$FChar])\.$//) { 162 | $suffix = ".\n$suffix"; 163 | if ($_ eq "") { 164 | $_ = $1; 165 | } else { 166 | $suffix = "$1\n$suffix"; 167 | } 168 | } 169 | else { 170 | $finished = 1; 171 | } 172 | } 173 | 174 | # deal with listed tokens 175 | if (defined($Token{$_})) { 176 | print "$_\n$suffix"; 177 | next; 178 | } 179 | 180 | # deal with abbrevs like U.S.A. 181 | if (/^([A-Za-zÀ-ÿ]\.)+$/) { 182 | print "$_\n$suffix"; 183 | next; 184 | } 185 | 186 | # ordinal numbers 187 | if (/^[0-9]+\.$/ && ! defined($opt_e)) { 188 | print "$_\n$suffix"; 189 | next; 190 | } 191 | 192 | # deal with differnt types of dots 193 | if (/^(..*)\.$/ && $_ ne "...") { 194 | $_ = $1; 195 | $suffix = ".\n$suffix"; 196 | if (defined($Token{$_})) { 197 | print "$_\n$suffix"; 198 | next; 199 | } 200 | } 201 | 202 | # cut clitics off 203 | while (s/^(--)(.)/$2/) { 204 | print $1,"\n"; 205 | } 206 | if (defined $PClitic) { 207 | while (s/^($PClitic)(.)/$2/i) { 208 | print $1,"\n"; 209 | } 210 | } 211 | 212 | while (s/(.)(--)$/$1/) { 213 | $suffix = "$2\n$suffix"; 214 | } 215 | if (defined $FClitic) { 216 | while (s/(.)($FClitic)$/$1/i) { 217 | $suffix = "$2\n$suffix"; 218 | } 219 | } 220 | 221 | print "$_\n$suffix"; 222 | } 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-ancient-greek-beta: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | PARFILE=${LIB}/ancient-greek-beta.par 15 | 16 | $TOKENIZER $* | 17 | # tagging 18 | $TAGGER $OPTIONS $PARFILE 19 | 20 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-ancient-greek-utf8: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | PARFILE=${LIB}/ancient-greek-utf8.par 15 | 16 | $TOKENIZER $* | 17 | # tagging 18 | $TAGGER $OPTIONS $PARFILE 19 | 20 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-bulgarian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/bulgarian-abbreviations 15 | PARFILE=${LIB}/bulgarian-utf8.par 16 | MWLFILE=${LIB}/bulgarian-mwls 17 | 18 | $TOKENIZER -a $ABBR_LIST $* | 19 | # recognition of MWLs 20 | $MWL -f $MWLFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE 23 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-catalan: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/catalan-abbreviations 14 | PARFILE=${LIB}/catalan-utf8.par 15 | 16 | $TOKENIZER -c -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-czech: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/czech-abbreviations-utf8 14 | PARFILE=${LIB}/czech-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-danish: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/danish-abbreviations-utf8 14 | PARFILE=${LIB}/danish-utf8.par 15 | 16 | $TOKENIZER -i -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-dutch: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TAGGER=${BIN}/tree-tagger 12 | TOKENIZER=${CMD}/utf8-tokenize.perl 13 | ABBR_LIST=${LIB}/dutch-abbreviations 14 | PARFILE=${LIB}/dutch-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-english: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/english-abbreviations 14 | PARFILE=${LIB}/english-utf8.par 15 | 16 | $TOKENIZER -e -a $ABBR_LIST $* | 17 | # remove empty lines 18 | grep -v '^$' | 19 | # tagging 20 | $TAGGER $OPTIONS $PARFILE | 21 | perl -pe 's/\tV[BDHV]/\tVB/;s/\tIN\/that/\tIN/;' 22 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-estonian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/estonian-abbreviations-utf8 15 | PARFILE=${LIB}/estonian-utf8.par 16 | MWLFILE=${LIB}/estonian-mwls-utf8 17 | 18 | $TOKENIZER -a $ABBR_LIST $* | 19 | # recognition of MWLs 20 | $MWL -f $MWLFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE 23 | 24 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-finnish: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/finnish-abbreviations-utf8 14 | PARFILE=${LIB}/finnish-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | # tagging 18 | $TAGGER $OPTIONS $PARFILE 19 | 20 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-french: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/french-abbreviations-utf8 14 | PARFILE=${LIB}/french-utf8.par 15 | 16 | $TOKENIZER -f -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-galician: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/galician-abbreviations-utf8 15 | PARFILE=${LIB}/galician-utf8.par 16 | MWLFILE=${LIB}/galician-mwls 17 | 18 | $TOKENIZER -z -a $ABBR_LIST $* | 19 | # recognition of MWLs 20 | $MWL -f $MWLFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE 23 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-german: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml -pt-with-lemma" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/german-abbreviations-utf8 14 | PARFILE=${LIB}/german-utf8.par 15 | LEXFILE=${LIB}/german-lexicon-utf8.txt 16 | FILTER=${CMD}/filter-german-tags 17 | 18 | $TOKENIZER -a $ABBR_LIST $* | 19 | # external lexicon lookup 20 | perl $CMD/lookup.perl $LEXFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE | 23 | # error correction 24 | $FILTER 25 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-greek: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/greek-mwls.pl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/greek-abbreviations-utf8 15 | PARFILE=${LIB}/greek-utf8.par 16 | 17 | $TOKENIZER -a $ABBR_LIST $* | 18 | $MWL | 19 | # tagging 20 | $TAGGER $OPTIONS $PARFILE 21 | 22 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-italian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/italian-abbreviations 14 | PARFILE=${LIB}/italian-utf8.par 15 | 16 | $TOKENIZER -i -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-korean: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TAGGER=${BIN}/tree-tagger 12 | TOKENIZER=${CMD}/tokenize-korean.pl 13 | PARFILE=${LIB}/korean-utf8.par 14 | 15 | $TOKENIZER $* | 16 | $TAGGER $OPTIONS $PARFILE | 17 | $CMD/add-korean-sentence-markers.pl | 18 | $CMD/reformat-korean-tagger-output.pl 19 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-latin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/latin-abbreviations 15 | PARFILE=${LIB}/latin.par 16 | MWLFILE=${LIB}/latin-mwls 17 | 18 | $TOKENIZER -a $ABBR_LIST $* | 19 | # recognition of MWLs 20 | $MWL -f $MWLFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE 23 | 24 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-middle-high-german: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/middle-high-german-abbreviations-utf8 14 | PARFILE=${LIB}/middle-high-german-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-polish: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/polish-abbreviations-utf8 14 | PARFILE=${LIB}/polish-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | # tagging 18 | $TAGGER $OPTIONS $PARFILE 19 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-portuguese: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | TOKENIZER=${BIN}/separate-punctuation 10 | SPLITTER=${CMD}/portuguese-splitter.perl 11 | TAGGER=${BIN}/tree-tagger 12 | ABBR_LIST=${LIB}/portuguese-abbreviations-utf8 13 | POST_TAGGING=${CMD}/portuguese-post-tagging 14 | PARFILE=${LIB}/portuguese-utf8.par 15 | 16 | # splitting 17 | $SPLITTER $* | 18 | # pre-tokenization 19 | sed "s/\([\)\"\'\?\!]\)\([\.\,\;\:]\)/ \1 \2/g" | 20 | # tokenizing 21 | $TOKENIZER +1 +s +l $ABBR_LIST | 22 | # remove empty lines 23 | grep -v '^$' | 24 | # tagging 25 | $TAGGER $PARFILE -token -lemma -sgml | 26 | $POST_TAGGING -no 27 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-portuguese-finegrained: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | TOKENIZER=${BIN}/separate-punctuation 10 | SPLITTER=${CMD}/portuguese-splitter.perl 11 | TAGGER=${BIN}/tree-tagger 12 | ABBR_LIST=${LIB}/portuguese-abbreviations-utf8 13 | POST_TAGGING=${CMD}/portuguese-post-tagging 14 | PARFILE=${LIB}/portuguese-finegrained-utf8.par 15 | 16 | # splitting 17 | $SPLITTER $* | 18 | # pre-tokenization 19 | sed "s/\([\)\"\'\?\!]\)\([\.\,\;\:]\)/ \1 \2/g" | 20 | # tokenizing 21 | $TOKENIZER +1 +s +l $ABBR_LIST | 22 | # remove empty lines 23 | grep -v '^$' | 24 | # tagging 25 | $TAGGER $PARFILE -token -lemma -sgml | 26 | $POST_TAGGING -yes 27 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-portuguese2: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | TAGGER=tree-tagger 14 | ABBR_LIST=${LIB}/portuguese-abbreviations-utf8 15 | PARFILE=${LIB}/portuguese2-utf8.par 16 | 17 | $TOKENIZER -p -a $ABBR_LIST $* | 18 | $TAGGER $OPTIONS $PARFILE 19 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-romanian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/romanian-abbreviations 14 | PARFILE=${LIB}/romanian-utf8.par 15 | 16 | $TOKENIZER -r -a $ABBR_LIST $* | 17 | ${CMD}/split-romanian.perl ${LIB}/romanian-tokens | 18 | $TAGGER $OPTIONS $PARFILE 19 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-russian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | PARFILE=${LIB}/russian-utf8.par 14 | 15 | # do tokenization 16 | $TOKENIZER $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-slovak: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TAGGER=${BIN}/tree-tagger 12 | TOKENIZER=${CMD}/utf8-tokenize.perl 13 | PARFILE=${LIB}/slovak-utf8.par 14 | 15 | $TOKENIZER $* | 16 | $TAGGER $OPTIONS $PARFILE 17 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-slovenian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TAGGER=${BIN}/tree-tagger 12 | TOKENIZER=${CMD}/utf8-tokenize.perl 13 | PARFILE=${LIB}/slovenian-utf8.par 14 | 15 | $TOKENIZER $* | 16 | $TAGGER $OPTIONS $PARFILE 17 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-spanish: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | MWL=${CMD}/mwl-lookup.perl 13 | TAGGER=${BIN}/tree-tagger 14 | ABBR_LIST=${LIB}/spanish-abbreviations 15 | PARFILE=${LIB}/spanish-utf8.par 16 | MWLFILE=${LIB}/spanish-mwls-utf8 17 | 18 | $TOKENIZER -a $ABBR_LIST $* | 19 | # recognition of MWLs 20 | $MWL -f $MWLFILE | 21 | # tagging 22 | $TAGGER $OPTIONS $PARFILE 23 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-spanish-ancora: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TOKENIZER=${CMD}/utf8-tokenize.perl 12 | TAGGER=${BIN}/tree-tagger 13 | ABBR_LIST=${LIB}/spanish-abbreviations 14 | PARFILE=${LIB}/spanish-ancora-utf8.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/tree-tagger-swahili: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set these paths appropriately 4 | 5 | BIN="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/bin" 6 | CMD="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/cmd" 7 | LIB="/Users/Amine/Documents/GeoTrend/Evaluations/Dates/HeidelTime/TreeTagger/lib" 8 | 9 | OPTIONS="-token -lemma -sgml" 10 | 11 | TAGGER=${BIN}/tree-tagger 12 | TOKENIZER=${CMD}/tokenize.pl 13 | ABBR_LIST=${LIB}/swahili-abbreviations 14 | PARFILE=${LIB}/swahili.par 15 | 16 | $TOKENIZER -a $ABBR_LIST $* | 17 | $TAGGER $OPTIONS $PARFILE 18 | -------------------------------------------------------------------------------- /TreeTagger/cmd/utf8-tokenize.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | ######################################################################## 4 | # # 5 | # tokenization script for tagger preprocessing # 6 | # Author: Helmut Schmid, IMS, University of Stuttgart # 7 | # Serge Sharoff, University of Leeds # 8 | # Description: # 9 | # - splits input text into tokens (one token per line) # 10 | # - cuts off punctuation, parentheses etc. # 11 | # - disambiguates periods # 12 | # - preserves SGML markup # 13 | # # 14 | ######################################################################## 15 | 16 | use Getopt::Std; 17 | use utf8; 18 | use Encode; 19 | 20 | getopts('chgfeipza:r'); 21 | 22 | # Modify the following lines in order to adapt the tokenizer to other 23 | # types of text and/or languages 24 | 25 | # characters which have to be cut off at the beginning of a word 26 | my $PChar='[¿¡{\'\\`"‚„†‡‹‘’“”•–—›»«'; 27 | 28 | # characters which have to be cut off at the end of a word 29 | my $FChar=']}\'\`\",;:\!\?\%‚„…†‡‰‹‘’“”•–—›»«'; 30 | 31 | # character sequences which have to be cut off at the beginning of a word 32 | my $PClitic=''; 33 | 34 | # character sequences which have to be cut off at the end of a word 35 | my $FClitic=''; 36 | 37 | if (defined($opt_r)) { 38 | # Romanian 39 | $PChar='[¿¡{\\`"‚„†‡‹‘’“”•–—›»«'; 40 | $FChar=']}\`\",;:\!\?\%‚„…†‡‰‹‘’“”•–—›»«'; 41 | } 42 | if (defined($opt_e)) { 43 | # English 44 | $FClitic = "['’´](s|re|ve|d|m|em|ll)|n['’´]t"; 45 | } 46 | if (defined($opt_i)) { 47 | # Italian 48 | $PClitic = "(?:d[ae]ll|nell|all|[ld]|sull|quest|un|senz|tutt|c|s)['´’]"; 49 | } 50 | if (defined($opt_f)) { 51 | # French 52 | $PClitic = "(?:[dcjlmnst]|qu|jusqu|lorsqu|quoiqu|puisqu)['’´]"; 53 | $FClitic = "-t-elles?|-t-ils?|-t-on|-ce|-elles?|-ils?|-je|-la|-les?|-leur|-lui|-mêmes?|-m['’´]|-moi|-nous|-on|-toi|-tu|-t['’´]|-vous|-en|-y|-ci|-là"; 54 | } 55 | if (defined($opt_p)) { 56 | # Portuguese 57 | $FClitic = "-a|-as|-la|-las|-lha|-lhas|-lhe|-lhes|-lho|-lhos|-lo|-los|-ma|-mas|-me|-mo|-mos|-na|-nas|-no|-no-la|-no-las|-no-lo|-no-los|-nos|-o|-os|-s|-se|-se-á|-se-ão|-se-é|-se-ia|-se-lha|-se-lhas|-se-lhe|-se-lhes|-se-lho|-se-lhos|-se-nos|-se-vos|-ta|-tas|-te|-to|-tos|-vo-la|-vo-las|-vo-lo|-vo-los|-vos"; 58 | } 59 | if (defined($opt_z)) { 60 | # Galician 61 | $FClitic = '-la|-las|-lo|-los|-nos'; 62 | } 63 | if (defined($opt_c)) { 64 | # Catalan 65 | $PClitic = "[dlmnst]['’´]"; 66 | $FClitic = "['’´](n|s|ls|l|hi|ns|t|m|ho)|-(se|lo|la|li|los|les|hi|ho|ne|nos|me|s|te|m)"; 67 | } 68 | 69 | 70 | ### NO MODIFICATIONS REQUIRED BEYOND THIS LINE ######################### 71 | 72 | if (defined($opt_h)) { 73 | die " 74 | Usage: utf8-tokenize.perl [ options ] ...files... 75 | 76 | Options: 77 | -e : English text 78 | -i : Italian text 79 | -f : French text 80 | -z : Galician text 81 | -a : contains a list of words which are either abbreviations or 82 | words which should not be further split. 83 | "; 84 | } 85 | 86 | # Read the list of abbreviations and words 87 | if (defined($opt_a)) { 88 | die "Can't read: $opt_a: $!\n" unless (open(FILE, $opt_a)); 89 | while () { 90 | $_ = decode('utf8',$_); 91 | s/^[ \t\r\n]+//; 92 | s/[ \t\r\n]+$//; 93 | next if (/^\#/ || /^\s$/); # ignore comments 94 | $Token{$_} = 1; 95 | } 96 | close FILE; 97 | } 98 | 99 | #SS: main loop; 100 | my $first_line = 1; 101 | while (<>) { 102 | $_ = decode('utf8',$_); 103 | 104 | # delete optional byte order markers (BOM) 105 | if ($first_line) { 106 | undef $first_line; 107 | s/^\x{FEFF}//; 108 | } 109 | 110 | # replace newlines and tab characters with blanks 111 | tr/\n\t/ /; 112 | #s/[\n\t\p{XPosixCntrl}]/ /g; # more general version 113 | 114 | # replace blanks within SGML tags 115 | while (s/(<[^<> ]*) ([^<>]*>)/$1\377$2/g) { 116 | } 117 | #Separ: ÿþ 118 | 119 | # replace whitespace with a special character 120 | tr/ /\376/; 121 | 122 | # restore SGML tags 123 | tr/\377\376/ \377/; 124 | 125 | # prepare SGML-Tags for tokenization 126 | s/(<[^<>]*>)/\377$1\377/g; 127 | s/^\377//; 128 | s/\377$//; 129 | s/\377\377\377*/\377/g; 130 | 131 | @S = split("\377"); 132 | for( $i=0; $i<=$#S; $i++) { 133 | $_ = $S[$i]; 134 | 135 | if (/^<.*>$/) { 136 | # SGML tag 137 | print encode('utf8',"$_\n"); 138 | } else { 139 | # add a blank at the beginning and the end of each segment 140 | $_ = " $_ "; 141 | # insert missing blanks after punctuation 142 | s/(\.\.\.)/ ... /g; 143 | s/([;\!\?])([^ ])/$1 $2/g; 144 | #s/([.,:])([^ 0-9.])/$1 $2/g; 145 | 146 | @F = split; 147 | for ( $j=0; $j<=$#F; $j++) { 148 | my $suffix=""; 149 | $_ = $F[$j]; 150 | 151 | # separate punctuation and parentheses from words 152 | $finished = 0; 153 | while (!$finished) { 154 | 155 | # preceding parentheses 156 | if (s/^(\()([^\)]*)(.)$/$2$3/) { 157 | print "$1\n"; 158 | } 159 | 160 | # following preceding parentheses 161 | elsif (s/^([^(]+)(\))$/$1/) { 162 | $suffix = "$2\n$suffix"; 163 | } 164 | 165 | elsif (s/^([$PChar])(.)/$2/) { 166 | print encode('utf8',"$1\n"); 167 | } 168 | 169 | # cut off preceding punctuation 170 | elsif (s/^([$PChar])(.)/$2/) { 171 | print encode('utf8',"$1\n"); 172 | } 173 | 174 | # cut off trailing punctuation 175 | elsif (s/(.)([$FChar])$/$1/) { 176 | $suffix = "$2\n$suffix"; 177 | } 178 | 179 | # cut off trailing periods if punctuation precedes 180 | elsif (s/([$FChar]|\))\.$//) { 181 | $suffix = ".\n$suffix"; 182 | if ($_ eq "") { 183 | $_ = $1; 184 | } else { 185 | $suffix = "$1\n$suffix"; 186 | } 187 | } 188 | 189 | else { 190 | $finished = 1; 191 | } 192 | } 193 | 194 | # handle explicitly listed tokens 195 | if (defined($Token{$_})) { 196 | print encode('utf8',"$_\n$suffix"); 197 | next; 198 | } 199 | 200 | # abbreviations of the form A. or U.S.A. 201 | if (/^([A-Za-z-]\.)+$/) { 202 | print encode('utf8',"$_\n$suffix"); 203 | next; 204 | } 205 | 206 | 207 | # disambiguate periods 208 | if (/^(..*)\.$/ && $_ ne "..." && !($opt_g && /^[0-9]+\.$/)) { 209 | $_ = $1; 210 | $suffix = ".\n$suffix"; 211 | if (defined($Token{$_})) { 212 | print encode('utf8',"$_\n$suffix"); 213 | next; 214 | } 215 | } 216 | 217 | # cut off clitics 218 | while (s/^(--)(.)/$2/) { 219 | print encode('utf8',"$1\n"); 220 | } 221 | if ($PClitic ne '') { 222 | while (s/^($PClitic)(.)/$2/i) { 223 | print encode('utf8',"$1\n"); 224 | } 225 | } 226 | 227 | while (s/(.)(--)$/$1/) { 228 | $suffix = "$2\n$suffix"; 229 | } 230 | if ($FClitic ne '') { 231 | while (s/(.)($FClitic)$/$1/i) { 232 | $suffix = "$2\n$suffix"; 233 | } 234 | } 235 | 236 | print encode('utf8',"$_\n$suffix"); 237 | } 238 | } 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /TreeTagger/install-tagger.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir cmd 4 | mkdir lib 5 | mkdir bin 6 | mkdir doc 7 | echo '' 8 | 9 | if [ -r tree-tagger-linux-3.2.1.tar.gz ] 10 | then 11 | tar -zxf tree-tagger-linux-3.2.1.tar.gz 12 | echo 'TreeTagger version for PC-Linux installed.' 13 | fi 14 | 15 | if [ -r tree-tagger-MacOSX-3.2.tar.gz ] 16 | then 17 | tar -zxf tree-tagger-MacOSX-3.2.tar.gz 18 | echo 'TreeTagger version for Mac OS-X installed.' 19 | fi 20 | 21 | if [ -r tree-tagger-ARM-3.2.tar.gz ] 22 | then 23 | tar -zxf tree-tagger-ARM-3.2.tar.gz 24 | echo 'TreeTagger version for ARM-Linux installed.' 25 | fi 26 | 27 | if [ -r tagger-scripts.tar.gz ] 28 | then 29 | gzip -cd tagger-scripts.tar.gz | tar -xf - 30 | chmod +x cmd/* 31 | echo 'Tagging scripts installed.' 32 | fi 33 | 34 | if [ -r estonian-par-linux-3.2-utf8.bin.gz ] 35 | then 36 | gzip -cd estonian-par-linux-3.2-utf8.bin.gz > lib/estonian-utf8.par 37 | echo 'Estonian parameter file (UTF8) installed.' 38 | fi 39 | 40 | if [ -r finnish-par-linux-3.2-utf8.bin.gz ] 41 | then 42 | gzip -cd finnish-par-linux-3.2-utf8.bin.gz > lib/finnish-utf8.par 43 | echo 'Finnish parameter file (UTF8) installed.' 44 | fi 45 | 46 | if [ -r greek-par-linux-3.2-utf8.bin.gz ] 47 | then 48 | gzip -cd greek-par-linux-3.2-utf8.bin.gz > lib/greek-utf8.par 49 | echo 'Greek parameter file (UTF8) installed.' 50 | fi 51 | 52 | if [ -r german-par-linux-3.2-utf8.bin.gz ] 53 | then 54 | gzip -cd german-par-linux-3.2-utf8.bin.gz > lib/german-utf8.par 55 | echo 'German parameter file (UTF8) installed.' 56 | fi 57 | 58 | if [ -r korean-par-linux-3.2-utf8.bin.gz ] 59 | then 60 | gzip -cd korean-par-linux-3.2-utf8.bin.gz > lib/korean-utf8.par 61 | echo 'Korean parameter file (UTF8) installed.' 62 | fi 63 | 64 | if [ -r middle-high-german-par-linux-3.2-utf8.bin.gz ] 65 | then 66 | gzip -cd middle-high-german-par-linux-3.2-utf8.bin.gz > lib/middle-high-german-utf8.par 67 | echo 'Middle High German parameter file (UTF8) installed.' 68 | fi 69 | 70 | if [ -r german-chunker-par-linux-3.2-utf8.bin.gz ] 71 | then 72 | gzip -cd german-chunker-par-linux-3.2-utf8.bin.gz > lib/german-chunker-utf8.par 73 | echo 'German chunker parameter file (UTF8) installed.' 74 | fi 75 | 76 | if [ -r english-par-linux-3.2-utf8.bin.gz ] 77 | then 78 | gzip -cd english-par-linux-3.2-utf8.bin.gz > lib/english-utf8.par 79 | echo 'English parameter file (UTF8) installed.' 80 | fi 81 | 82 | if [ -r english-chunker-par-linux-3.2-utf8.bin.gz ] 83 | then 84 | gzip -cd english-chunker-par-linux-3.2-utf8.bin.gz > lib/english-chunker-utf8.par 85 | echo 'English chunker parameter file (UTF8) installed.' 86 | fi 87 | 88 | if [ -r spanish-chunker-par-linux-3.2-utf8.bin.gz ] 89 | then 90 | gzip -cd spanish-chunker-par-linux-3.2-utf8.bin.gz > lib/spanish-chunker-utf8.par 91 | echo 'Spanish chunker parameter file (UTF8) installed.' 92 | fi 93 | 94 | if [ -r french-par-linux-3.2-utf8.bin.gz ] 95 | then 96 | gzip -cd french-par-linux-3.2-utf8.bin.gz > lib/french-utf8.par 97 | echo 'French parameter file (UTF8) installed.' 98 | fi 99 | 100 | if [ -r french-chunker-par-linux-3.2-utf8.bin.gz ] 101 | then 102 | gzip -cd french-chunker-par-linux-3.2-utf8.bin.gz > lib/french-chunker-utf8.par 103 | echo 'French chunker parameter file (UTF8) installed.' 104 | fi 105 | 106 | if [ -r romanian-par-linux-3.2-utf8.bin.gz ] 107 | then 108 | gzip -cd romanian-par-linux-3.2-utf8.bin.gz > lib/romanian-utf8.par 109 | echo 'Romanian parameter file (UTF8) installed.' 110 | fi 111 | 112 | if [ -r italian-par-linux-3.2-utf8.bin.gz ] 113 | then 114 | gzip -cd italian-par-linux-3.2-utf8.bin.gz > lib/italian-utf8.par 115 | echo 'Italian parameter file (UTF8) installed.' 116 | fi 117 | 118 | if [ -r italian-par2-linux-3.2-utf8.bin.gz ] 119 | then 120 | gzip -cd italian-par2-linux-3.2-utf8.bin.gz > lib/italian.par 121 | echo 'alternative Italian parameter file installed.' 122 | fi 123 | 124 | if [ -r bulgarian-par-linux-3.2-utf8.bin.gz ] 125 | then 126 | gzip -cd bulgarian-par-linux-3.2-utf8.bin.gz > lib/bulgarian-utf8.par 127 | echo 'Bulgarian parameter file (UTF8) installed.' 128 | fi 129 | 130 | if [ -r catalan-par-linux-3.2-utf8.bin.gz ] 131 | then 132 | gzip -cd catalan-par-linux-3.2-utf8.bin.gz > lib/catalan-utf8.par 133 | echo 'Catalan parameter file (UTF8) installed.' 134 | fi 135 | 136 | if [ -r polish-par-linux-3.2-utf8.bin.gz ] 137 | then 138 | gzip -cd polish-par-linux-3.2-utf8.bin.gz > lib/polish-utf8.par 139 | echo 'Polish parameter file (UTF8) installed.' 140 | fi 141 | 142 | if [ -r czech-par-linux-3.2-utf8.bin.gz ] 143 | then 144 | gzip -cd czech-par-linux-3.2-utf8.bin.gz > lib/czech-utf8.par 145 | echo 'Czech parameter file (UTF8) installed.' 146 | fi 147 | 148 | if [ -r portuguese-par-linux-3.2-utf8.bin.gz ] 149 | then 150 | gzip -cd portuguese-par-linux-3.2-utf8.bin.gz > lib/portuguese-utf8.par 151 | echo 'Portuguese parameter file (UTF8) installed.' 152 | fi 153 | 154 | if [ -r portuguese-par2-linux-3.2-utf8.bin.gz ] 155 | then 156 | gzip -cd portuguese-par2-linux-3.2-utf8.bin.gz > lib/portuguese2-utf8.par 157 | echo 'Alternative Portuguese parameter file (UTF8) installed.' 158 | fi 159 | 160 | if [ -r portuguese-finegrained-par-linux-3.2-utf8.bin.gz ] 161 | then 162 | gzip -cd portuguese-finegrained-par-linux-3.2-utf8.bin.gz > lib/portuguese-finegrained-utf8.par 163 | echo 'Portuguese parameter file (UTF8) with fine-grained tagset installed.' 164 | fi 165 | 166 | if [ -r russian-par-linux-3.2-utf8.bin.gz ] 167 | then 168 | gzip -cd russian-par-linux-3.2-utf8.bin.gz > lib/russian-utf8.par 169 | echo 'Russian parameter file (UTF8) installed.' 170 | fi 171 | 172 | if [ -r spanish-par-linux-3.2-utf8.bin.gz ] 173 | then 174 | gzip -cd spanish-par-linux-3.2-utf8.bin.gz > lib/spanish-utf8.par 175 | echo 'Spanish parameter file (UTF8) installed.' 176 | fi 177 | 178 | if [ -r spanish-ancora-par-linux-3.2-utf8.bin.gz ] 179 | then 180 | gzip -cd spanish-ancora-par-linux-3.2-utf8.bin.gz > lib/spanish-ancora-utf8.par 181 | echo 'Spanish Ancora parameter file (UTF8) installed.' 182 | fi 183 | 184 | if [ -r galician-par-linux-3.2.bin.gz ] 185 | then 186 | gzip -cd galician-par-linux-3.2.bin.gz > lib/galician-utf8.par 187 | echo 'Galician parameter file (UTF8) installed.' 188 | fi 189 | 190 | if [ -r danish-par-linux-3.2-utf8.bin.gz ] 191 | then 192 | gzip -cd danish-par-linux-3.2-utf8.bin.gz > lib/danish-utf8.par 193 | echo 'Danish parameter file (UTF8) installed.' 194 | fi 195 | 196 | if [ -r dutch-par-linux-3.2-utf8.bin.gz ] 197 | then 198 | gzip -cd dutch-par-linux-3.2-utf8.bin.gz > lib/dutch-utf8.par 199 | echo 'Dutch parameter file (UTF8) installed.' 200 | fi 201 | 202 | if [ -r dutch2-par-linux-3.2-utf8.bin.gz ] 203 | then 204 | gzip -cd dutch2-par-linux-3.2-utf8.bin.gz > lib/dutch-utf8.par 205 | echo 'Dutch parameter file (UTF8) installed.' 206 | fi 207 | 208 | if [ -r swahili-par-linux-3.2.bin.gz ] 209 | then 210 | gzip -cd swahili-par-linux-3.2.bin.gz > lib/swahili.par 211 | echo 'Swahili parameter file installed.' 212 | fi 213 | 214 | if [ -r slovak-par-linux-3.2-utf8.bin.gz ] 215 | then 216 | gzip -cd slovak-par-linux-3.2-utf8.bin.gz > lib/slovak-utf8.par 217 | echo 'Slovak parameter file (UTF8) installed.' 218 | fi 219 | 220 | if [ -r slovak2-par-linux-3.2-utf8.bin.gz ] 221 | then 222 | gzip -cd slovak2-par-linux-3.2-utf8.bin.gz > lib/slovak-utf8.par 223 | echo 'Slovak parameter file (Linux, UTF8, full tagset) installed.' 224 | fi 225 | 226 | if [ -r slovenian-par-linux-3.2-utf8.bin.gz ] 227 | then 228 | gzip -cd slovenian-par-linux-3.2-utf8.bin.gz > lib/slovenian-utf8.par 229 | echo 'Slovenian parameter file (UTF8) installed.' 230 | fi 231 | 232 | if [ -r latin-par-linux-3.2.bin.gz ] 233 | then 234 | gzip -cd latin-par-linux-3.2.bin.gz > lib/latin.par 235 | echo 'Latin parameter file installed.' 236 | fi 237 | 238 | if [ -r latinIT-par-linux-3.2.bin.gz ] 239 | then 240 | gzip -cd latinIT-par-linux-3.2.bin.gz > lib/latin.par 241 | echo 'Latin Index Thomisticus parameter file installed.' 242 | fi 243 | 244 | if [ -r ancient-greek-par-linux-3.2-beta.bin.gz ] 245 | then 246 | gzip -cd ancient-greek-par-linux-3.2-beta.bin.gz > lib/ancient-greek-beta.par 247 | echo 'Ancient Greek parameter file (beta encoding) installed.' 248 | fi 249 | 250 | if [ -r ancient-greek-par-linux-3.2-utf8.bin.gz ] 251 | then 252 | gzip -cd ancient-greek-par-linux-3.2-utf8.bin.gz > lib/ancient-greek-utf8.par 253 | echo 'Ancient Greek parameter file (UTF-8 encoding) installed.' 254 | fi 255 | 256 | 257 | # installation of uncompressed files 258 | 259 | if [ -r tree-tagger-linux-3.2.1.tar ] 260 | then 261 | tar -xf tree-tagger-linux-3.2.1.tar 262 | echo 'TreeTagger version for PC-Linux installed.' 263 | fi 264 | 265 | if [ -r tree-tagger-MacOSX-3.2.tar ] 266 | then 267 | tar -xf tree-tagger-MacOSX-3.2.tar 268 | echo 'TreeTagger version for Mac OS-X installed.' 269 | fi 270 | 271 | if [ -r tree-tagger-ARM-3.2.tar ] 272 | then 273 | tar -xf tree-tagger-ARM-3.2.tar 274 | echo 'TreeTagger version for ARM-Linux installed.' 275 | fi 276 | 277 | if [ -r tagger-scripts.tar ] 278 | then 279 | tar -xf tagger-scripts.tar 280 | chmod +x cmd/* 281 | echo 'Tagging scripts installed.' 282 | fi 283 | 284 | if [ -r estonian-par-linux-3.2-utf8.bin ] 285 | then 286 | mv estonian-par-linux-3.2-utf8.bin lib/estonian-utf8.par 287 | echo 'Estonian parameter file (UTF8) installed.' 288 | fi 289 | 290 | if [ -r finnish-par-linux-3.2-utf8.bin ] 291 | then 292 | mv finnish-par-linux-3.2-utf8.bin lib/finnish-utf8.par 293 | echo 'Finnish parameter file (UTF8) installed.' 294 | fi 295 | 296 | if [ -r greek-par-linux-3.2-utf8.bin ] 297 | then 298 | mv greek-par-linux-3.2-utf8.bin lib/greek-utf8.par 299 | echo 'Greek parameter file (UTF8) installed.' 300 | fi 301 | 302 | if [ -r german-par-linux-3.2-utf8.bin ] 303 | then 304 | mv german-par-linux-3.2-utf8.bin lib/german-utf8.par 305 | echo 'German parameter file (UTF8) installed.' 306 | fi 307 | 308 | if [ -r korean-par-linux-3.2-utf8.bin ] 309 | then 310 | mv korean-par-linux-3.2-utf8.bin lib/korean-utf8.par 311 | echo 'Korean parameter file (UTF8) installed.' 312 | fi 313 | 314 | if [ -r middle-high-german-par-linux-3.2-utf8.bin ] 315 | then 316 | mv middle-high-german-par-linux-3.2-utf8.bin lib/middle-high-german-utf8.par 317 | echo 'Middle High German parameter file (UTF8) installed.' 318 | fi 319 | 320 | if [ -r german-chunker-par-linux-3.2-utf8.bin ] 321 | then 322 | mv german-chunker-par-linux-3.2-utf8.bin lib/german-chunker-utf8.par 323 | echo 'German chunker parameter file (UTF8) installed.' 324 | fi 325 | 326 | if [ -r english-par-linux-3.2-utf8.bin ] 327 | then 328 | mv english-par-linux-3.2-utf8.bin lib/english-utf8.par 329 | echo 'English parameter file (UTF8) installed.' 330 | fi 331 | 332 | if [ -r english-chunker-par-linux-3.2-utf8.bin ] 333 | then 334 | mv english-chunker-par-linux-3.2-utf8.bin lib/english-chunker-utf8.par 335 | echo 'English chunker parameter file (UTF8) installed.' 336 | fi 337 | 338 | if [ -r spanish-chunker-par-linux-3.2-utf8.bin ] 339 | then 340 | mv spanish-chunker-par-linux-3.2-utf8.bin lib/spanish-chunker-utf8.par 341 | echo 'Spanish chunker parameter file (UTF8) installed.' 342 | fi 343 | 344 | if [ -r french-par-linux-3.2-utf8.bin ] 345 | then 346 | mv french-par-linux-3.2-utf8.bin lib/french-utf8.par 347 | echo 'French parameter file (UTF8) installed.' 348 | fi 349 | 350 | if [ -r french-chunker-par-linux-3.2-utf8.bin ] 351 | then 352 | mv french-chunker-par-linux-3.2-utf8.bin lib/french-chunker-utf8.par 353 | echo 'French chunker parameter file (UTF8) installed.' 354 | fi 355 | 356 | if [ -r romanian-par-linux-3.2-utf8.bin ] 357 | then 358 | mv romanian-par-linux-3.2-utf8.bin lib/romanian-utf8.par 359 | echo 'Romanian parameter file (UTF8) installed.' 360 | fi 361 | 362 | if [ -r italian-par-linux-3.2-utf8.bin ] 363 | then 364 | mv italian-par-linux-3.2-utf8.bin lib/italian-utf8.par 365 | echo 'Italian parameter file (UTF8) installed.' 366 | fi 367 | 368 | if [ -r italian-par2-linux-3.2-utf8.bin ] 369 | then 370 | mv italian-par2-linux-3.2-utf8.bin lib/italian.par 371 | echo 'alternative Italian parameter file installed.' 372 | fi 373 | 374 | if [ -r bulgarian-par-linux-3.2-utf8.bin ] 375 | then 376 | mv bulgarian-par-linux-3.2-utf8.bin lib/bulgarian-utf8.par 377 | echo 'Bulgarian parameter file (UTF8) installed.' 378 | fi 379 | 380 | if [ -r catalan-par-linux-3.2-utf8.bin ] 381 | then 382 | mv catalan-par-linux-3.2-utf8.bin lib/catalan-utf8.par 383 | echo 'Catalan parameter file (UTF8) installed.' 384 | fi 385 | 386 | if [ -r polish-par-linux-3.2-utf8.bin ] 387 | then 388 | mv polish-par-linux-3.2-utf8.bin lib/polish-utf8.par 389 | echo 'Polish parameter file (UTF8) installed.' 390 | fi 391 | 392 | if [ -r czech-par-linux-3.2-utf8.bin ] 393 | then 394 | mv czech-par-linux-3.2-utf8.bin lib/czech-utf8.par 395 | echo 'Czech parameter file (UTF8) installed.' 396 | fi 397 | 398 | if [ -r portuguese-par-linux-3.2-utf8.bin ] 399 | then 400 | mv portuguese-par-linux-3.2-utf8.bin lib/portuguese-utf8.par 401 | echo 'Portuguese parameter file (UTF8) installed.' 402 | fi 403 | 404 | if [ -r portuguese-par2-linux-3.2-utf8.bin ] 405 | then 406 | mv portuguese-par2-linux-3.2-utf8.bin lib/portuguese2-utf8.par 407 | echo 'Alternative Portuguese parameter file (UTF8) installed.' 408 | fi 409 | 410 | if [ -r portuguese-finegrained-par-linux-3.2-utf8.bin ] 411 | then 412 | mv portuguese-finegrained-par-linux-3.2-utf8.bin lib/portuguese-finegrained-utf8.par 413 | echo 'Portuguese parameter file (UTF8) with fine-grained tagset installed.' 414 | fi 415 | 416 | if [ -r russian-par-linux-3.2-utf8.bin ] 417 | then 418 | mv russian-par-linux-3.2-utf8.bin lib/russian-utf8.par 419 | echo 'Russian parameter file (UTF8) installed.' 420 | fi 421 | 422 | if [ -r spanish-par-linux-3.2-utf8.bin ] 423 | then 424 | mv spanish-par-linux-3.2-utf8.bin lib/spanish-utf8.par 425 | echo 'Spanish parameter file (UTF8) installed.' 426 | fi 427 | 428 | if [ -r spanish-ancora-par-linux-3.2-utf8.bin ] 429 | then 430 | mv spanish-ancora-par-linux-3.2-utf8.bin lib/spanish-ancora-utf8.par 431 | echo 'Spanish Ancora parameter file (UTF8) installed.' 432 | fi 433 | 434 | if [ -r galician-par-linux-3.2.bin ] 435 | then 436 | mv galician-par-linux-3.2.bin lib/galician-utf8.par 437 | echo 'Galician parameter file (UTF8) installed.' 438 | fi 439 | 440 | if [ -r danish-par-linux-3.2-utf8.bin ] 441 | then 442 | mv danish-par-linux-3.2-utf8.bin lib/danish-utf8.par 443 | echo 'Danish parameter file (UTF8) installed.' 444 | fi 445 | 446 | if [ -r dutch-par-linux-3.2-utf8.bin ] 447 | then 448 | mv dutch-par-linux-3.2-utf8.bin lib/dutch-utf8.par 449 | echo 'Dutch parameter file (UTF8) installed.' 450 | fi 451 | 452 | if [ -r dutch2-par-linux-3.2-utf8.bin ] 453 | then 454 | mv dutch2-par-linux-3.2-utf8.bin lib/dutch-utf8.par 455 | echo 'Dutch parameter file (UTF8) installed.' 456 | fi 457 | 458 | if [ -r swahili-par-linux-3.2.bin ] 459 | then 460 | mv swahili-par-linux-3.2.bin lib/swahili.par 461 | echo 'Swahili parameter file installed.' 462 | fi 463 | 464 | if [ -r slovak-par-linux-3.2-utf8.bin ] 465 | then 466 | mv slovak-par-linux-3.2-utf8.bin lib/slovak-utf8.par 467 | echo 'Slovak parameter file (UTF8) installed.' 468 | fi 469 | 470 | if [ -r slovak2-par-linux-3.2-utf8.bin ] 471 | then 472 | mv slovak2-par-linux-3.2-utf8.bin lib/slovak-utf8.par 473 | echo 'Slovak parameter file (Linux, UTF8, full tagset) installed.' 474 | fi 475 | 476 | if [ -r slovenian-par-linux-3.2-utf8.bin ] 477 | then 478 | mv slovenian-par-linux-3.2-utf8.bin lib/slovenian-utf8.par 479 | echo 'Slovenian parameter file (UTF8) installed.' 480 | fi 481 | 482 | if [ -r latin-par-linux-3.2.bin ] 483 | then 484 | mv latin-par-linux-3.2.bin lib/latin.par 485 | echo 'Latin parameter file installed.' 486 | fi 487 | 488 | if [ -r latinIT-par-linux-3.2.bin ] 489 | then 490 | mv latinIT-par-linux-3.2.bin lib/latin.par 491 | echo 'Latin Index Thomisticus parameter file installed.' 492 | fi 493 | 494 | 495 | 496 | 497 | for file in cmd/* 498 | do 499 | awk '$0=="BIN=./bin"{print "BIN=\"'`pwd`'/bin\"";next}\ 500 | $0=="CMD=./cmd"{print "CMD=\"'`pwd`'/cmd\"";next}\ 501 | $0=="LIB=./lib"{print "LIB=\"'`pwd`'/lib\"";next}\ 502 | {print}' $file > $file.tmp; 503 | mv $file.tmp $file; 504 | done 505 | echo 'Path variables modified in tagging scripts.' 506 | 507 | chmod 0755 cmd/* 508 | 509 | echo '' 510 | echo 'You might want to add '`pwd`'/cmd and '`pwd`'/bin to the PATH variable so that you do not need to specify the full path to run the tagging scripts.' 511 | echo '' 512 | -------------------------------------------------------------------------------- /TreeTagger/lib/bulgarian-abbreviations: -------------------------------------------------------------------------------- 1 | M. 2 | R. 3 | S. 4 | А. 5 | Ал. 6 | Б. 7 | Б.р. 8 | В. 9 | Г. 10 | Ген. 11 | Д. 12 | Е. 13 | Ж. 14 | З. 15 | И. 16 | Ив. 17 | Й. 18 | Кл. 19 | Л. 20 | М. 21 | М.г. 22 | Н. 23 | О. 24 | П. 25 | Преп. 26 | Прор. 27 | Проф. 28 | Р. 29 | С. 30 | Св. 31 | Сп. 32 | Т. 33 | У. 34 | Х. 35 | ал. 36 | ап. 37 | арх. 38 | б.а. 39 | бел. 40 | бл. 41 | благов. 42 | бр. 43 | бул. 44 | в. 45 | вж. 46 | вх. 47 | г. 48 | ген. 49 | гл. 50 | год. 51 | гр. 52 | д.и.н. 53 | доц. 54 | др. 55 | е. 56 | ев. 57 | еп. 58 | ет. 59 | ж.к. 60 | з. 61 | зам. 62 | и.д. 63 | изд. 64 | к.с. 65 | кв. 66 | кг. 67 | куб. 68 | куб.м. 69 | лв. 70 | м. 71 | м.г. 72 | мин. 73 | млн. 74 | млрд. 75 | н. 76 | нар. 77 | о. 78 | о.з. 79 | пл. 80 | подп. 81 | полк. 82 | пр. 83 | проф. 84 | р. 85 | ред. 86 | с. 87 | св. 88 | сек. 89 | серж. 90 | см. 91 | сп. 92 | ст. 93 | стр. 94 | т. 95 | т.г. 96 | т.е. 97 | т.м. 98 | т.н. 99 | т.нар. 100 | ул. 101 | хил. 102 | ч. 103 | чл. 104 | яз. 105 | -------------------------------------------------------------------------------- /TreeTagger/lib/bulgarian-mwls: -------------------------------------------------------------------------------- 1 | за да 2 | себе си 3 | след като 4 | тъй като 5 | както и 6 | може би 7 | така че 8 | След като 9 | Може би 10 | Иван Костов 11 | Така че 12 | Надежда Михайлова 13 | За да 14 | Петър Жотев 15 | все пак 16 | Ню Йорк 17 | Емануил Йорданов 18 | по време на 19 | Петър Стоянов 20 | така и 21 | по отношение на 22 | като че ли 23 | Йордан Соколов 24 | Въпреки че 25 | Тъй като 26 | Димитър Абаджиев 27 | Ал Гор 28 | -------------------------------------------------------------------------------- /TreeTagger/lib/catalan-abbreviations: -------------------------------------------------------------------------------- 1 | A. 2 | A.A. 3 | A.C. 4 | A.D. 5 | A.M. 6 | A.R. 7 | B. 8 | B.B. 9 | B.C. 10 | B.T. 11 | C. 12 | C.A. 13 | C.P. 14 | C.T. 15 | D. 16 | Dr. 17 | E. 18 | Ed. 19 | E.K. 20 | E.M. 21 | E.N.M. 22 | etc. 23 | F. 24 | F.C. 25 | F.L. 26 | F.M. 27 | F.R. 28 | G. 29 | G.G. 30 | G.K. 31 | G.M. 32 | G.S. 33 | H. 34 | H.C. 35 | H.P. 36 | I. 37 | I.M. 38 | J. 39 | J.C. 40 | J.G. 41 | J.S.G. 42 | J.V. 43 | K. 44 | kg. 45 | L. 46 | L.P. 47 | m. 48 | M. 49 | M.A. 50 | M.B. 51 | M.C.B. 52 | M.C.C. 53 | M.H. 54 | M.I. 55 | M.L. 56 | M.M. 57 | Mr. 58 | N. 59 | N.S. 60 | núm. 61 | P. 62 | P.H. 63 | P.M. 64 | P.R. 65 | pral. 66 | pta. 67 | ptes. 68 | R. 69 | R.A. 70 | R.M. 71 | S. 72 | S.A. 73 | S.C. 74 | S.G. 75 | S.L. 76 | S.P. 77 | Sr. 78 | S.R. 79 | V. 80 | V.B. 81 | V.C. 82 | V.D. 83 | Z. 84 | -------------------------------------------------------------------------------- /TreeTagger/lib/czech-abbreviations-utf8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/czech-abbreviations-utf8 -------------------------------------------------------------------------------- /TreeTagger/lib/dutch-abbreviations: -------------------------------------------------------------------------------- 1 | 't 2 | 's 3 | 'n 4 | 2bis. 5 | 3bis. 6 | 7bis. 7 | AR. 8 | Actualit. 9 | Afd. 10 | Antw. 11 | Arbh. 12 | Art. 13 | B.St. 14 | B.s. 15 | Besl.W. 16 | Bull. 17 | Bull.Bel. 18 | Cass. 19 | Cf. 20 | Com.I.B. 21 | D.t/V.I. 22 | Dhr. 23 | Doc. 24 | Dr. 25 | Fisc. 26 | Fr. 27 | Gec. 28 | II. 29 | III. 30 | J.-L.M. 31 | NR. 32 | NRS. 33 | Nat. 34 | No. 35 | Nr. 36 | Onderafd. 37 | PAR. 38 | Par. 39 | RECHTSFAK. 40 | RKW. 41 | TELEF. 42 | Volksvert. 43 | Vr. 44 | a. 45 | adv.-gen. 46 | afd. 47 | aj. 48 | al. 49 | arb. 50 | art. 51 | artt. 52 | b. 53 | b.v. 54 | b.w. 55 | bijv. 56 | blz. 57 | bv. 58 | c.q. 59 | cf. 60 | cfr. 61 | concl. 62 | d. 63 | d.d. 64 | d.i. 65 | d.w.z. 66 | dd. 67 | doc. 68 | e. 69 | e.d. 70 | e.v. 71 | enz. 72 | f. 73 | fr. 74 | g.w. 75 | gepubl. 76 | i.p.v. 77 | i.v.m. 78 | j.t.t. 79 | jl. 80 | k.b. 81 | kol. 82 | m.b.t. 83 | m.i. 84 | max. 85 | n.a.v. 86 | nl. 87 | nr. 88 | nrs. 89 | o.a. 90 | o.b.s.i. 91 | o.m. 92 | opm. 93 | p. 94 | par. 95 | pct. 96 | pp. 97 | ref. 98 | resp. 99 | respekt. 100 | t.a.v. 101 | t.o.v. 102 | vb. 103 | w. 104 | -------------------------------------------------------------------------------- /TreeTagger/lib/english-abbreviations: -------------------------------------------------------------------------------- 1 | &Co. 2 | AUG. 3 | Adm. 4 | Ala. 5 | Ald. 6 | App.Cas. 7 | Ariz. 8 | Ark. 9 | Assn. 10 | Assoc. 11 | Att. 12 | Aug. 13 | Av. 14 | Ave. 15 | Bancorp. 16 | Bde. 17 | Bhd. 18 | Blvd. 19 | Brig. 20 | Bros. 21 | C.-in-C. 22 | CO. 23 | CORP. 24 | COS. 25 | Ca. 26 | Calif. 27 | Canada-U.S. 28 | Canadian-U.S. 29 | Capt. 30 | Cas. 31 | Ch. 32 | Ch.App. 33 | Ch.D. 34 | Cia. 35 | Cie. 36 | Cm. 37 | Cmd. 38 | Cmnd. 39 | Co. 40 | Col. 41 | Colo. 42 | Conn. 43 | Corp. 44 | Cos. 45 | Cowp. 46 | Cr.App.R. 47 | Crim.L.R. 48 | D-Mass. 49 | D.Litt. 50 | D.Phil. 51 | DFl. 52 | Dec. 53 | Del. 54 | Dep. 55 | Dept. 56 | Deut. 57 | Diod. 58 | Div. 59 | Dr. 60 | Drs. 61 | Dtr. 62 | Durn. 63 | E.g. 64 | ESQ. 65 | Eph. 66 | Eq. 67 | Eqn. 68 | Eqns. 69 | Esq. 70 | Etc. 71 | Exch. 72 | Exod. 73 | Ext. 74 | FIG. 75 | Fam. 76 | Feb. 77 | Fig. 78 | Figs. 79 | Fla. 80 | Ft. 81 | G.m.b.H. 82 | Ga. 83 | Gen. 84 | Gov. 85 | Hdt. 86 | Hon. 87 | INC. 88 | Ibid. 89 | Ill. 90 | Inc. 91 | Ind. 92 | InfoCorp. 93 | Intercorp. 94 | Invest. 95 | JJ. 96 | JR. 97 | Jan. 98 | Japan-U.S. 99 | Jr. 100 | Jud. 101 | Kan. 102 | Korean-U.S. 103 | Ky. 104 | L.JJ. 105 | L.R.Ir. 106 | LL.M. 107 | LTD. 108 | La. 109 | Lt. 110 | Lt.-Col. 111 | Ltd. 112 | Ltda. 113 | M.Ed. 114 | M.Litt. 115 | M.Phil. 116 | Maj. 117 | Mass. 118 | Md. 119 | Me.T.A. 120 | Messrs. 121 | Mfg. 122 | Mich. 123 | Minn. 124 | Miss. 125 | Mo. 126 | Mod.Rep. 127 | Mont. 128 | Mr. 129 | Mrs. 130 | Ms. 131 | Neb. 132 | Nev. 133 | No. 134 | Non-U.S. 135 | Nos. 136 | Nov. 137 | Oct. 138 | Oe. 139 | Okla. 140 | Ont. 141 | Op. 142 | Ore. 143 | P.o.s. 144 | Pa. 145 | Ph. 146 | Ph.D. 147 | Pp. 148 | Prev. 149 | Prof. 150 | Prop. 151 | Pte. 152 | Ptr. 153 | Pty. 154 | Reg. 155 | Regt. 156 | Rep. 157 | Reps. 158 | Repub. 159 | Ret. 160 | Rev. 161 | Rom. 162 | S.p.A. 163 | Sec. 164 | Sen. 165 | Sens. 166 | Sept. 167 | Sgt. 168 | Sh.Ct. 169 | Sino-U.S. 170 | Soc. 171 | Som. 172 | Soviet-U.S. 173 | Sp. 174 | Sr. 175 | St. 176 | Ste. 177 | Suff. 178 | Syll. 179 | T.B.G.A.S. 180 | Tenn. 181 | Tex. 182 | Thess. 183 | Thuc. 184 | Transp. 185 | Trop. 186 | U.S.-U.K. 187 | U.S.-U.S.S.R. 188 | U.S.P.G.A. 189 | Univ. 190 | V.-C. 191 | Va. 192 | Vict. 193 | Vol. 194 | Vt. 195 | W.Va. 196 | Wash. 197 | Wis. 198 | Wyo. 199 | a-Ex-dividend. 200 | a.c. 201 | a.g.m. 202 | a.k.a. 203 | a.m. 204 | al. 205 | anti-U.S. 206 | approx. 207 | b.s. 208 | bldg. 209 | c.c.d. 210 | c.e.o. 211 | c.f. 212 | c.g. 213 | c.v. 214 | c/s. 215 | cap. 216 | cf. 217 | ch. 218 | cit. 219 | clar. 220 | co. 221 | col. 222 | cols. 223 | constr. 224 | cp. 225 | cwt. 226 | d.c. 227 | d.f. 228 | d.i.l. 229 | d.p.c. 230 | def. 231 | dw. 232 | e-Estimated. 233 | e.g. 234 | e.m.f. 235 | e.p.s.p. 236 | edn. 237 | edns. 238 | est. 239 | etc. 240 | ex-L.C.C. 241 | fig. 242 | fl. 243 | fol. 244 | ft. 245 | gen. 246 | govt. 247 | h.p. 248 | hon. 249 | hrs. 250 | i.c. 251 | i.e. 252 | ibid. 253 | inc. 254 | incl. 255 | juv. 256 | k.p.h. 257 | l.e.d. 258 | lbs. 259 | loc. 260 | m.d. 261 | m.p.h. 262 | msec. 263 | n.d. 264 | n.m.r. 265 | non-U.K. 266 | non-U.S. 267 | norw. 268 | nos. 269 | oz. 270 | ozs. 271 | p. 272 | p.a. 273 | p.c. 274 | p.m. 275 | p.o.s. 276 | p.p.m. 277 | p.s.i. 278 | p.w. 279 | pl. 280 | pls. 281 | pos. 282 | pp. 283 | pres. 284 | president-U.S. 285 | pro-U.S. 286 | q.v. 287 | qq.v. 288 | r.f. 289 | r.h. 290 | r.m.s. 291 | r.m.s.d. 292 | r.p.m. 293 | r.s.s. 294 | ref. 295 | s. 296 | s.a. 297 | s.a.e. 298 | s.d. 299 | s.e.m. 300 | s.r.l. 301 | s.t.p. 302 | spp. 303 | sq.ft. 304 | sq.m. 305 | subss. 306 | v. 307 | v.B. 308 | v.w. 309 | var. 310 | viz. 311 | vol. 312 | vols. 313 | vs. 314 | w.c. 315 | -------------------------------------------------------------------------------- /TreeTagger/lib/english-utf8.par: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/english-utf8.par -------------------------------------------------------------------------------- /TreeTagger/lib/estonian-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | A. 2 | A.-E. 3 | A.W. 4 | B. 5 | C. 6 | Co. 7 | Dr. 8 | E. 9 | F. 10 | G. 11 | H. 12 | I. 13 | Inc. 14 | J. 15 | K. 16 | K.C. 17 | K.R. 18 | Kesk-A. 19 | Kesk-B. 20 | Kr. 21 | L. 22 | L.M.R.A. 23 | Lk. 24 | Lp. 25 | M. 26 | N. 27 | N.G. 28 | N.ö. 29 | P. 30 | Prof. 31 | R. 32 | S. 33 | Sj. 34 | St. 35 | T. 36 | U. 37 | V. 38 | W. 39 | Z. 40 | [13. 41 | a. 42 | ajal. 43 | aug. 44 | cand. 45 | dets. 46 | dots. 47 | dr. 48 | e. 49 | e.Kr. 50 | e.m.a. 51 | e.m.a.. 52 | eluk. 53 | hr. 54 | ingl. 55 | jm. 56 | jms. 57 | jmt. 58 | jne. 59 | joon. 60 | jt. 61 | jõust. 62 | k. 63 | koop. 64 | lg. 65 | lk. 66 | m.a.j. 67 | mln. 68 | mlrd. 69 | mõl. 70 | n.-ö. 71 | n.ö. 72 | nn. 73 | nov. 74 | nr. 75 | nt. 76 | oec. 77 | p. 78 | pen. 79 | pr. 80 | prof. 81 | ptk. 82 | s.t. 83 | sh. 84 | sm. 85 | st. 86 | tuh. 87 | u. 88 | v. 89 | v.a. 90 | vms. 91 | vt. 92 | Ü. 93 | -------------------------------------------------------------------------------- /TreeTagger/lib/estonian-mwls-utf8: -------------------------------------------------------------------------------- 1 | + miljonid 2 | AB.SIacute; N 3 | Al Gore 4 | Dagens Nyheterile 5 | Franz Josephi 6 | GÍ R. TAB 7 | Las Vegasesse 8 | Los Angeles 9 | Los Angelese 10 | Los Angelesega 11 | Mauna Loa 12 | Mauna Loaga 13 | Monte Carlo 14 | Monte Carlos 15 | Mount Everesti 16 | New Delhisse 17 | New Jersey 18 | New Yorgi 19 | New Yorgile 20 | New Yorgis 21 | New Yorgist 22 | New York 23 | New Yorki 24 | Novaja Zemlja 25 | Novaja Zemljaga 26 | Novaja Zemljale 27 | Puerto Rico 28 | Rio de Janeiro 29 | Rio de Janeiros 30 | RÍ N 31 | San Jose 32 | Sao Paulost 33 | Sri Lanka 34 | Sri Lankal 35 | Sri Lankale 36 | Tel Avivis 37 | Tel Avivist 38 | -------------------------------------------------------------------------------- /TreeTagger/lib/finnish-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | A. 2 | A.G. 3 | A.S. 4 | Alk.m. 5 | B. 6 | B.P. 7 | B.V. 8 | Br. 9 | C. 10 | C.I. 11 | C.V. 12 | CO. 13 | Co. 14 | D. 15 | D.J. 16 | Dr. 17 | E. 18 | E.F.C. 19 | Esim. 20 | F. 21 | G. 22 | H. 23 | HUOM. 24 | Huom. 25 | I. 26 | I.B. 27 | Inc. 28 | J. 29 | K. 30 | Kenr.maj. 31 | Kok. 32 | Ks. 33 | L. 34 | L.R. 35 | Lindl. 36 | Ltd. 37 | M. 38 | M.A. 39 | Milj. 40 | Mill. 41 | Mr. 42 | N. 43 | N.A. 44 | N.V. 45 | Ns. 46 | O. 47 | P. 48 | P.O. 49 | Pers. 50 | Prof. 51 | Puh. 52 | Q. 53 | R. 54 | S. 55 | S.A. 56 | S.I.T.E. 57 | S.L. 58 | S.P.A. 59 | Sp. 60 | St. 61 | T. 62 | Tekn. 63 | Tél. 64 | U. 65 | U.V.M. 66 | V. 67 | Vrt. 68 | W. 69 | X. 70 | Y. 71 | Yht. 72 | Z. 73 | Z.z. 74 | a. 75 | a.s. 76 | al. 77 | alv. 78 | art. 79 | b. 80 | c. 81 | cf. 82 | d. 83 | d.d. 84 | d.o.o. 85 | e. 86 | e.V. 87 | e.g. 88 | e.m.l. 89 | eläk. 90 | em. 91 | esim. 92 | etc. 93 | f. 94 | g. 95 | h. 96 | huom. 97 | i. 98 | i.e. 99 | j. 100 | jatk. 101 | jne. 102 | k. 103 | k.y. 104 | kans. 105 | ko. 106 | ks. 107 | kust. 108 | l. 109 | m. 110 | milj. 111 | ml. 112 | mm. 113 | mod. 114 | mrd. 115 | myöh. 116 | n. 117 | n.a. 118 | nk. 119 | ns. 120 | o. 121 | o.o. 122 | p. 123 | p.m. 124 | pl. 125 | prov. 126 | puh. 127 | r. 128 | s. 129 | s.a. 130 | s.r.o. 131 | so. 132 | sp. 133 | spp. 134 | synt. 135 | t. 136 | t.p. 137 | tj. 138 | tms. 139 | ts. 140 | u. 141 | v. 142 | var. 143 | varat. 144 | vrt. 145 | ylim. 146 | ym. 147 | yms. 148 | z. 149 | -------------------------------------------------------------------------------- /TreeTagger/lib/french-abbreviations: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/french-abbreviations -------------------------------------------------------------------------------- /TreeTagger/lib/french-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | rendez-vous 2 | d'abord 3 | d'accord 4 | d'ailleurs 5 | d'après 6 | d'autant 7 | d'œuvre 8 | d'oeuvre 9 | c'est-à-dire 10 | moi-même 11 | toi-même 12 | lui-même 13 | elle-même 14 | nous-mêmes 15 | vous-mêmes 16 | eux-mêmes 17 | elles-mêmes 18 | par-ci 19 | par-là 20 | Rendez-vous 21 | D'abord 22 | D'accord 23 | D'ailleurs 24 | D'après 25 | D'autant 26 | D'œuvre 27 | D'oeuvre 28 | C'est-à-dire 29 | Moi-même 30 | Toi-même 31 | Lui-même 32 | Elle-même 33 | Nous-mêmes 34 | Vous-mêmes 35 | Eux-mêmes 36 | Elles-mêmes 37 | Par-ci 38 | Par-là 39 | -------------------------------------------------------------------------------- /TreeTagger/lib/french.par: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/french.par -------------------------------------------------------------------------------- /TreeTagger/lib/galician-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | ... 2 | 1. 3 | 2. 4 | A ti , meu Cambados ... 5 | A. S. A. 6 | Antonio F. C. 7 | B. O. E. 8 | C. I. G. 9 | C. L. U. 10 | C. S. I. 11 | C.L.U. 12 | Condeport S. A. 13 | Construccións e Auxiliar de Ferrocarriles S. A. 14 | D. 15 | D. O. 16 | Donald Trump Jr . 17 | Dr. 18 | E.R.M. 19 | Empresa Freire S. L. 20 | Etc. 21 | Famosa , S. A. 22 | Feiraco , Soc. Coop. Ltda. 23 | Fuerzas Eléctricas del Noroeste , S. A. 24 | H. V. D. K. 25 | I. G. E. 26 | I. O. L. 27 | Intelsis Sistemas Inteligentes , S. A. 28 | Inverpesca , S. A. 29 | J. G. B. A. 30 | J. M. A. A. 31 | J.P.G. 32 | K.O. 33 | M. G. G. 34 | M. I. R. 35 | Mr. 36 | P. I. B. 37 | P. Juan Rodríguez Cabrea C. M. 38 | P.G.C. 39 | Pedro J. 40 | S. A. 41 | Sr. 42 | Sras. 43 | Sres . 44 | Sres. 45 | TEL . 46 | Telefónica S. A. 47 | Tengo una pregunta para usted ... 48 | Turgalicia , S. A. 49 | U. H. T. 50 | U. S. A. 51 | V. R. G. 52 | Vde . 53 | Vde. 54 | art . 55 | art. 56 | dra . 57 | etc. 58 | h. 59 | núm . 60 | ptas . 61 | pts. 62 | s. 63 | vs. 64 | -------------------------------------------------------------------------------- /TreeTagger/lib/german-abbreviations: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/german-abbreviations -------------------------------------------------------------------------------- /TreeTagger/lib/german-lexicon-utf8.txt: -------------------------------------------------------------------------------- 1 | muessen VMFIN müssen VMINF müssen 2 | -------------------------------------------------------------------------------- /TreeTagger/lib/german-lexicon.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/german-lexicon.txt -------------------------------------------------------------------------------- /TreeTagger/lib/greek-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | ... 2 | A. 3 | A.C. 4 | A.L. 5 | A.S. 6 | AA. 7 | AAG. 8 | ADR. 9 | ATS. 10 | AUC. 11 | B. 12 | B.A. 13 | B.J. 14 | B.S. 15 | B.V. 16 | B.Y. 17 | BVE. 18 | C. 19 | C.J. 20 | C.J.W. 21 | C.W.A. 22 | C.Ρ.C. 23 | CA. 24 | CDC. 25 | CE. 26 | CEE. 27 | CES. 28 | CFC. 29 | CRC. 30 | D. 31 | D.A.C. 32 | D.B. 33 | D.C. 34 | D.F. 35 | D.I. 36 | D.W. 37 | DDT. 38 | DNA. 39 | E. 40 | ECU. 41 | EIM. 42 | EL. 43 | EPF. 44 | ERV. 45 | EUR. 46 | EΕΓ. 47 | F. 48 | F.G. 49 | FAO. 50 | FAS. 51 | FDA. 52 | FVC. 53 | G. 54 | GAG. 55 | GEM. 56 | H. 57 | H.A. 58 | H.N. 59 | HBO. 60 | HCV. 61 | HF. 62 | HIV. 63 | HRC. 64 | I. 65 | I.K. 66 | I.U.B.E. 67 | ICT. 68 | IGS. 69 | II. 70 | IIA. 71 | IIB. 72 | III. 73 | IIΙ. 74 | ILO. 75 | ITL. 76 | IU. 77 | IV. 78 | IX. 79 | J. 80 | J.C. 81 | J.D. 82 | J.L. 83 | J.N. 84 | J.R. 85 | J.S. 86 | J.T. 87 | J.W. 88 | JAI. 89 | K. 90 | K.B. 91 | K.W. 92 | KΓΠ. 93 | L. 94 | L.A. 95 | L.B. 96 | L.E. 97 | L.J. 98 | L.S. 99 | M. 100 | M.B. 101 | M.D. 102 | M.J. 103 | M.P. 104 | M.S. 105 | MAV. 106 | MG. 107 | MKO. 108 | MTL. 109 | MΑV. 110 | N. 111 | N.C.Ρ.C. 112 | O. 113 | O.E.E.K. 114 | O.T.A. 115 | OR. 116 | P. 117 | P.L. 118 | P.O. 119 | P.W. 120 | PSA. 121 | R. 122 | R.A. 123 | R.E. 124 | R.F. 125 | R.J. 126 | R.K. 127 | R.P. 128 | R.R. 129 | R.S. 130 | R.U.E. 131 | RNΑ. 132 | S. 133 | S.A. 134 | S.B. 135 | S.D. 136 | S.F. 137 | S.H.E. 138 | S.J. 139 | S.L. 140 | SA. 141 | SLC. 142 | SSA. 143 | T. 144 | T.D. 145 | T.E.I. 146 | T.P. 147 | T.Θ. 148 | U. 149 | U.S. 150 | UOE. 151 | US. 152 | V. 153 | V.K. 154 | VI. 155 | VII. 156 | VΙ. 157 | VΙΙ. 158 | W. 159 | W.F. 160 | W.G. 161 | WGM. 162 | WHO. 163 | X. 164 | XI. 165 | XV. 166 | Y.K. 167 | Z. 168 | ZPO. 169 | c.p.c. 170 | Α. 171 | Α.Ε. 172 | Α.Ε.Ι. 173 | Α.Ε.Π. 174 | Α.Σ.Κ.Τ. 175 | ΑΒ. 176 | ΑΓ. 177 | ΑΔ. 178 | ΑΕΠ. 179 | ΑΙΜ. 180 | ΑΣΕ. 181 | Β. 182 | Β.Α. 183 | Β.Ι.Ο. 184 | ΒΑΤ. 185 | Γ. 186 | Γ.Γ.Ε.Τ. 187 | Γ.Δ. 188 | Γ.Σ.Ε.Ε. 189 | Γ.Υ.Σ. 190 | ΓΙΙ. 191 | ΓΡΑ.Σ.Ε.Π. 192 | Δ. 193 | Δ.Ε.Π. 194 | ΔΓΕ. 195 | ΔΓΠ. 196 | ΔΔ. 197 | ΔΕΕ. 198 | ΔΙ.Κ.Α.Τ.Σ.Α. 199 | ΔΙ.ΚΑ.Τ.Σ.Α. 200 | ΔΟΕ. 201 | Ε. 202 | Ε.Α.Π. 203 | Ε.Ε. 204 | Ε.Ε.Υ. 205 | Ε.Κ.Κ.Ε.Ε. 206 | Ε.Κ.Τ. 207 | Ε.ΚΕ.ΠΙΣ. 208 | Ε.Ο.Σ. 209 | Ε.Ο.Σ.Δ.Ο.Σ. 210 | Ε.Ο.Τ. 211 | Ε.Π. 212 | Ε.Σ.Ε.Ε.Κ. 213 | Ε.Σ.Ο.Α. 214 | Ε.Σ.Ο.Α.Β. 215 | ΕK. 216 | ΕΑ. 217 | ΕΕ. 218 | ΕΕΑ. 219 | ΕΕΓ. 220 | ΕΕΚ. 221 | ΕΕΠ. 222 | ΕΕΥ. 223 | ΕΚ. 224 | ΕΚA. 225 | ΕΚΤ. 226 | ΕΟΔ. 227 | ΕΟΤ. 228 | ΕΟΧ. 229 | ΕΡΑ. 230 | ΕΣΟ. 231 | ΕΣΠ. 232 | ΕΤΑ. 233 | ΕΤΕ. 234 | ΕΤΠ. 235 | Ζ. 236 | Η. 237 | Η.Β. 238 | Η.Π.Α. 239 | ΗΒ. 240 | ΗΚΓ. 241 | ΗΜΠ. 242 | ΗΠΑ. 243 | Θ. 244 | Θ.Ε. 245 | Ι. 246 | Ι.Γ.Μ.Ε. 247 | Ι.Ε.Κ. 248 | Ι.Κ.Υ. 249 | Ι.Τ.Ε. 250 | ΙV. 251 | ΙΑ. 252 | ΙΒ. 253 | ΙΓ. 254 | ΙΔ. 255 | ΙΕ. 256 | ΙΖ. 257 | ΙΗ. 258 | ΙΘ. 259 | ΙΙ. 260 | ΙΙΙ. 261 | ΙΣΤ. 262 | Ιαν. 263 | Ιαν. 264 | Κ. 265 | Κ.Δ.Π. 266 | Κ.Ε.Θ.Ι. 267 | Κ.Ε.Κ. 268 | Κ.Π.Α. 269 | Κ.Π.Γ. 270 | Κ.Π.Ν. 271 | ΚΑ. 272 | ΚΒ. 273 | ΚΓ. 274 | ΚΔ. 275 | ΚΕ. 276 | ΚΕ.Θ.Ε.Α. 277 | ΚΕ.ΠΕ.Α. 278 | ΚΕ.ΣΥ.Π. 279 | ΚΠΔ. 280 | Λ. 281 | Μ. 282 | Μ.Δ.Ε. 283 | Μ.Κ.Ο. 284 | ΜΔΚ. 285 | ΜΚΟ. 286 | ΜΜΕ. 287 | ΜΣΕ. 288 | Ν. 289 | Ν.V. 290 | Ν.Α. 291 | Ν.Α.Τ.Ο. 292 | Ν.Π.Δ.Δ. 293 | Ο. 294 | Ο.Α.Ε.Δ. 295 | Ο.Δ. 296 | Ο.Ε.Ε.Κ. 297 | Ο.Η.Ε. 298 | Ο.ΚΑ.ΝΑ. 299 | Ο.Ο.Σ.Α. 300 | Ο.Σ.Σ. 301 | Ο.Τ.Α. 302 | ΟΗΕ. 303 | ΟΠΔ. 304 | ΟΡ. 305 | ΟΥΕ. 306 | Π. 307 | Π.Δ. 308 | Π.Ι. 309 | Π.Μ.Σ. 310 | Π.Ο.Δ.Ε. 311 | Π.Ο.Τ. 312 | ΠΕ.ΧΩ.ΔΕ. 313 | ΠΟΕ. 314 | ΠΟΥ. 315 | ΠΣΠ. 316 | Σ. 317 | Σ.Δ.Ε. 318 | Σ.Ε.Κ. 319 | ΣΕΒ. 320 | ΣΕΠ. 321 | ΣΚ. 322 | ΣΤ. 323 | ΣΥΚ. 324 | Σεπτ. 325 | Τ. 326 | Τ.Ε.Ε. 327 | Τ.Ε.Ι. 328 | Τ.Θ. 329 | Τ.Κ. 330 | Τ.Ν. 331 | ΤΠΕ. 332 | ΥΠ.Ε.Π.Θ. 333 | ΥΠΠ. 334 | Φ. 335 | ΦΠΑ. 336 | Χ. 337 | ΧΙΚ. 338 | ΧΡ. 339 | βλ. 340 | κ.λπ. 341 | σας. 342 | της. 343 | -------------------------------------------------------------------------------- /TreeTagger/lib/italian-abbreviations: -------------------------------------------------------------------------------- 1 | L. 2 | Lit. 3 | art. 4 | lett. 5 | n. 6 | no. 7 | pagg. 8 | prot. 9 | tel. 10 | po' 11 | -------------------------------------------------------------------------------- /TreeTagger/lib/latin-abbreviations: -------------------------------------------------------------------------------- 1 | A. 2 | C. 3 | Cn. 4 | D. 5 | L. 6 | LIII. 7 | M. 8 | Q. 9 | Ser. 10 | T. 11 | Ti. 12 | act. 13 | animalib. 14 | antec. 15 | art. 16 | boet. 17 | cap. 18 | comment. 19 | consol. 20 | cor. 21 | corp. 22 | dist. 23 | distinct. 24 | eccl. 25 | etc. 26 | ethic. 27 | fin. 28 | gener. 29 | generat. 30 | hier. 31 | lib. 32 | luc. 33 | matth. 34 | metaph. 35 | metaphys. 36 | philip. 37 | philipp. 38 | phys. 39 | physic. 40 | praeced. 41 | princ. 42 | qu. 43 | quaest. 44 | quaestiunc. 45 | respons. 46 | s. 47 | thess. 48 | trin. 49 | trinit. 50 | ult. 51 | -------------------------------------------------------------------------------- /TreeTagger/lib/latin-mwls: -------------------------------------------------------------------------------- 1 | quam ob rem 2 | -------------------------------------------------------------------------------- /TreeTagger/lib/middle-high-german-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | Prof. 2 | Dr. 3 | -------------------------------------------------------------------------------- /TreeTagger/lib/polish-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | 10-tys. 2 | 2-proc. 3 | 400-tys. 4 | 50-proc. 5 | 5-proc. 6 | 7-proc. 7 | A. 8 | abp. 9 | A.D. 10 | agd. 11 | al. 12 | Al. 13 | And. 14 | ang. 15 | arch. 16 | art. 17 | Art. 18 | asp. 19 | b. 20 | B. 21 | Bie. 22 | bm. 23 | bp. 24 | br. 25 | Br. 26 | b.r. 27 | bryg. 28 | C. 29 | ccm. 30 | cd. 31 | Cd. 32 | c.d.n. 33 | Ch. 34 | c.k. 35 | cm. 36 | cm 3. 37 | c.o. 38 | cyt. 39 | cz. 40 | Cz. 41 | czyt. 42 | d. 43 | D. 44 | Dep. 45 | dh. 46 | dł. 47 | dn. 48 | doc. 49 | Doc. 50 | dol. 51 | Dol. 52 | dot. 53 | dr. 54 | Dr. 55 | ds. 56 | dyr. 57 | Dyr. 58 | dysk. 59 | dyw. 60 | Dz. 61 | Dz.U. 62 | e. 63 | E. 64 | ed. 65 | -Ekon. 66 | el. 67 | emergie. 68 | etc. 69 | ew. 70 | e.w. 71 | F. 72 | fot. 73 | Fot. 74 | fund. 75 | g. 76 | G. 77 | Gd. 78 | GD. 79 | gen. 80 | Gen. 81 | geogr. 82 | GHz. 83 | głęb. 84 | gm. 85 | godz. 86 | Godz. 87 | gr. 88 | Graż. 89 | h. 90 | H. 91 | ha. 92 | hab. 93 | H-K. 94 | hm. 95 | hr. 96 | I. 97 | ilustr. 98 | im. 99 | IMHO. 100 | in. 101 | INF. 102 | insp. 103 | inż. 104 | Inż. 105 | is. 106 | itd. 107 | itede. 108 | itp. 109 | j. 110 | J. 111 | jez. 112 | Jez. 113 | jr. 114 | k. 115 | K. 116 | kard. 117 | kasz. 118 | kaszb. 119 | kaszub. 120 | kat. 121 | kg. 122 | KK. 123 | kl. 124 | km. 125 | km2. 126 | km². 127 | kmdr. 128 | kol. 129 | kolej. 130 | kom. 131 | Konst. 132 | kop. 133 | k.p.c. 134 | kpt. 135 | Kryst. 136 | ks. 137 | Ks. 138 | kw. 139 | k.w. 140 | l. 141 | L. 142 | Ł. 143 | łac. 144 | Le. 145 | lek. 146 | Lic. 147 | lit. 148 | Ltd. 149 | m. 150 | M. 151 | m2. 152 | m 2. 153 | mag. 154 | maks. 155 | Małg. 156 | Mat. 157 | mec. 158 | Mec. 159 | med. 160 | mg. 161 | Mg. 162 | mies. 163 | mieszk. 164 | min. 165 | Min. 166 | m.in. 167 | m. in. 168 | M.in. 169 | m.inn. 170 | Mjr. 171 | mk. 172 | mkw. 173 | Mł. 174 | mld. 175 | mln. 176 | mm. 177 | M.P. 178 | ms. 179 | m.st. 180 | n. 181 | N. 182 | nadkom. 183 | Najśw. 184 | nb. 185 | Nb. 186 | n.e. 187 | niem. 188 | nm. 189 | np. 190 | Np. 191 | n.p.m. 192 | nr. 193 | nt. 194 | N.T. 195 | o. 196 | O. 197 | obw. 198 | odbud. 199 | Oddz. 200 | odl. 201 | odp. 202 | ok. 203 | Ok. 204 | OO. 205 | o.o. 206 | Op. 207 | oprac. 208 | os. 209 | p. 210 | P. 211 | pap. 212 | par. 213 | paraf. 214 | pd. 215 | PeeS. 216 | pkt. 217 | pl. 218 | Pl. 219 | Płd. 220 | płk. 221 | płn. 222 | Płn. 223 | pn. 224 | p.n.e. 225 | p.o. 226 | pocz. 227 | podkom. 228 | podst. 229 | poj. 230 | pojed. 231 | pok. 232 | pol. 233 | poł. 234 | położ. 235 | pon. 236 | P.O.N.R.P.IM.MJR.H.D.H. 237 | por. 238 | Por. 239 | pos. 240 | post. 241 | pow. 242 | poz. 243 | późn. 244 | pp. 245 | ppanc. 246 | ppor. 247 | proc. 248 | prof. 249 | Prof. 250 | proj. 251 | prok. 252 | prom. 253 | przebud. 254 | Przeł. 255 | przyj. 256 | przyp. 257 | ps. 258 | Ps. 259 | PS. 260 | p.s. 261 | P.S. 262 | pt. 263 | p.t. 264 | pw. 265 | PW. 266 | Q. 267 | r. 268 | R. 269 | red. 270 | Red. 271 | RED. 272 | Ref. 273 | rez. 274 | reż. 275 | rozbud. 276 | rozdz. 277 | R.P. 278 | ryc. 279 | Ryc. 280 | rys. 281 | rz. 282 | s. 283 | S. 284 | Ś. 285 | SA. 286 | S.A. 287 | s.c. 288 | S.C. 289 | sek. 290 | sen. 291 | Ska. 292 | sp. 293 | Sp. 294 | śp. 295 | ś.p. 296 | Społ. 297 | śr. 298 | ss. 299 | st. 300 | St. 301 | ST. 302 | S.T. 303 | Stow. 304 | str. 305 | S.T.W. 306 | św. 307 | Św. 308 | ŚW. 309 | sygn. 310 | Sygn. 311 | Sz. 312 | szer. 313 | sześc. 314 | sześć. 315 | szt. 316 | t. 317 | T. 318 | tab. 319 | Tad. 320 | tel. 321 | Temp. 322 | Th. 323 | tj. 324 | tow. 325 | Tow. 326 | tyg. 327 | tys. 328 | ty s. 329 | tzn. 330 | Tzn. 331 | tzw. 332 | Tzw. 333 | u. 334 | U. 335 | ub. 336 | ub.r. 337 | ul. 338 | Ul. 339 | uł. 340 | ur. 341 | ust. 342 | V. 343 | w. 344 | W. 345 | wag. 346 | w.c. 347 | w. c. 348 | wf. 349 | wkł. 350 | wł. 351 | Wł. 352 | WŁ. 353 | Wlkp. 354 | woj. 355 | wsch. 356 | Wsch. 357 | -wsch. 358 | ww. 359 | w.w. 360 | wyd. 361 | wys. 362 | wysok. 363 | wyst. 364 | wzgl. 365 | X. 366 | z. 367 | Z. 368 | ż. 369 | Ż. 370 | zach. 371 | założ. 372 | zam. 373 | zast. 374 | Zaw. 375 | zbud. 376 | zdj. 377 | Zen. 378 | zł. 379 | zm. 380 | zn. 381 | Zn. 382 | zob. 383 | Zw. 384 | -------------------------------------------------------------------------------- /TreeTagger/lib/portuguese-abbreviations-utf8: -------------------------------------------------------------------------------- 1 | dr. 2 | dra. 3 | drs. 4 | drª. 5 | esq. 6 | etc. 7 | ex. 8 | exa. 9 | exma. 10 | exmo. 11 | exmos. 12 | exª. 13 | iac. 14 | ibid. 15 | mr. 16 | portuga. 17 | prof. 18 | ref. 19 | séc. 20 | sos. 21 | sr. 22 | sra. 23 | srs. 24 | stª. 25 | stº. 26 | telef. 27 | univ. 28 | -------------------------------------------------------------------------------- /TreeTagger/lib/romanian-abbreviations: -------------------------------------------------------------------------------- 1 | a. 2 | abr. 3 | a.c. 4 | acad. 5 | acc. 6 | ad. 7 | a.d. 8 | a.D. 9 | adj. 10 | A.D.P. 11 | adr. 12 | ad_us. 13 | adv. 14 | A.F.D.P.R. 15 | A.G. 16 | agr. 17 | agric. 18 | agron. 19 | a.h.l. 20 | a.i. 21 | Al. 22 | alb. 23 | alc. 24 | al.e. 25 | alim. 26 | alt. 27 | a.m. 28 | Amb. 29 | amid. 30 | antic. 31 | aor. 32 | ap. 33 | a.p. 34 | aprox. 35 | aq. 36 | Ar. 37 | arg. 38 | arh. 39 | Arh. 40 | arheol. 41 | arhit. 42 | art. 43 | as. 44 | A.S. 45 | A.S.E. 46 | asist. 47 | astr. 48 | astron. 49 | A.T.F. 50 | aug. 51 | Aus. 52 | Austr. 53 | av. 54 | Av. 55 | Ave. 56 | B. 57 | bd. 58 | Bd. 59 | B.D.F. 60 | Bel. 61 | B.I. 62 | biochim. 63 | biol. 64 | bir. 65 | bis. 66 | bl. 67 | B.N.R. 68 | bot. 69 | buc. 70 | bumb. 71 | C. 72 | Ca. 73 | cab. 74 | C.A.C. 75 | cal. 76 | Cal. 77 | caol. 78 | cap. 79 | C.A.P. 80 | cart. 81 | căt. 82 | categ. 83 | C.C. 84 | C.C.P.C.R. 85 | cd. 86 | C.D.R. 87 | Cdt. 88 | C.E. 89 | ceh. 90 | C.F.R. 91 | cg. 92 | Ch. 93 | chim. 94 | C.I.A. 95 | cim. 96 | C.I.O. 97 | cit. 98 | Cl. 99 | C.M. 100 | C.N.E.A.A. 101 | Co. 102 | cofet. 103 | col. 104 | Col. 105 | com. 106 | Comb. 107 | comerc. 108 | Comp. 109 | con. 110 | concr. 111 | conf. 112 | conj. 113 | constr. 114 | cont. 115 | conv. 116 | coop. 117 | cor. 118 | Cor. 119 | cr. 120 | C.R. 121 | crt. 122 | C.S.M. 123 | C.T. 124 | C.T.P. 125 | cuv. 126 | c.v. 127 | C.V. 128 | d. 129 | D. 130 | d.a. 131 | Dan. 132 | dat. 133 | D.B. 134 | dcd. 135 | d.civ. 136 | D.D.T. 137 | d.e. 138 | dec. 139 | def. 140 | dep. 141 | depr. 142 | desfac. 143 | D.G.P.M.O.S. 144 | D.I. 145 | dial. 146 | dir. 147 | div. 148 | dl. 149 | D.L. 150 | dol. 151 | d.pen. 152 | dr. 153 | dr.doc. 154 | Dr.H.C. 155 | Dr.J. 156 | D.S. 157 | dv. 158 | dvs. 159 | E. 160 | e.a. 161 | e.c. 162 | econ. 163 | ed. 164 | E.D.P. 165 | E.E.T. 166 | e.g. 167 | electr. 168 | el.tehn. 169 | e.m. 170 | e.n. 171 | engl. 172 | e.o. 173 | ep. 174 | e.p. 175 | esper. 176 | et. 177 | et.al. 178 | etc. 179 | et_comp. 180 | ev. 181 | ev.med. 182 | ex. 183 | Exc. 184 | excl. 185 | ex.lib. 186 | expl. 187 | expo. 188 | expr. 189 | ext. 190 | E.Z. 191 | f. 192 | f.a. 193 | fabr. 194 | farm. 195 | fbt. 196 | F.C. 197 | F.D. 198 | feb. 199 | f.f. 200 | fierb. 201 | fig. 202 | fil. 203 | filol. 204 | fin. 205 | fiz. 206 | f.m. 207 | F.M. 208 | fo. 209 | fon. 210 | fort. 211 | fotogr. 212 | fr. 213 | Fr. 214 | f.r. 215 | frig. 216 | F.S.C.E. 217 | f.v. 218 | g. 219 | G. 220 | gal. 221 | g.cal. 222 | g.cm 223 | gen. 224 | Gen. 225 | geogr. 226 | geol. 227 | geom. 228 | ger. 229 | germ. 230 | Gh. 231 | gr. 232 | Gr. 233 | gram. 234 | grama. 235 | greut. 236 | h. 237 | H. 238 | h.a. 239 | hârt. 240 | H.G. 241 | h.l. 242 | h.m. 243 | h.n. 244 | hort. 245 | hot. 246 | I. 247 | ian. 248 | I.A.S. 249 | ib. 250 | ibid. 251 | I.C. 252 | I.C.A. 253 | î.Cr. 254 | I.C.S. 255 | id. 256 | I.D. 257 | i.e. 258 | î.e.n. 259 | I.J.P. 260 | Il. 261 | I.L. 262 | imper. 263 | imperf. 264 | impers. 265 | Imp.Exp. 266 | incl. 267 | ind. 268 | inf. 269 | ing. 270 | inst. 271 | int. 272 | interj. 273 | înv. 274 | invar. 275 | I.P. 276 | i.q. 277 | ir. 278 | ist. 279 | iul. 280 | iun. 281 | j. 282 | J.O. 283 | jr. 284 | jud. 285 | jur. 286 | k. 287 | kgf. 288 | K.O. 289 | l. 290 | L. 291 | L.A. 292 | lag. 293 | lat. 294 | lat.med. 295 | lat.pop. 296 | lb. 297 | l.c. 298 | lingv. 299 | lit. 300 | lm. 301 | loc. 302 | loc.adj. 303 | loc.adv. 304 | loc.col. 305 | loc.conj. 306 | locot. 307 | loc.prep. 308 | loc.verb. 309 | log. 310 | lt. 311 | lt.col. 312 | lt.maj. 313 | m. 314 | M. 315 | mag. 316 | magh. 317 | M.A.I. 318 | măn. 319 | mart. 320 | mat. 321 | max. 322 | mc. 323 | mcn. 324 | mcsec. 325 | md. 326 | mec. 327 | med. 328 | mem. 329 | mest. 330 | met. 331 | meteor. 332 | metr. 333 | mijl. 334 | mil. 335 | milen. 336 | min. 337 | mit. 338 | mitol. 339 | mld. 340 | m.m.c.perf. 341 | m.m.p. 342 | mob. 343 | mon. 344 | Mons. 345 | morm.com. 346 | m.p. 347 | M.P. 348 | Mr. 349 | Mrs. 350 | ms. 351 | mss. 352 | M.S.V. 353 | mt. 354 | mţii. 355 | muz. 356 | M.V. 357 | mw. 358 | n. 359 | N. 360 | n.a. 361 | N.a. 362 | naţ. 363 | nav. 364 | N.B. 365 | N.E. 366 | neacc. 367 | nefer. 368 | neg. 369 | nehot. 370 | ngr. 371 | N.I. 372 | n.l. 373 | n.n. 374 | no. 375 | nom. 376 | Norv. 377 | nov. 378 | n.pr. 379 | nr. 380 | n.r. 381 | N.trad. 382 | nucl. 383 | num. 384 | num.adv 385 | num.card. 386 | num.col. 387 | num.multipl. 388 | num.ord. 389 | N.V. 390 | O. 391 | ob. 392 | obs. 393 | oc. 394 | Occ. 395 | oct. 396 | Od. 397 | of. 398 | of.p. 399 | O.I. 400 | op. 401 | op.cit. 402 | opt. 403 | or. 404 | Or. 405 | org. 406 | ov. 407 | O.Z.N. 408 | p. 409 | P. 410 | p.a. 411 | pag. 412 | paleont. 413 | parf. 414 | part. 415 | pas. 416 | păs. 417 | P.C.R. 418 | ped. 419 | perf.c. 420 | perf.s. 421 | pict. 422 | pisc. 423 | pl. 424 | plut. 425 | p.m. 426 | pod. 427 | pp. 428 | pr. 429 | pref. 430 | prep. 431 | prez. 432 | princ. 433 | prod. 434 | prof. 435 | P.S. 436 | psih. 437 | pt. 438 | p.t. 439 | ptr. 440 | pulv. 441 | r. 442 | R.A. 443 | rad. 444 | recipr. 445 | refl. 446 | reg. 447 | relig. 448 | Rm. 449 | rot. 450 | rus. 451 | s. 452 | S. 453 | s.a. 454 | S.A. 455 | ş.a. 456 | S.A.H. 457 | ş.a.m.d. 458 | sc. 459 | S.C. 460 | şc. 461 | scil. 462 | ş.cl. 463 | s.d. 464 | S.E. 465 | sec. 466 | sect. 467 | sem. 468 | serg. 469 | serg.maj. 470 | serv. 471 | sf. 472 | Sf.Pancras 473 | sg. 474 | Sh. 475 | sl. 476 | s.l. 477 | slov. 478 | slt. 479 | S.M. 480 | ş.m.a. 481 | s.n. 482 | sold. 483 | şos. 484 | spt. 485 | Sr. 486 | S.R.L. 487 | ss. 488 | s.s. 489 | Şt. 490 | şt.nat. 491 | str. 492 | stud. 493 | S.U.A. 494 | subst. 495 | suc. 496 | suf. 497 | t. 498 | T. 499 | T.B.C. 500 | tehn. 501 | tel. 502 | T.F.F. 503 | Tg. 504 | TG. 505 | T.J. 506 | top. 507 | tov. 508 | Tr. 509 | TR. 510 | trad. 511 | tranz. 512 | trim. 513 | tv. 514 | TV. 515 | U.E. 516 | u.m. 517 | univ. 518 | urb. 519 | urm. 520 | u.s. 521 | v. 522 | V. 523 | vac. 524 | vân. 525 | var. 526 | var.lect. 527 | vb. 528 | vf. 529 | V.F. 530 | vid. 531 | viit. 532 | v.l. 533 | vo. 534 | voc. 535 | vol. 536 | vs. 537 | v.v. 538 | w.c. 539 | W.C. 540 | Z. 541 | zool. 542 | -------------------------------------------------------------------------------- /TreeTagger/lib/romanian-tokens: -------------------------------------------------------------------------------- 1 | fra-te 2 | până-n 3 | într-adevăr 4 | într-acolo 5 | într-adins 6 | într-atât 7 | de-abia 8 | de-adevăratelea 9 | nici-o 10 | bu-u-un 11 | -------------------------------------------------------------------------------- /TreeTagger/lib/spanish-abbreviations: -------------------------------------------------------------------------------- 1 | Ref. 2 | Vol. 3 | etc. 4 | App. 5 | Rec. 6 | -------------------------------------------------------------------------------- /TreeTagger/lib/spanish-mwls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amineabdaoui/python-heideltime/74f25723e82342d03c0baa20c8d67669efd9908b/TreeTagger/lib/spanish-mwls -------------------------------------------------------------------------------- /TreeTagger/lib/spanish-mwls-utf8: -------------------------------------------------------------------------------- 1 | A diferencia de 2 | A diferencia del 3 | A fin de 4 | A lo largo de 5 | A medida que 6 | A menudo 7 | A partir de 8 | A pesar de 9 | A título de 10 | A veces 11 | Abu Dhabi 12 | Ahora bien 13 | Al contrario 14 | Al mismo tiempo 15 | Antigua y Barbuda 16 | Así pues 17 | Burkina Faso 18 | Cabo Verde 19 | Con relación a 20 | Con respecto a 21 | Con respecto al 22 | Costa Rica 23 | Cote d'Ivoire 24 | Dado que 25 | De acuerdo con 26 | De conformidad con 27 | De hecho 28 | De ordinario 29 | Debido a 30 | Dentro de 31 | Dentro del 32 | Después de 33 | Después del 34 | El Salvador 35 | Emiratos Árabes Unidos 36 | En cambio 37 | En caso de que 38 | En consecuencia 39 | En cuanto a 40 | En efecto 41 | En especial 42 | En función de 43 | En general 44 | En lo que 45 | En lugar de 46 | En particular 47 | En principio 48 | En relación con 49 | En todo caso 50 | En vista de 51 | Entre tanto 52 | Es decir 53 | Estados Unidos 54 | Estados Unidos de América 55 | Esto es 56 | Fuera de 57 | Gracias a 58 | Ha de 59 | Habrá que 60 | Hasta ahora 61 | Hay que 62 | Islas Vírgenes 63 | Joint Committee Technical 64 | Junto con 65 | Mar del Plata 66 | Mientras que 67 | Mientras tanto 68 | Naciones Unidas 69 | No obstante 70 | Nueva Caledonia 71 | Nueva Delhi 72 | Nueva York 73 | Nueva Zelandia 74 | Papua Nueva Guinea 75 | Países Bajos 76 | Pese a 77 | Por consiguiente 78 | Por ejemplo 79 | Por el contrario 80 | Por el momento 81 | Por encima de 82 | Por encima del 83 | Por lo general 84 | Por lo tanto 85 | Por medio de 86 | Por medio del 87 | Por regla general 88 | Por tanto 89 | Por último 90 | Puerto Rico 91 | Puesto que 92 | Ras Al Khaimah 93 | Reino Unido 94 | Reino Unido de Gran Bretaña 95 | Saint Kitts 96 | San Marino 97 | San Pedro 98 | San Vicente 99 | Santa Lucía 100 | Santo Tomé 101 | Si bien 102 | Siempre que 103 | Sierra Leona 104 | Sin embargo 105 | Sri Lanka 106 | Tal vez 107 | Tan pronto como 108 | Tienen que 109 | Umm al Qaiwain 110 | Una vez que 111 | Unión de Repúblicas Socialistas Soviéticas 112 | Vírgenes Británicas 113 | a cambio de 114 | a excepción de 115 | a fin de 116 | a fondo 117 | a la derecha 118 | a la izquierda de 119 | a la izquierda del 120 | a la que 121 | a la vez 122 | a lo largo de 123 | a lo largo del 124 | a medida que 125 | a menos que 126 | a menudo 127 | a nivel de 128 | a partir de 129 | a partir del 130 | a pesar de 131 | a primera vista 132 | a priori 133 | a propósito 134 | a punto de 135 | a razón de 136 | a saber 137 | a través de 138 | a través del 139 | a título de 140 | a veces 141 | acerca de 142 | ahora bien 143 | al cabo de 144 | al cabo del 145 | al contrario que 146 | al día 147 | al fin 148 | al lado 149 | al menos 150 | al mismo tiempo 151 | al pie de 152 | al respecto 153 | así como 154 | así que 155 | cada vez que 156 | como mínimo 157 | con detalle 158 | con ocasión de 159 | con relación a 160 | con respecto a 161 | con respecto al 162 | conforme a 163 | conforme al 164 | dado que 165 | de acuerdo con 166 | de antemano 167 | de conformidad con 168 | de forma que 169 | de hecho 170 | de manera que 171 | de modo que 172 | de momento 173 | de nuevo 174 | de ordinario 175 | de todas formas 176 | debajo de 177 | debajo del 178 | debe de 179 | debido a 180 | debido a que 181 | debido al 182 | del orden de 183 | delante de 184 | dentro de 185 | dentro del 186 | desde el momento en que 187 | después de 188 | después del 189 | en absoluto 190 | en breve 191 | en cambio 192 | en caso de que 193 | en cierta medida 194 | en común 195 | en conformidad con 196 | en consecuencia 197 | en cuanto a 198 | en cuanto al 199 | en cuestión 200 | en definitiva 201 | en detalle 202 | en efecto 203 | en especial 204 | en función de 205 | en función del 206 | en general 207 | en lo que 208 | en lugar de 209 | en lugar del 210 | en materia de 211 | en menos de 212 | en parte 213 | en particular 214 | en principio 215 | en realidad 216 | en relación con 217 | en tanto 218 | en teoría 219 | en todo caso 220 | en torno al 221 | en vez de 222 | en vez del 223 | en virtud de 224 | en virtud del 225 | en vista de 226 | encima de 227 | es decir 228 | esto es 229 | frente a 230 | fuera de 231 | fuera del 232 | gracias a 233 | ha de 234 | habrá de 235 | habrá que 236 | habrán de 237 | habría que 238 | han de 239 | hay que 240 | haya de 241 | haya que 242 | hayan de 243 | hoy día 244 | in situ 245 | junto con 246 | mientras que 247 | no más 248 | no obstante 249 | o sea 250 | por ahora 251 | por cierto 252 | por consecuencia 253 | por consiguiente 254 | por cuanto 255 | por ejemplo 256 | por el momento 257 | por encima de 258 | por encima del 259 | por ende 260 | por extensión 261 | por lo general 262 | por lo menos 263 | por lo tanto 264 | por medio de 265 | por medio del 266 | por parte de 267 | por parte del 268 | por qué 269 | por separado 270 | por supuesto 271 | por tanto 272 | por un lado 273 | puesto que 274 | respecto de 275 | respecto del 276 | si bien 277 | siempre que 278 | siempre y cuando 279 | sin distinción 280 | sin embargo 281 | sobre todo 282 | tal vez 283 | tan pronto como 284 | tanto por ciento 285 | tendrá que 286 | tendrán que 287 | tener que 288 | tenga que 289 | tengan que 290 | tiene que 291 | tienen que 292 | una vez que 293 | ya que 294 | -------------------------------------------------------------------------------- /TreeTagger/lib/swahili-abbreviations: -------------------------------------------------------------------------------- 1 | Bw. 2 | Sh. 3 | Dk. 4 | Bi. 5 | sh. 6 | W. 7 | Prof. 8 | Mhe. 9 | Na. 10 | M. 11 | n.k. 12 | S. 13 | A. 14 | P. 15 | J. 16 | E. 17 | Dr. 18 | L. 19 | tz. 20 | C. 21 | B. 22 | D. 23 | H. 24 | Mh. 25 | K. 26 | F. 27 | R. 28 | G. 29 | k.m. 30 | S.L.P. 31 | N. 32 | I. 33 | T. 34 | shs. 35 | V. 36 | Ltd. 37 | nk. 38 | O. 39 | taz. 40 | U.W.T. 41 | k. 42 | Bro. 43 | s.w. 44 | Z. 45 | B.O.M. 46 | Shs. 47 | m. 48 | Y. 49 | na. 50 | D.C. 51 | P.S. 52 | C.M.S. 53 | s.a.w. 54 | U. 55 | uk. 56 | U.M.C.A. 57 | Tsh. 58 | P.O. 59 | Inc. 60 | B.A. 61 | k.v. 62 | Vol. 63 | Ph.D. 64 | Mwl. 65 | A.D. 66 | mil. 67 | bw. 68 | St. 69 | v. 70 | s. 71 | a.s. 72 | Mw. 73 | K.K. 74 | I.L.C. 75 | Co. 76 | kumb. 77 | agh. 78 | a. 79 | Taz. 80 | P.C. 81 | Nov. 82 | M.A. 83 | C.I.A. 84 | B.K. 85 | A.H. 86 | yamb. 87 | n. 88 | ltd. 89 | l. 90 | f. 91 | e.g. 92 | cCM. 93 | c. 94 | V.T.C. 95 | T.O.H.S. 96 | T.E.C. 97 | T.C. 98 | Q.C. 99 | Q. 100 | O.C.D. 101 | Mr. 102 | Mm. 103 | K.C.M.C. 104 | I.C.O. 105 | F.W. 106 | F.F.U. 107 | Dk.B. 108 | Corp. 109 | C.C. 110 | A.A. 111 | z. 112 | y. 113 | s.w.T. 114 | op. 115 | cit. 116 | n.k.w. 117 | mf. 118 | m.m. 119 | kv. 120 | kt. 121 | km. 122 | i.e. 123 | etc. 124 | e. 125 | bt. 126 | Y.M.C.A. 127 | X. 128 | W.H. 129 | Viii. 130 | Ush. 131 | Uk. 132 | U.S. 133 | U.N. 134 | T.V. 135 | T.L.P. 136 | Sept. 137 | SH. 138 | S.s. 139 | S.S. 140 | S.L. 141 | S.J. 142 | S.D. 143 | S.A.W. 144 | Rd. 145 | Phd. 146 | P.s. 147 | P.o. 148 | P.P.F. 149 | P.O.P. 150 | P.J. 151 | N.N. 152 | N.J. 153 | N.F. 154 | Mt. 155 | Mrs. 156 | M.m. 157 | M.W. 158 | M.S. 159 | M.M. 160 | M.D. 161 | L.G. 162 | Ksh. 163 | K.m. 164 | K.k. 165 | K.S. 166 | K.O. 167 | K.N.C.U. 168 | K.A.U. 169 | K.A.R. 170 | Jr. 171 | Jan. 172 | J.V.W.A. 173 | J.S. 174 | J.F. 175 | Fr. 176 | Feb. 177 | F.M. 178 | E.n. 179 | E.M. 180 | E.C. 181 | Cap. 182 | C.W. 183 | C.S.Sp. 184 | Brig. 185 | B.J. 186 | B.C. 187 | Afr. 188 | A.M. 189 | A.G. 190 | A.C. 191 | -------------------------------------------------------------------------------- /config.props: -------------------------------------------------------------------------------- 1 | ################################ 2 | ## MAIN ## 3 | ################################ 4 | # Consideration of different timex3-types 5 | # Date 6 | considerDate = true 7 | 8 | # Duration 9 | considerDuration = true 10 | 11 | # Set 12 | considerSet = true 13 | 14 | # Time 15 | considerTime = true 16 | 17 | # Temponyms (make sure you know what you do if you set this to "true") 18 | considerTemponym = false 19 | 20 | ################################### 21 | # Path to TreeTagger home directory 22 | ################################### 23 | # Ensure there is no white space in path (try to escape white spaces) 24 | treeTaggerHome = TreeTagger 25 | # This one is only necessary if you want to process chinese documents. 26 | chineseTokenizerPath = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/treetagger/chinese-tokenizer) 27 | 28 | ################################## 29 | # paths to JVnTextPro model paths: 30 | ################################## 31 | sent_model_path = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/jvntextpro/models/jvnsensegmenter) 32 | word_model_path = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/jvntextpro/models/jvnsegmenter) 33 | pos_model_path = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/jvntextpro/models/jvnpostag/maxent) 34 | 35 | ##################################################### 36 | # paths to Stanford POS Tagger model or config files: 37 | ##################################################### 38 | model_path = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/stanford-postagger-full-2014-01-04/models/arabic.tagger) 39 | # leave this unset if you do not need one (e.g., /home/jannik/stanford-postagger-full-2014-01-04/tagger.config) 40 | config_path = 41 | 42 | ######################################## 43 | ## paths to hunpos and its tagger files: 44 | ######################################## 45 | hunpos_path = SET ME IN CONFIG.PROPS! (e.g., /home/jannik/hunpos) 46 | hunpos_model_name = SET ME IN CONFIG.PROPS! (e.g., model.hunpos.mte5.defnpout) 47 | 48 | 49 | 50 | # DO NOT CHANGE THE FOLLOWING 51 | ################################ 52 | # Relative path of type system in HeidelTime home directory 53 | typeSystemHome = desc/type/HeidelTime_TypeSystem.xml 54 | 55 | # Relative path of dkpro type system in HeidelTime home directory 56 | typeSystemHome_DKPro = desc/type/DKPro_TypeSystem.xml 57 | 58 | # Name of uima-context variables... 59 | # ...for date-consideration 60 | uimaVarDate = Date 61 | 62 | # ...for duration-consideration 63 | uimaVarDuration = Duration 64 | 65 | # ...for language 66 | uimaVarLanguage = Language 67 | 68 | # ...for set-consideration 69 | uimaVarSet = Set 70 | 71 | # ...for time-consideration 72 | uimaVarTime = Time 73 | 74 | # ...for temponym-consideration 75 | uimaVarTemponym = Temponym 76 | 77 | # ...for type to process 78 | uimaVarTypeToProcess = Type 79 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | JPype1 --------------------------------------------------------------------------------