├── .gitignore
├── .idea
    └── scopes
    │   └── scope_settings.xml
├── LICENSE.md
├── README.txt
├── WikParser.php
├── composer.json
├── language.config.php
├── lib
    ├── DefParse.php
    ├── GenderParse.php
    ├── HyperParse.php
    ├── Lang
    │   ├── AbstractLang.php
    │   ├── De.php
    │   ├── En.php
    │   ├── Es.php
    │   └── Fr.php
    ├── PosParse.php
    ├── StripTags.php
    ├── SynParse.php
    ├── WikiExtract.php
    └── conc.php
└── tests
    └── test.php


/.gitignore:
--------------------------------------------------------------------------------
1 | /vendor
2 | /composer.lock
3 | 


--------------------------------------------------------------------------------
/.idea/scopes/scope_settings.xml:
--------------------------------------------------------------------------------
1 | <component name="DependencyValidationManager">
2 |   <state>
3 |     <option name="SKIP_IMPORT_STATEMENTS" value="false" />
4 |   </state>
5 | </component>


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
  1 | GNU GENERAL PUBLIC LICENSE
  2 |                        Version 2, June 1991
  3 | 
  4 |  Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
  5 |  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  6 |  Everyone is permitted to copy and distribute verbatim copies
  7 |  of this license document, but changing it is not allowed.
  8 | 
  9 |                             Preamble
 10 | 
 11 |   The licenses for most software are designed to take away your
 12 | freedom to share and change it.  By contrast, the GNU General Public
 13 | License is intended to guarantee your freedom to share and change free
 14 | software--to make sure the software is free for all its users.  This
 15 | General Public License applies to most of the Free Software
 16 | Foundation's software and to any other program whose authors commit to
 17 | using it.  (Some other Free Software Foundation software is covered by
 18 | the GNU Lesser General Public License instead.)  You can apply it to
 19 | your programs, too.
 20 | 
 21 |   When we speak of free software, we are referring to freedom, not
 22 | price.  Our General Public Licenses are designed to make sure that you
 23 | have the freedom to distribute copies of free software (and charge for
 24 | this service if you wish), that you receive source code or can get it
 25 | if you want it, that you can change the software or use pieces of it
 26 | in new free programs; and that you know you can do these things.
 27 | 
 28 |   To protect your rights, we need to make restrictions that forbid
 29 | anyone to deny you these rights or to ask you to surrender the rights.
 30 | These restrictions translate to certain responsibilities for you if you
 31 | distribute copies of the software, or if you modify it.
 32 | 
 33 |   For example, if you distribute copies of such a program, whether
 34 | gratis or for a fee, you must give the recipients all the rights that
 35 | you have.  You must make sure that they, too, receive or can get the
 36 | source code.  And you must show them these terms so they know their
 37 | rights.
 38 | 
 39 |   We protect your rights with two steps: (1) copyright the software, and
 40 | (2) offer you this license which gives you legal permission to copy,
 41 | distribute and/or modify the software.
 42 | 
 43 |   Also, for each author's protection and ours, we want to make certain
 44 | that everyone understands that there is no warranty for this free
 45 | software.  If the software is modified by someone else and passed on, we
 46 | want its recipients to know that what they have is not the original, so
 47 | that any problems introduced by others will not reflect on the original
 48 | authors' reputations.
 49 | 
 50 |   Finally, any free program is threatened constantly by software
 51 | patents.  We wish to avoid the danger that redistributors of a free
 52 | program will individually obtain patent licenses, in effect making the
 53 | program proprietary.  To prevent this, we have made it clear that any
 54 | patent must be licensed for everyone's free use or not licensed at all.
 55 | 
 56 |   The precise terms and conditions for copying, distribution and
 57 | modification follow.
 58 | 
 59 |                     GNU GENERAL PUBLIC LICENSE
 60 |    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 61 | 
 62 |   0. This License applies to any program or other work which contains
 63 | a notice placed by the copyright holder saying it may be distributed
 64 | under the terms of this General Public License.  The "Program", below,
 65 | refers to any such program or work, and a "work based on the Program"
 66 | means either the Program or any derivative work under copyright law:
 67 | that is to say, a work containing the Program or a portion of it,
 68 | either verbatim or with modifications and/or translated into another
 69 | language.  (Hereinafter, translation is included without limitation in
 70 | the term "modification".)  Each licensee is addressed as "you".
 71 | 
 72 | Activities other than copying, distribution and modification are not
 73 | covered by this License; they are outside its scope.  The act of
 74 | running the Program is not restricted, and the output from the Program
 75 | is covered only if its contents constitute a work based on the
 76 | Program (independent of having been made by running the Program).
 77 | Whether that is true depends on what the Program does.
 78 | 
 79 |   1. You may copy and distribute verbatim copies of the Program's
 80 | source code as you receive it, in any medium, provided that you
 81 | conspicuously and appropriately publish on each copy an appropriate
 82 | copyright notice and disclaimer of warranty; keep intact all the
 83 | notices that refer to this License and to the absence of any warranty;
 84 | and give any other recipients of the Program a copy of this License
 85 | along with the Program.
 86 | 
 87 | You may charge a fee for the physical act of transferring a copy, and
 88 | you may at your option offer warranty protection in exchange for a fee.
 89 | 
 90 |   2. You may modify your copy or copies of the Program or any portion
 91 | of it, thus forming a work based on the Program, and copy and
 92 | distribute such modifications or work under the terms of Section 1
 93 | above, provided that you also meet all of these conditions:
 94 | 
 95 |     a) You must cause the modified files to carry prominent notices
 96 |     stating that you changed the files and the date of any change.
 97 | 
 98 |     b) You must cause any work that you distribute or publish, that in
 99 |     whole or in part contains or is derived from the Program or any
100 |     part thereof, to be licensed as a whole at no charge to all third
101 |     parties under the terms of this License.
102 | 
103 |     c) If the modified program normally reads commands interactively
104 |     when run, you must cause it, when started running for such
105 |     interactive use in the most ordinary way, to print or display an
106 |     announcement including an appropriate copyright notice and a
107 |     notice that there is no warranty (or else, saying that you provide
108 |     a warranty) and that users may redistribute the program under
109 |     these conditions, and telling the user how to view a copy of this
110 |     License.  (Exception: if the Program itself is interactive but
111 |     does not normally print such an announcement, your work based on
112 |     the Program is not required to print an announcement.)
113 | 
114 | These requirements apply to the modified work as a whole.  If
115 | identifiable sections of that work are not derived from the Program,
116 | and can be reasonably considered independent and separate works in
117 | themselves, then this License, and its terms, do not apply to those
118 | sections when you distribute them as separate works.  But when you
119 | distribute the same sections as part of a whole which is a work based
120 | on the Program, the distribution of the whole must be on the terms of
121 | this License, whose permissions for other licensees extend to the
122 | entire whole, and thus to each and every part regardless of who wrote it.
123 | 
124 | Thus, it is not the intent of this section to claim rights or contest
125 | your rights to work written entirely by you; rather, the intent is to
126 | exercise the right to control the distribution of derivative or
127 | collective works based on the Program.
128 | 
129 | In addition, mere aggregation of another work not based on the Program
130 | with the Program (or with a work based on the Program) on a volume of
131 | a storage or distribution medium does not bring the other work under
132 | the scope of this License.
133 | 
134 |   3. You may copy and distribute the Program (or a work based on it,
135 | under Section 2) in object code or executable form under the terms of
136 | Sections 1 and 2 above provided that you also do one of the following:
137 | 
138 |     a) Accompany it with the complete corresponding machine-readable
139 |     source code, which must be distributed under the terms of Sections
140 |     1 and 2 above on a medium customarily used for software interchange; or,
141 | 
142 |     b) Accompany it with a written offer, valid for at least three
143 |     years, to give any third party, for a charge no more than your
144 |     cost of physically performing source distribution, a complete
145 |     machine-readable copy of the corresponding source code, to be
146 |     distributed under the terms of Sections 1 and 2 above on a medium
147 |     customarily used for software interchange; or,
148 | 
149 |     c) Accompany it with the information you received as to the offer
150 |     to distribute corresponding source code.  (This alternative is
151 |     allowed only for noncommercial distribution and only if you
152 |     received the program in object code or executable form with such
153 |     an offer, in accord with Subsection b above.)
154 | 
155 | The source code for a work means the preferred form of the work for
156 | making modifications to it.  For an executable work, complete source
157 | code means all the source code for all modules it contains, plus any
158 | associated interface definition files, plus the scripts used to
159 | control compilation and installation of the executable.  However, as a
160 | special exception, the source code distributed need not include
161 | anything that is normally distributed (in either source or binary
162 | form) with the major components (compiler, kernel, and so on) of the
163 | operating system on which the executable runs, unless that component
164 | itself accompanies the executable.
165 | 
166 | If distribution of executable or object code is made by offering
167 | access to copy from a designated place, then offering equivalent
168 | access to copy the source code from the same place counts as
169 | distribution of the source code, even though third parties are not
170 | compelled to copy the source along with the object code.
171 | 
172 |   4. You may not copy, modify, sublicense, or distribute the Program
173 | except as expressly provided under this License.  Any attempt
174 | otherwise to copy, modify, sublicense or distribute the Program is
175 | void, and will automatically terminate your rights under this License.
176 | However, parties who have received copies, or rights, from you under
177 | this License will not have their licenses terminated so long as such
178 | parties remain in full compliance.
179 | 
180 |   5. You are not required to accept this License, since you have not
181 | signed it.  However, nothing else grants you permission to modify or
182 | distribute the Program or its derivative works.  These actions are
183 | prohibited by law if you do not accept this License.  Therefore, by
184 | modifying or distributing the Program (or any work based on the
185 | Program), you indicate your acceptance of this License to do so, and
186 | all its terms and conditions for copying, distributing or modifying
187 | the Program or works based on it.
188 | 
189 |   6. Each time you redistribute the Program (or any work based on the
190 | Program), the recipient automatically receives a license from the
191 | original licensor to copy, distribute or modify the Program subject to
192 | these terms and conditions.  You may not impose any further
193 | restrictions on the recipients' exercise of the rights granted herein.
194 | You are not responsible for enforcing compliance by third parties to
195 | this License.
196 | 
197 |   7. If, as a consequence of a court judgment or allegation of patent
198 | infringement or for any other reason (not limited to patent issues),
199 | conditions are imposed on you (whether by court order, agreement or
200 | otherwise) that contradict the conditions of this License, they do not
201 | excuse you from the conditions of this License.  If you cannot
202 | distribute so as to satisfy simultaneously your obligations under this
203 | License and any other pertinent obligations, then as a consequence you
204 | may not distribute the Program at all.  For example, if a patent
205 | license would not permit royalty-free redistribution of the Program by
206 | all those who receive copies directly or indirectly through you, then
207 | the only way you could satisfy both it and this License would be to
208 | refrain entirely from distribution of the Program.
209 | 
210 | If any portion of this section is held invalid or unenforceable under
211 | any particular circumstance, the balance of the section is intended to
212 | apply and the section as a whole is intended to apply in other
213 | circumstances.
214 | 
215 | It is not the purpose of this section to induce you to infringe any
216 | patents or other property right claims or to contest validity of any
217 | such claims; this section has the sole purpose of protecting the
218 | integrity of the free software distribution system, which is
219 | implemented by public license practices.  Many people have made
220 | generous contributions to the wide range of software distributed
221 | through that system in reliance on consistent application of that
222 | system; it is up to the author/donor to decide if he or she is willing
223 | to distribute software through any other system and a licensee cannot
224 | impose that choice.
225 | 
226 | This section is intended to make thoroughly clear what is believed to
227 | be a consequence of the rest of this License.
228 | 
229 |   8. If the distribution and/or use of the Program is restricted in
230 | certain countries either by patents or by copyrighted interfaces, the
231 | original copyright holder who places the Program under this License
232 | may add an explicit geographical distribution limitation excluding
233 | those countries, so that distribution is permitted only in or among
234 | countries not thus excluded.  In such case, this License incorporates
235 | the limitation as if written in the body of this License.
236 | 
237 |   9. The Free Software Foundation may publish revised and/or new versions
238 | of the General Public License from time to time.  Such new versions will
239 | be similar in spirit to the present version, but may differ in detail to
240 | address new problems or concerns.
241 | 
242 | Each version is given a distinguishing version number.  If the Program
243 | specifies a version number of this License which applies to it and "any
244 | later version", you have the option of following the terms and conditions
245 | either of that version or of any later version published by the Free
246 | Software Foundation.  If the Program does not specify a version number of
247 | this License, you may choose any version ever published by the Free Software
248 | Foundation.
249 | 
250 |   10. If you wish to incorporate parts of the Program into other free
251 | programs whose distribution conditions are different, write to the author
252 | to ask for permission.  For software which is copyrighted by the Free
253 | Software Foundation, write to the Free Software Foundation; we sometimes
254 | make exceptions for this.  Our decision will be guided by the two goals
255 | of preserving the free status of all derivatives of our free software and
256 | of promoting the sharing and reuse of software generally.
257 | 
258 |                             NO WARRANTY
259 | 
260 |   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
268 | REPAIR OR CORRECTION.
269 | 
270 |   12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278 | POSSIBILITY OF SUCH DAMAGES.
279 | 
280 |                      END OF TERMS AND CONDITIONS
281 | 
282 |             How to Apply These Terms to Your New Programs
283 | 
284 |   If you develop a new program, and you want it to be of the greatest
285 | possible use to the public, the best way to achieve this is to make it
286 | free software which everyone can redistribute and change under these terms.
287 | 
288 |   To do so, attach the following notices to the program.  It is safest
289 | to attach them to the start of each source file to most effectively
290 | convey the exclusion of warranty; and each file should have at least
291 | the "copyright" line and a pointer to where the full notice is found.
292 | 
293 |     {description}
294 |     Copyright (C) {year}  {fullname}
295 | 
296 |     This program is free software; you can redistribute it and/or modify
297 |     it under the terms of the GNU General Public License as published by
298 |     the Free Software Foundation; either version 2 of the License, or
299 |     (at your option) any later version.
300 | 
301 |     This program is distributed in the hope that it will be useful,
302 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
303 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
304 |     GNU General Public License for more details.
305 | 
306 |     You should have received a copy of the GNU General Public License along
307 |     with this program; if not, write to the Free Software Foundation, Inc.,
308 |     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
309 | 
310 | Also add information on how to contact you by electronic and paper mail.
311 | 
312 | If the program is interactive, make it output a short notice like this
313 | when it starts in an interactive mode:
314 | 
315 |     Gnomovision version 69, Copyright (C) year name of author
316 |     Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
317 |     This is free software, and you are welcome to redistribute it
318 |     under certain conditions; type `show c' for details.
319 | 
320 | The hypothetical commands `show w' and `show c' should show the appropriate
321 | parts of the General Public License.  Of course, the commands you use may
322 | be called something other than `show w' and `show c'; they could even be
323 | mouse-clicks or menu items--whatever suits your program.
324 | 
325 | You should also get your employer (if you work as a programmer) or your
326 | school, if any, to sign a "copyright disclaimer" for the program, if
327 | necessary.  Here is a sample; alter the names:
328 | 
329 |   Yoyodyne, Inc., hereby disclaims all copyright interest in the program
330 |   `Gnomovision' (which makes passes at compilers) written by James Hacker.
331 | 
332 |   {signature of Ty Coon}, 1 April 1989
333 |   Ty Coon, President of Vice
334 | 
335 | This General Public License does not permit incorporating your program into
336 | proprietary programs.  If your program is a subroutine library, you may
337 | consider it more useful to permit linking proprietary applications with the
338 | library.  If this is what you want to do, use the GNU Lesser General
339 | Public License instead of this License.
340 | 


--------------------------------------------------------------------------------
/README.txt:
--------------------------------------------------------------------------------
  1 | /****************************************************************/
  2 | // NOTE: These instructions are no longer valid for this version
  3 | // of the Wikparser. Please refer to the Wikparser-0.3a repo
  4 | // for use with these instructions.
  5 | /****************************************************************/
  6 | 
  7 | Wikparser 0.3a
  8 | =============
  9 | 
 10 | Wiktionary Text Parser
 11 | Author: Yves Bourque
 12 | http://www.igrec.ca/projects/wiktionary-text-parser/
 13 | 
 14 | CHANGELOG 0.3a
 15 | ==============
 16 | - Language parameters are now set via an array ($langParameters)
 17 | - Parser classes no longer set all language parameters
 18 | 
 19 | CHANGELOG 0.3
 20 | =============
 21 | - Added gender parse functionality
 22 | - Added support for Spanish (partial)
 23 | - Added support for German (partial)
 24 | - Changed to mysqli
 25 | - Simplified mysql data retrieval
 26 | - Reduced number of variables
 27 | - Added additional error messages
 28 | 
 29 | 
 30 | DESCRIPTION
 31 | ===========
 32 | The Wiktionary Parser (or Wikparser) is a small tool written in PHP that allows users to extract specific information from the Wiktionary API or a local copy of Wiktionary's database in MySQL.
 33 | 
 34 | Currently, this software is able to extract the following information:
 35 | - Part of speech/Lexical categories (pos)
 36 | - Synonyms (syn)
 37 | - Hypernyms (hyper)
 38 | - Gender (gender)
 39 | - Definitions (def)
 40 | 
 41 | for the following languages:
 42 | - English
 43 | - French
 44 | - Spanish (partial)
 45 | - German (partial)
 46 | 
 47 | Additional language support may be added by following the guide at http://www.igrec.ca/projects/wiktionary-text-parser/ or at the end of this file.
 48 | 
 49 | REQUIREMENTS
 50 | ============
 51 | You will need:
 52 | - Apache or some other web server platform
 53 | - PHP 5
 54 | - cURL
 55 | 
 56 | INSTALLATION
 57 | ============
 58 | Simply download and copy files and folders to a location accessible from your Web server.
 59 | 
 60 | USAGE
 61 | =====
 62 | To use Wikparser, you need to call or point your browser to the wikparser.php file with the following parameters and values (* indicates a mandatory parameter):
 63 | 
 64 | - *word: any string (e.g. /wikparser.php?word=dog)
 65 | - *query: the type of query; "def" for definitions, "syn" for synonyms, "pos" for parts of speech, "hyper" for hypernyms, and "gender" for gender (e.g. /wikparser.php?query=pos)
 66 | - lang: Wiktionary language code. Script currently supports English ("en"), french ("fr"), Spanish ("es"), and German ("de") natively [default: en].
 67 | - count: number of items to return [default: 100]
 68 | - source: location of Wiktionary data; "local" for a local MySql copy of Wiktionary; "api" for Wiktionary’s API [default: api]
 69 | 
 70 | EXAMPLES
 71 | ========
 72 | The examples below use the Wikparser hosted at www.igrec.ca.
 73 | 
 74 | - Get first 2 definitions of the word "table" in English directly from Wiktionary:
 75 | http://www.igrec.ca/project-files/wikparser/wikparser.php?word=table&query=def&count=2
 76 | 
 77 | - Get all parts of speech for the word "puissance" in French directly from Wiktionary:
 78 | http://www.igrec.ca/project-files/wikparser/wikparser.php?word=puissance&query=pos&lang=fr
 79 | 
 80 | ADDING SUPPORT FOR OTHER LANGUAGES
 81 | ==================================
 82 | In order to add support for other languages, you must first determine the language code used by Wiktionary. It's usually the standard two letter code, but you can always check by going to wiktionary.org and selecting the language you're interested in. Then look at the first few letters of the URL:
 83 | 
 84 | http://tr.wiktionary.org/ : "tr" for Turkish
 85 | http://vi.wiktionary.org/ : "vi" for Vietnamese
 86 | 
 87 | Now open the language.config.php file in the root of the Wiktionary Parser. You'll see a PHP switch. You must add a new case (or modify one of the ones included if you don't care about keeping current language functionality) for the language you want to work with. You'll see the following:
 88 | 
 89 | case "INSERT LANGUAGE CODE HERE":
 90 | 	$langParameters = array(
 91 | 		"langCode" => "",
 92 | 		"langHeader" => "",
 93 | 		"langSeparator" => "",
 94 | 		"defHeader" => "",
 95 | 		"defTag" => "",
 96 | 		"synHeader" => "",
 97 | 		"hyperHeader" => "",
 98 | 		"genderPattern" => "",
 99 | 		"posMatchType" => "",
100 | 		"posPattern" => "",
101 | 		"posArray" => "",
102 | 		"posExtraString" => "",
103 | 	);
104 | 	break;
105 | 
106 | For instance, if you're working with Turkish, you would insert tr between the case quotes. As for the rest, you'll need to actually have a look at the output generated by the Wiktionary API (the output is also identical for a local copy of the database). You'll need to call the API with a word and look at the output to identify each one of the parameters above. Follow the link below for an example of the output for the word abuelo via the Wiktionary API using the spanish language code (es):
107 | 
108 | http://es.wiktionary.org/w/api.php?action=parse&prop=wikitext&page=abuelo&format=xmlfm
109 | 
110 | You'll need to scan multiple words to determine what patterns to use for whatever language you're interested in. It can be tricky, as the raw data is messy and inconsistent. You'll often find identifiers that differ from one entry to the next. Once you've figured out how Wiktionary encodes its data for that language, you can begin to fill in the parameters. Not all parameters need to be set for the parser to work; if you're only interested in extracting synonyms, then only synHeader requires a value. One by one:
111 | 
112 | langCode:
113 | The string that identifies the language within Wiktionary (e.g. en, de, tr, etc.)
114 | 
115 | langheader:
116 | The string that identifies the section for whatever language you're working with. Wiktionary will often list multiple languages on a page for a given word (table, for instance, is both valid in English and French). It's important to identify the string that starts a language section so that info from another language isn't parsed. Ex. ==English==.
117 | 
118 | langSeparator:
119 | The string that separates each language on a given page. Sometimes it's a simple string (e.g. "----" in the English Wiktionary), but in other cases you might have to use a partial string. For example, the French Wiktionary wraps languages within "== {{=fr=}} ==", so we can assume that each new language section will begin with "== {{=". This is therefore the langSeparator for French entries.
120 | 
121 | defHeader:
122 | The string that begins the definitions section. Not always present (e.g. English). In German, all definitions fall under the {{Bedeutungen}} string.
123 | 
124 | defTag:
125 | Definitions are usually preceded by some non-alphanumeric character (e.g. in English by "# " (notice the space)). This differs between languages, however.
126 | 
127 | synHeader:
128 | String that identifies the synonyms section (e.g. English: ====Synonyms====).
129 | 
130 | hyperHeader:
131 | String that identifies the hypernyms section (e.g. English: ====Hypernyms====).
132 | 
133 | genderPattern:
134 | A regular expression that captures a words gender. Patterns used are often inconsistent, so you'll need to go through a few pages to make sure you've identified all possible strings.
135 | 
136 | posMatchType:
137 | Either "array" or "preg". This is how the parts of speech will be identified. If, like for English, there is a limited number of possibilities, you can simply store them in an array and set this variable to "array". If the parts of speech vary greatly (like they do for French), then you'll want to use a regular expression and set this variable to "preg".
138 | 
139 | posPattern:
140 | If the parts of speech vary greatly, you'll need to write a regular expression in order to identify them. If you're unfamiliar with regular expressions, have a look at this quick guide, which also has a link to a tutorial.
141 | 
142 | posArray:
143 | If the parts of speech do not vary and are limited in number, you can store them all in this array and set the posmatchtype variable to "array."
144 | 
145 | posExtraString:
146 | When using regular expressions to match POS, you often need to add unrelated strings in order to capture the correct entry (e.g. in German, POS is preceded by {{Wortart|). Add this string here to have the parser strip at output.
147 | 
148 | Once these parameters are set, you should be able to call the script with the new language code set to the lang parameter.
149 | 


--------------------------------------------------------------------------------
/WikParser.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace ybourque\Wikparser;
  3 | use ybourque\Wikparser\lib\DefParse;
  4 | use ybourque\Wikparser\lib\GenderParse;
  5 | use ybourque\Wikparser\lib\HyperParse;
  6 | use ybourque\Wikparser\lib\PosParse;
  7 | use ybourque\Wikparser\lib\SynParse;
  8 | use ybourque\Wikparser\lib\WikiExtract;
  9 | 
 10 | class WikParser{
 11 |     public $parsedDefinition='';
 12 | 
 13 |     /***********************************************************************************/
 14 |     /***********************************************************************************/
 15 |     // Language code for search, default english (en)
 16 |     /***********************************************************************************/
 17 |     // Number of results; default '100'
 18 |     /***********************************************************************************/
 19 |     // Set wikisource to local if not set. Values either 'local' or 'api'.
 20 |     /***********************************************************************************/
 21 |     /***********************************************************************************/
 22 |     public function getWordDefiniton($word,$query='def',$langCode='en',$count=100,$source='api'){
 23 |         $this->word = $word;
 24 |         $this->count = $count;
 25 |         $this->source = $source;
 26 |         $this->langParameters = $this->newLang($langCode);
 27 | 
 28 |         $wikitext = $this->getWikiText($this->langParameters, $this->source, $this->word);
 29 |         $parsed = [];
 30 |         foreach ((array) $query as $q) {
 31 |             $parsed[$q] = $this->parseQuery($q, $wikitext);
 32 |         }
 33 |         return $parsed;
 34 |     }
 35 | 
 36 |     private function newLang($langCode)
 37 |     {
 38 |         $class = 'ybourque\Wikparser\lib\Lang\\' . ucfirst(strtolower($langCode));
 39 |         return new $class();
 40 |     }
 41 | 
 42 |     private function parseQuery($query, $wikitext){
 43 |         $parsedDefinition = [];
 44 |         switch ($query) {
 45 |             /***********************************************************************************/
 46 |             // Include defparse class and create new object with 3 variables.
 47 |             /***********************************************************************************/
 48 |             case "def":
 49 |                 $DefParse = new DefParse($this->langParameters);
 50 |                 $parsedDefinition = $DefParse->getDef($wikitext, $this->count);
 51 |                 break;
 52 |             /***********************************************************************************/
 53 |             // Include posparse class and create new object with 3 variables.
 54 |             /***********************************************************************************/
 55 |             case "pos":
 56 |                 $posparse = new PosParse($this->langParameters);
 57 |                 $parsedDefinition = $posparse->getPos($wikitext, $this->count);
 58 |                 break;
 59 |             /***********************************************************************************/
 60 |             // Include synparse class and create new object with 3 variables.
 61 |             /***********************************************************************************/
 62 |             case "syn":
 63 |                 $SynParse = new SynParse($this->langParameters);
 64 |                 $parsedDefinition = $SynParse->getSyn($wikitext, $this->count);
 65 |                 break;
 66 |             /***********************************************************************************/
 67 |             // Include hyperparse class and create new object with 3 variables. (Hypernyms)
 68 |             /***********************************************************************************/
 69 |             case "hyper":
 70 |                 $HyperParse = new HyperParse($this->langParameters);
 71 |                 $parsedDefinition = $HyperParse->getHyper($wikitext, $this->count);
 72 |                 break;
 73 |             /***********************************************************************************/
 74 |             // Include genderparse class and create new object with 3 variables. (Gender)
 75 |             /***********************************************************************************/
 76 |             case "gender":
 77 |                 $GenderParse = new GenderParse($this->langParameters);
 78 |                 $parsedDefinition = $GenderParse->getGender($wikitext, $this->count);
 79 |                 break;
 80 |             /***********************************************************************************/
 81 |             default:
 82 |                 echo "You must specify a valid query type ('pos', 'def', 'syn', 'hyper', or 'gender').";
 83 |                 break;
 84 |         }
 85 | 
 86 |         return $parsedDefinition;
 87 |     }
 88 | 
 89 |     /***********************************************************************************/
 90 |     // Include wikiextract class and create new object with 2 variables. Returns the
 91 |     // contents of the wiktionary entry for a given word.
 92 |     /***********************************************************************************/
 93 |     public function getWikiText($langParameters, $wikiSource, $word) {
 94 |         $WikiExtract = new WikiExtract($langParameters, $wikiSource);
 95 |         return $WikiExtract->getWikiText($word);
 96 |     }
 97 | 
 98 |     /***********************************************************************************/
 99 |     // Prints contents of array.
100 |     /***********************************************************************************/
101 |     public function printResults($array) {
102 |         $resultseparator = " | ";
103 |         $printresults = null;
104 |         foreach ($array as $value) {
105 |             $printresults .= $value.$resultseparator;
106 |         }
107 |         echo rtrim($printresults, $resultseparator);
108 |     }
109 | }
110 | 


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "ybourque/wikparser",
 3 |     "description": "Wiktionary Text Parser",
 4 |     "keywords": ["wiktionary", "parser"],
 5 |     "homepage": "https://github.com/ybourque/Wikparser",
 6 |     "license": "MIT",
 7 |     "authors": [
 8 |         {
 9 |             "name": "Yves Bourque ",
10 |             "email": "ybourque@gmail.com",
11 |             "homepage" :"https://github.com/ybourque"
12 |         }
13 |     ],
14 |     "require": {
15 |         "php": ">=5.3.0"
16 |     },
17 |     "autoload": {
18 |         "psr-4": {"ybourque\\Wikparser\\": ""}
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/language.config.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | /***********************************************************************************/
  3 | // Add a case for any additional languages and set variables according to the strings
  4 | // used by Wiktionary for the given language. See www.igrec.ca/projects/ for more
  5 | // info on how to do this.
  6 | // Basics:
  7 | /*case "2-letter language code":
  8 | 		langCode = 2-letter language code
  9 | 		langHeader = string that begins section for a given language
 10 | 		angSeparator = string that indicates a new language section
 11 | 		defHeader = header for definitions section; may not be present
 12 | 		defTag = string that begins a definition
 13 | 		synHeader = header for synonyms
 14 | 		hyperHeader = header for hypernyms
 15 | 		genderPattern = regex for gender
 16 | 		posExtraString = extra string around POS that can be stripped
 17 | 		posPattern = regex pattern for POS
 18 | 		posArray = array(all possible values for POS); leave empty if using preg
 19 | 		posMatchType = 'preg' for regular expressions; 'array' for array */
 20 | /***********************************************************************************/
 21 | 
 22 | switch ($langCode) {
 23 | // English parameters	
 24 | 	case "en":
 25 | 		$langParameters = array(
 26 | 			"langCode" => "en",
 27 | 			"langHeader" => "==English==",
 28 | 			"langSeparator" => "----",
 29 | 			"defHeader" => "",
 30 | 			"defTag" => "# ",
 31 | 			"synHeader" => "====Synonyms====",
 32 | 			"hyperHeader" => "====Hypernyms====",
 33 | 			"genderPattern" => "",
 34 | 			"posMatchType" => "array",
 35 | 			"posPattern" => "",
 36 | 			"posArray" => array(
 37 | 			'===Noun===', '===Verb===', '===Adjective===', '===Adverb===', '===Preposition===',
 38 | 			'===Particle===', '===Pronouns===', '===Interjection===', '===Conjunction===',
 39 | 			'===Article==='),
 40 | 			"posExtraString" => "=",
 41 | 		);
 42 | 		break;	
 43 | // French parameters
 44 | 	case "fr":
 45 | 		$langParameters = array(
 46 | 			"langCode" => "fr",
 47 | 			"langHeader" => "fr}} ==",
 48 | 			"langSeparator" => "== {{=",
 49 | 			"defHeader" => "",
 50 | 			"defTag" => "# ",
 51 | 			"synHeader" => "==== {{S|synonymes}} ====",
 52 | 			"hyperHeader" => "==== {{S|hyperonymes}} ====",
 53 | 			"genderPattern" => "(\{\{([mf]|mf)\??\}\})",
 54 | 			"posMatchType" => "preg",
 55 | 			"posPattern" => "(\{\{\S\|[\d\w\s]+\|fr(\|num=[0-9])?\}\})u",
 56 | 			"posArray" => array(),
 57 | 			"posExtraString" => "{{S|",
 58 | 		);
 59 | 		break;	
 60 | // Spanish parameters
 61 | 	case "es":
 62 | 			$langParameters = array(
 63 | 			"langCode" => "es",
 64 | 			"langHeader" => "{{ES",
 65 | 			"langSeparator" => "",
 66 | 			"defHeader" => "",
 67 | 			"defTag" => ";",
 68 | 			"synHeader" => "'''Sinónimo",
 69 | 			"hyperHeader" => "",
 70 | 			"genderPattern" => "(\s?(masculino|femenino)(\|es)?\}\}\s?===)",
 71 | 			"posMatchType" => "preg",
 72 | 			"posPattern" => "(===\s?\{\{\w*[\|\s])u",
 73 | 			"posArray" => array(),
 74 | 			"posExtraString" => "",
 75 | 		);
 76 | 		break;	
 77 | // German parameters
 78 | 	case "de":
 79 | 		$langParameters = array(
 80 | 			"langCode" => "de",
 81 | 			"langHeader" => "Deutsch}}) ==",
 82 | 			"langSeparator" => "({{Sprache|",
 83 | 			"defHeader" => "{{Bedeutungen}}",
 84 | 			"defTag" => ":",
 85 | 			"synHeader" => "{{Synonyme}}",
 86 | 			"hyperHeader" => "{{Oberbegriffe}}",
 87 | 			"genderPattern" => "(\{\{[mfn]\}\}\s===)",
 88 | 			"posMatchType" => "preg",
 89 | 			"posPattern" => "(\{\{Wortart\|\w+\|)",
 90 | 			"posArray" => array(),
 91 | 			"posExtraString" => "{{Wortart|",
 92 | 		);
 93 | 		break;
 94 | // Fill in the following settings for a language of your choice.
 95 | 	case "":
 96 | 		$langParameters = array(
 97 | 			"langCode" => "",		// string
 98 | 			"langHeader" => "",		// string
 99 | 			"langSeparator" => "",	// string
100 | 			"defHeader" => "",		// string
101 | 			"defTag" => "",			// string
102 | 			"synHeader" => "",		// string
103 | 			"hyperHeader" => "",	// string
104 | 			"genderPattern" => "",	// regex
105 | 			"posMatchType" => "",	// 'preg' or 'array'
106 | 			"posPattern" => "",		// regex
107 | 			"posArray" => "",		// array
108 | 			"posExtraString" => "",	// string
109 | 		);
110 | 		break;		
111 | // Default parameters (currently english)		
112 | 	default:
113 | 		$langParameters = array(
114 | 			"langCode" => "en",
115 | 			"langHeader" => "==English==",
116 | 			"langSeparator" => "----",
117 | 			"defHeader" => "",
118 | 			"defTag" => "# ",
119 | 			"synHeader" => "====Synonyms====",
120 | 			"hyperHeader" => "====Hypernyms====",
121 | 			"genderPattern" => "",
122 | 			"posMatchType" => "array",
123 | 			"posPattern" => "",
124 | 			"posArray" => array(
125 | 			'===Noun===', '===Verb===', '===Adjective===', '===Adverb===', '===Preposition===',
126 | 			'===Particle===', '===Pronouns===', '===Interjection===', '===Conjunction===',
127 | 			'===Article==='),
128 | 			"posExtraString" => "=",
129 | 		);
130 | 		break;
131 | }
132 | ?>


--------------------------------------------------------------------------------
/lib/DefParse.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace ybourque\Wikparser\lib;
  3 | 
  4 | use ybourque\Wikparser\lib\Lang\AbstractLang;
  5 | 
  6 | /***********************************************************************************/
  7 | // This class is used to extract all definitions for a word.
  8 | // See the language.config.php file for setting language specific parameters.
  9 | /***********************************************************************************/
 10 | 
 11 | class DefParse {
 12 | /***********************************************************************************/
 13 | // Variables
 14 | /***********************************************************************************/
 15 | 	private $sectionPattern;
 16 | 	private $defTag; // definitions tag, set in config file
 17 | 
 18 | /***********************************************************************************/
 19 | // construct
 20 | /***********************************************************************************/
 21 | 	public function __construct(AbstractLang $langParameters) {
 22 | 
 23 | 		// Find all matches for header + text until double newline.
 24 | 		if ($langParameters['defHeader']) {
 25 | 			// use an explicit definition header string
 26 | 			$this->sectionPattern = '/('.preg_quote($langParameters['defHeader']).'.*?\n\n)/su';
 27 | 		} elseif ($langParameters['posPattern']) {
 28 | 			// some languages, such as french, have definitions after the part-of-speech
 29 | 			$this->sectionPattern = "/(" . $langParameters['posPattern'] . '.*?\n\n)/su';
 30 | 		}
 31 | 
 32 | 		$this->defTag = $langParameters['defTag'];
 33 | 		$this->tagPattern = "/\n".str_replace(" ", "\s", preg_quote($this->defTag)).".*/u";
 34 | 	}
 35 | 
 36 | /***********************************************************************************/
 37 | // public methods
 38 | /***********************************************************************************/
 39 | 	public function getDef($wikitext, $count) {
 40 | 		$defArray = $this->extractDef($wikitext, $count);
 41 | 		return $this->stripTags($defArray);
 42 | 	}
 43 | /***********************************************************************************/
 44 | // private methods
 45 | /***********************************************************************************/
 46 | // Extracts all definitions by splitting at new lines and matching for definition
 47 | // tags set in paramaters.
 48 | /***********************************************************************************/
 49 | 	private function extractDef($wikitext, $count) {
 50 | 		if ($this->sectionPattern) {
 51 | 			return $this->extractDefSection($wikitext, $count);
 52 | 		}
 53 | 
 54 | 		// no section pattern, examine entire wikitext
 55 | 		preg_match_all($this->tagPattern, $wikitext, $matches);
 56 | 		if ($matches) {
 57 | 			return array_slice($matches[0], 0, $count);
 58 | 		} else {
 59 | 			return array();
 60 | 		}
 61 | 	}
 62 | 
 63 | 	private function extractDefSection($wikitext, $count)
 64 | 	{
 65 | 		preg_match_all($this->sectionPattern, $wikitext, $sectionMatches);
 66 | 		if (! $sectionMatches) {
 67 | 			return array();
 68 | 		}
 69 | 
 70 | 		$defArray = array();
 71 | 		foreach ($sectionMatches[0] as $value) {
 72 | 			// Find all matches for deftag + text until newline.
 73 | 			preg_match_all($this->tagPattern, $value, $tagMatches);
 74 | 			if ($tagMatches) {
 75 | 				foreach ($tagMatches[0] as $value) {
 76 | 					$defArray[] = $value;
 77 | 				}
 78 | 			}
 79 | 		}
 80 | 		return array_slice($defArray, 0, $count);
 81 | 	}
 82 | 
 83 | /***********************************************************************************/
 84 | // Strips tags used for additional info and links to other words.
 85 | /***********************************************************************************/
 86 | 	private function stripTags($defArray) {
 87 | 	// Strip anything enclosed between {{ }}
 88 | 		$strippedArray = preg_replace('(\{\{.*?\}\})', "", $defArray);
 89 | 	// Remove 1st half of [[word|Word]] strings.
 90 | 		$strippedArray = preg_replace('(\[\[[^\]]*?\|)u', "", $strippedArray);
 91 | 	// Remove brackets [[
 92 | 		$strippedArray = str_replace("[[", "", $strippedArray);
 93 | 	// Remove brackets ]]
 94 | 		$strippedArray = str_replace("]]", "", $strippedArray);
 95 | 	// Remove definition identifier
 96 | 		$strippedArray = str_replace($this->defTag, "", $strippedArray);
 97 | 
 98 | 		return $strippedArray;
 99 | 	}
100 | /***********************************************************************************/
101 | }
102 | 


--------------------------------------------------------------------------------
/lib/GenderParse.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib;
 3 | 
 4 | /***********************************************************************************/
 5 | // This class is used to parse the wikitionary raw data in order to extract gender
 6 | // for a given word.
 7 | // See the language.config.php file for setting language specific parameters.
 8 | /***********************************************************************************/
 9 | 
10 | class GenderParse {
11 | /***********************************************************************************/
12 | // Variables
13 | /***********************************************************************************/
14 | 	private $langCode;			// language code (e.g. en, fr, da, etc.)
15 | 	private $genderPattern;		// Gender regex pattern, set in config file
16 | 
17 | /***********************************************************************************/
18 | // construct
19 | /***********************************************************************************/
20 | 	public function __construct($langParameters) {
21 | 		if (empty($langParameters['genderPattern'])) {
22 | 			die("ERROR: Gender parameters are not set for this language in language.config.php.");
23 | 		}
24 | 		else {
25 | 			$this->langCode = $langParameters['langCode'];
26 | 			$this->genderPattern = $langParameters['genderPattern'];
27 | 		}
28 | 	}
29 | /***********************************************************************************/
30 | // public methods; used to retrieve contents of variables
31 | /***********************************************************************************/
32 | 	public function getGender($wikitext, $count) {
33 | 		$genderArray = $this->extractGender($wikitext, $count);
34 | 		$stripTagsObject = new StripTags();
35 | 		return $stripTagsObject->stripTags($genderArray, $this->langCode);
36 | 	}
37 | /***********************************************************************************/
38 | // private methods
39 | /***********************************************************************************/
40 | // Extracts every occurrence of gender.
41 | /***********************************************************************************/
42 | 	private function extractGender($wikitext, $count) {
43 | 		$tempGenderResults = array();
44 | 
45 | 		if ($this->genderPattern != "") {
46 | 
47 | 			preg_match_all($this->genderPattern, $wikitext, $matches);
48 | 
49 | 			if (empty($matches[0]) !== true) {
50 | 				foreach ($matches[0] as $value) {
51 | 					$tempGenderResults[] = $value;
52 | 				}
53 | 			// Remove values based on the count provide by the user.
54 | 				$tempGenderResults = array_slice($tempGenderResults, 0, $count);
55 | 			}
56 | 		}
57 | 		else {
58 | 			die("No gender pattern specified for this language.");
59 | 		}
60 | 
61 | 	// Return results if array not empty.
62 | 		if (empty($tempGenderResults) !==true) {
63 | 			return array_unique($tempGenderResults);
64 | 		}
65 | 		else {
66 | 			return array();
67 | 		}
68 | 	}
69 | /***********************************************************************************/
70 | } // End of class


--------------------------------------------------------------------------------
/lib/HyperParse.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib;
 3 | 
 4 | /***********************************************************************************/
 5 | // This class is used to parse the wikitionary raw data in order to extract hypernyms
 6 | // for a given word. Hypernyms are infrequent in Wiktionary entries.
 7 | // See the language.config.php file for setting language specific parameters.
 8 | /***********************************************************************************/
 9 | 
10 | class HyperParse {
11 | /***********************************************************************************/
12 | // Variables
13 | /***********************************************************************************/
14 | 	private $langCode;			// language code (e.g. en, fr, da, etc.)
15 | 	private $hyperHeader;		// Hypernyms header, set in config file
16 | 
17 | /***********************************************************************************/
18 | // construct
19 | /***********************************************************************************/
20 | 	public function __construct($langParameters) {
21 | 		if (empty($langParameters['hyperHeader'])) {
22 | 			die("ERROR: Hypernym parameter is not set for this language in language.config.php.");
23 | 		}
24 | 		else {
25 | 			$this->langCode = $langParameters['langCode'];
26 | 			$this->hyperHeader = $langParameters['hyperHeader'];
27 | 		}
28 | 	}
29 | /***********************************************************************************/
30 | // public methods
31 | /***********************************************************************************/
32 | 	public function getHyper($wikitext, $count) {
33 | 		$hyperArray = $this->extractHyper($wikitext, $count);
34 | 		return $this->stripTags($hyperArray);
35 | 	}
36 | /***********************************************************************************/
37 | // private methods
38 | /***********************************************************************************/
39 | // Extracts hypernyms from wikitext
40 | /***********************************************************************************/
41 | 	private function extractHyper($wikitext, $count) {
42 | 		$hyperString = null;
43 | 		$hyperPattern = "/$this->hyperHeader.*?\n\n/us";
44 | 		$itemPattern = "/\[\[.*?\]\]/u";
45 | 
46 | 	// If pattern returns results, then extract hypernyms
47 | 		if (preg_match_all($hyperPattern, $wikitext, $hyperMatch, PREG_PATTERN_ORDER) > 0) {
48 | 		// There may be more than one hypernym section. Fuse them together as string.
49 | 			foreach ($hyperMatch[0] as $value) {
50 | 				$hyperString .= $value;
51 | 			}
52 | 		// Match all double-bracketed words ([[...]])
53 | 			if (preg_match_all($itemPattern, $hyperString, $itemMatch, PREG_PATTERN_ORDER) > 0) {
54 | 				return array_slice($itemMatch[0], 0, $count);
55 | 			}
56 | 			else {
57 | 				return array();
58 | 			}
59 | 		}
60 | 		else {
61 | 			return array();
62 | 		}
63 | 	}
64 | /***********************************************************************************/
65 | // Removes unnecessary string elements from results
66 | /***********************************************************************************/
67 | 	private function stripTags($hyperArray) {
68 | 		$strippedArray = $hyperArray;
69 | 	// Remove first half of entries such as [[...:word]]
70 | 		$strippedArray = preg_replace("/\[\[.*?[|:]/u", "", $strippedArray);
71 | 	// Remove brackets [[
72 | 		$strippedArray = str_replace("[[", "", $strippedArray);
73 | 	// Remove brackets ]]
74 | 		$strippedArray = str_replace("]]", "", $strippedArray);
75 | 
76 | 		return $strippedArray;
77 | 	}
78 | /***********************************************************************************/
79 | } // End of class
80 | ?>


--------------------------------------------------------------------------------
/lib/Lang/AbstractLang.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib\Lang;
 3 | 
 4 | use ArrayAccess;
 5 | 
 6 | abstract class AbstractLang implements ArrayAccess
 7 | {
 8 |     protected $langCode;
 9 |     protected $langSeparator;
10 |     protected $defHeader;
11 |     protected $defTag;
12 |     protected $synHeader;
13 |     protected $hyperHeader;
14 |     protected $genderPattern;
15 |     protected $posMatchType;
16 |     protected $posPattern;
17 |     protected $posArray = array();
18 |     protected $posExtraString;
19 |     protected $langHeader;
20 | 
21 |     public function offsetExists($key)
22 |     {
23 |         return property_exists($this, $key);
24 |     }
25 | 
26 |     public function offsetGet($key)
27 |     {
28 |         return $this->$key;
29 |     }
30 | 
31 |     public function offsetSet($key, $val)
32 |     {
33 |         $this->$key = $val;
34 |     }
35 | 
36 |     public function offsetUnset($key)
37 |     {
38 |         $this->$key = null;
39 |     }
40 | }
41 | 


--------------------------------------------------------------------------------
/lib/Lang/De.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib\Lang;
 3 | 
 4 | class De extends AbstractLang
 5 | {
 6 |     protected $langCode = "de";
 7 |     protected $langSeparator = "({{Sprache|";
 8 |     protected $defHeader = "{{Bedeutungen}}";
 9 |     protected $defTag = ":";
10 |     protected $synHeader = "{{Synonyme}}";
11 |     protected $hyperHeader = "{{Oberbegriffe}}";
12 |     protected $genderPattern = "(\{\{[mfn]\}\}\s===)";
13 |     protected $posMatchType = "preg";
14 |     protected $posPattern = "(\{\{Wortart\|\w+\|)";
15 |     protected $posArray = array();
16 |     protected $posExtraString = "{{Wortart|";
17 |     protected $langHeader = 'Deutsch}})\s*==';
18 | }
19 | 


--------------------------------------------------------------------------------
/lib/Lang/En.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib\Lang;
 3 | 
 4 | class En extends AbstractLang
 5 | {
 6 |     protected $langCode = "en";
 7 |     protected $langSeparator = "----";
 8 |     protected $defHeader = "";
 9 |     protected $defTag = "# ";
10 |     protected $synHeader = "====Synonyms====";
11 |     protected $hyperHeader = "====Hypernyms====";
12 |     protected $genderPattern = "";
13 |     protected $posMatchType = "array";
14 |     protected $posPattern = "";
15 |     protected $posArray = array(
16 |         '===Noun===',
17 |         '===Verb===',
18 |         '===Adjective===',
19 |         '===Adverb===',
20 |         '===Preposition===',
21 |         '===Particle===',
22 |         '===Pronouns===',
23 |         '===Interjection===',
24 |         '===Conjunction===',
25 |         '===Article==='
26 |     );
27 |     protected $posExtraString = "=";
28 |     protected $langHeader = '==\s*English\s*==';
29 | }
30 | 


--------------------------------------------------------------------------------
/lib/Lang/Es.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib\Lang;
 3 | 
 4 | class Es extends AbstractLang
 5 | {
 6 |     protected $langCode = "es";
 7 |     protected $langSeparator = "";
 8 |     protected $defHeader = "";
 9 |     protected $defTag = ";";
10 |     protected $synHeader = "'''Sinónimo";
11 |     protected $hyperHeader = "";
12 |     protected $genderPattern = "(\s?(masculino|femenino)(\|es)?\}\}\s?===)";
13 |     protected $posMatchType = "preg";
14 |     protected $posPattern = "(===\s?\{\{\w*[\|\s])";
15 |     protected $posArray = array();
16 |     protected $posExtraString = "";
17 |     protected $langHeader = '{{ES';
18 | }
19 | 


--------------------------------------------------------------------------------
/lib/Lang/Fr.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib\Lang;
 3 | 
 4 | class Fr extends AbstractLang
 5 | {
 6 |     protected $langCode = "fr";
 7 |     protected $langSeparator = "== {{=";
 8 |     protected $defHeader = "";
 9 |     protected $defTag = "# ";
10 |     protected $synHeader = "==== {{S|synonymes}} ====";
11 |     protected $hyperHeader = "==== {{S|hyperonymes}} ====";
12 |     protected $genderPattern = "(\{\{([mf]|mf)\??\}\})";
13 |     protected $posMatchType = "preg";
14 |     protected $posPattern = "(\{\{\S\|[\d\w\s]+\|fr(\|num=[0-9])?\}\})";
15 |     protected $posArray = array();
16 |     protected $posExtraString = "{{S|";
17 |     protected $langHeader = 'fr}}\s*==';
18 | }
19 | 


--------------------------------------------------------------------------------
/lib/PosParse.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib;
 3 | 
 4 | /***********************************************************************************/
 5 | // This class is used to parse the wikitionary raw data in order to extract parts
 6 | // of speech for a given word.
 7 | // See the language.config.php file for setting language specific parameters.
 8 | /***********************************************************************************/
 9 | 
10 | class PosParse {
11 | /***********************************************************************************/
12 | // Variables
13 | /***********************************************************************************/
14 | 	private $langCode;			// language code (e.g. en, fr, da, etc.)
15 | 	private $posMatchType;		// POS Match Type, set in config file
16 | 	private $posArray;			// POS Array, set in config file
17 | 	private $posPattern;		// POS regex pattern, set in config file
18 | 	private $posExtraString;	// POS extra string, set in config file
19 | 
20 | /***********************************************************************************/
21 | // construct
22 | /***********************************************************************************/
23 | 	public function __construct($langParameters) {
24 | 		if
25 | 		(empty($langParameters['posMatchType']) ||
26 | 		(empty($langParameters['posPattern']) && empty($langParameters['posArray']))) {
27 | 			die("ERROR: POS parameters are not set correctly for this language in language.config.php.");
28 | 		}
29 | 		else {
30 | 			$this->langCode = $langParameters['langCode'];
31 | 			$this->posMatchType = $langParameters['posMatchType'];
32 | 			$this->posArray = $langParameters['posArray'];
33 | 			$this->posPattern = $langParameters['posPattern'];
34 | 			$this->posExtraString = $langParameters['posExtraString'];
35 | 		}
36 | 	}
37 | /***********************************************************************************/
38 | // public methods; used to retrieve contents of variables
39 | /***********************************************************************************/
40 | 	public function getPos($wikitext, $count) {
41 | 		$posArray = $this->extractPos($wikitext, $count);
42 | 		$stripTagsObject = new StripTags();
43 | 		$posArray = $stripTagsObject->stripTags($posArray, $this->langCode);
44 | 		return array_unique($posArray);
45 | 	}
46 | /***********************************************************************************/
47 | // private methods
48 | /***********************************************************************************/
49 | 
50 | /***********************************************************************************/
51 | // Extracts every occurrence of a part of speech.
52 | /***********************************************************************************/
53 | 	private function extractPos($wikitext, $count) {
54 | 		$tempPosResults = array();
55 | 
56 | 	// If the matches are in an array
57 | 		if ($this->posMatchType == "array") {
58 | 			foreach ($this->posArray as $value) {
59 | 				if (strpos($wikitext, $value)) {
60 | 					$tempPosResults[] = str_replace($this->posExtraString, "", $value);
61 | 				}
62 | 			}
63 | 		}
64 | 	// Else if the matches are part of a regular expression
65 | 		else if ($this->posMatchType == "preg") {
66 | 			preg_match_all('/' . $this->posPattern . '/', $wikitext, $matches);
67 | 			if (empty($matches[0]) !== true) {
68 | 				foreach ($matches[0] as $value) {
69 | 					$tempPosResults[] = str_replace($this->posExtraString, "", $value);
70 | 				}
71 | 			}
72 | 		}
73 | 		else {
74 | 			die("No POS type (preg or array) specified.");
75 | 		}
76 | 
77 | 	// Return results if array not empty.
78 | 		if (empty($tempPosResults) !== true) {
79 | 			return array_slice($tempPosResults, 0, $count);
80 | 		}
81 | 		else {
82 | 			return array();
83 | 		}
84 | 	}
85 | /***********************************************************************************/
86 | } // End of class


--------------------------------------------------------------------------------
/lib/StripTags.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib;
 3 | 
 4 | /***********************************************************************************/
 5 | // This class strips common tags from Wiktionary's raw data.
 6 | // Accepts an array.
 7 | /***********************************************************************************/
 8 | 
 9 | class StripTags {
10 | /***********************************************************************************/
11 | // Variables
12 | /***********************************************************************************/
13 | 
14 | /***********************************************************************************/
15 | // construct
16 | /***********************************************************************************/
17 | 	public function __construct() {
18 | 	}
19 | /***********************************************************************************/
20 | // public methods
21 | /***********************************************************************************/
22 | 	public function stripTags($array, $langCode) {
23 | 		$finalArray = array();
24 | 		$tags_array = array("[", "]", "{", "}", "=");
25 | 		foreach ($tags_array as $tag) {
26 | 			$array = str_replace($tag, "", $array);
27 | 		}
28 | 		foreach ($array as $string) {
29 | 			$string = trim($string);
30 | 
31 | 		// Trims language code from end of string if preceded by | (e.g. Spanish gender)
32 | 			$finalArray[] = preg_replace("/\|$langCode$/", "", $string);
33 | 		}
34 | 		return $finalArray;
35 | 	}
36 | /***********************************************************************************/
37 | } // End of class
38 | ?>


--------------------------------------------------------------------------------
/lib/SynParse.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace ybourque\Wikparser\lib;
 3 | 
 4 | /***********************************************************************************/
 5 | // This class is used to parse the wikitionary raw data in order to extract synonyms
 6 | // for a given word. Synonyms are not systematically included in Wiktionary entries.
 7 | // See the language.config.php file for setting language specific parameters.
 8 | /***********************************************************************************/
 9 | 
10 | class SynParse {
11 | /***********************************************************************************/
12 | // Variables
13 | /***********************************************************************************/
14 | 	private $langCode;		// language code (e.g. en, fr, da, etc.)
15 | 	private $synHeader; 	// Synonyms header, set in config file
16 | 
17 | /***********************************************************************************/
18 | // construct
19 | /***********************************************************************************/
20 | 	public function __construct($langParameters) {
21 | 		if (empty($langParameters['synHeader'])) {
22 | 			die("ERROR: Synonym parameter is not set for this language in language.config.php.");
23 | 		}
24 | 		else {
25 | 			$this->langCode = $langParameters['langCode'];
26 | 			$this->synHeader = $langParameters['synHeader'];
27 | 		}
28 | 	}
29 | /***********************************************************************************/
30 | // public methods
31 | /***********************************************************************************/
32 | 	public function getSyn($wikitext, $count) {
33 | 		$synArray = $this->extractSyn($wikitext, $count);
34 | 		return $this->stripTags($synArray);
35 | 	}
36 | /***********************************************************************************/
37 | // private methods
38 | /***********************************************************************************/
39 | // Extracts synonyms from wikitext
40 | /***********************************************************************************/
41 | 	private function extractSyn($wikitext, $count) {
42 | 		$synString = null;
43 | 		$synPattern = "/$this->synHeader.*?\n\n/us";
44 | 		$itemPattern = "/\[\[.*?\]\]/u";
45 | 
46 | 	// If pattern returns results, then extract synonyms
47 | 		if (preg_match_all($synPattern, $wikitext, $synMatch, PREG_PATTERN_ORDER) > 0) {
48 | 		// There may be more than one synonym section. Fuse them together as string.
49 | 			foreach ($synMatch[0] as $value) {
50 | 				$synString .= $value;
51 | 			}
52 | 		// Match all double-bracketed words ([[...]])
53 | 			if (preg_match_all($itemPattern, $synString, $itemMatch, PREG_PATTERN_ORDER) > 0) {
54 | 				return array_slice($itemMatch[0], 0, $count);
55 | 			}
56 | 			else {
57 | 				return array();
58 | 			}
59 | 		}
60 | 		else {
61 | 			return array();
62 | 		}
63 | 	}
64 | /***********************************************************************************/
65 | // Removes unnecessary string elements from results
66 | /***********************************************************************************/
67 | 	private function stripTags($synArray) {
68 | 		$strippedArray = $synArray;
69 | 	// Remove first half of entries such as [[...:word]]
70 | 		$strippedArray = preg_replace("/\[\[.*?[|:]/u", "", $strippedArray);
71 | 	// Remove brackets [[
72 | 		$strippedArray = str_replace("[[", "", $strippedArray);
73 | 	// Remove brackets ]]
74 | 		$strippedArray = str_replace("]]", "", $strippedArray);
75 | 
76 | 		return $strippedArray;
77 | 	}
78 | /***********************************************************************************/
79 | } // End of class
80 | ?>


--------------------------------------------------------------------------------
/lib/WikiExtract.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace ybourque\Wikparser\lib;
  3 | /***********************************************************************************/
  4 | // Gets all raw text for a given word via either Wiktionary's API or a local MySQL
  5 | // copy (defined in conc.php). Accepts 3 variables:
  6 | // $word = string
  7 | // $langCode = string (2-letter language code)
  8 | // $wikiSource = string (either 'api' or 'local' for local MySQL).
  9 | /***********************************************************************************/
 10 | 
 11 | class WikiExtract {
 12 | /***********************************************************************************/
 13 | // Variables
 14 | /***********************************************************************************/
 15 | 	private $wikiSource; // api or local
 16 | 	private $langCode; // 2-letter language code
 17 | 	private $langHeader; // language header from config file
 18 | 	private $langSeparator; // language separator string from config file
 19 | 
 20 | /***********************************************************************************/
 21 | // construct
 22 | /***********************************************************************************/
 23 | 	public function __construct($langParameters, $wikiSource) {
 24 | 		$this->wikiSource = $wikiSource;
 25 | 		$this->langCode = $langParameters['langCode'];
 26 | 		$this->langHeader = $langParameters['langHeader'];
 27 | 		$this->langSeparator = $langParameters['langSeparator'];
 28 | 	}
 29 | 
 30 | /***********************************************************************************/
 31 | // public methods
 32 | /***********************************************************************************/
 33 | 	public function getWikiText($word) {
 34 | 		if ($this->wikiSource == 'local') {
 35 | 			$wikitext = $this->sqlFetchData($word);
 36 | 		}
 37 | 		else if ($this->wikiSource == 'api') {
 38 | 			$wikitext = $this->getWikiTextFromWiktionary($word);
 39 | 		}
 40 | 
 41 | 		return $this->langExtract($wikitext);
 42 | 	}
 43 | /***********************************************************************************/
 44 | // private methods
 45 | /***********************************************************************************/
 46 | 
 47 | /***********************************************************************************/
 48 | // Retrieves raw data via Wiktionary's API.
 49 | /***********************************************************************************/
 50 | 	private function getWikiTextFromWiktionary($word) {
 51 | 		$word = urlencode($word);
 52 | 
 53 | 	// Must be supplied, otherwise IP will be banned
 54 | 		$userAgent = "Wikitionary Text Parser 0.3 (http://www.igrec.ca/projects)";
 55 | 	// Paramaters passed to the Wik API, including search word.
 56 | 		$params = '?action=parse&prop=wikitext&page='.$word.'&format=xml';
 57 | 
 58 | 		$ch = curl_init('https://'.$this->langCode.'.wiktionary.org/w/api.php'.$params);
 59 | 		curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
 60 | 		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
 61 | 		curl_setopt($ch,CURLOPT_ENCODING , "gzip");
 62 | 
 63 | 		$wikiAPIResult = curl_exec($ch);
 64 | 		curl_close($ch);
 65 | 
 66 | 		if (strpos($wikiAPIResult, "error code=\"missingtitle\"") !== false) {
 67 | 			die("ERROR: The Wiktionary API did not return a page for that word.");
 68 | 		}
 69 | 		else {
 70 | 			return $wikiAPIResult;
 71 | 		}
 72 | 	}
 73 | /***********************************************************************************/
 74 | // Retrieves raw data via a local MySQL copy of Wiktionary (defined in conc.php).
 75 | /***********************************************************************************/
 76 | 	private function sqlFetchData($word) {
 77 | 
 78 | 		include 'conc.php';
 79 | 		$word = mysqli_real_escape_string($conn, $word);
 80 | 
 81 | 	// 3 tables are used page->revision->text
 82 | 		$query = "SELECT t.old_text FROM text t ";
 83 | 		$query .= "JOIN revision r ON r.rev_text_id = t.old_id ";
 84 | 		$query .= "JOIN page p ON r.rev_page = p.page_id ";
 85 | 		$query .= "WHERE p.page_title = '$word' AND p.page_namespace = 0";
 86 | 
 87 | 		if (!$queryResult = $conn->query($query)) {
 88 | 			die("Error: Couldn't query word.");
 89 | 		}
 90 | 
 91 | 		if ($queryResult->num_rows > 0) {
 92 | 			while ($row = $queryResult->fetch_assoc()){
 93 | 				$wikitext = $row['old_text'];
 94 | 			}
 95 | 			return $wikitext;
 96 | 		}
 97 | 		else {
 98 | 			die("No such word found.");
 99 | 		}
100 | 	}
101 | /***********************************************************************************/
102 | // Extracts content for specified language from raw wiktionary data. Some entries
103 | // contain text for other languages.
104 | /***********************************************************************************/
105 | 	private function langExtract($wikitext){
106 | 		$bln = false;
107 | 
108 | 		if ($this->langSeparator !== "") {
109 | 			$languages = explode($this->langSeparator, $wikitext);
110 | 
111 | 			foreach ($languages as $value) {
112 | 				if (preg_match('/'.$this->langHeader.'/', $value) > 0) {
113 | 					$wikitext = $value;
114 | 					$bln = true;
115 | 				}
116 | 			}
117 | 			if ($bln !== true) {
118 | 				die("No such word for specified language.");
119 | 			}
120 | 			else {
121 | 				return $wikitext;
122 | 			}
123 | 		}
124 | 		else {
125 | 			return $wikitext;
126 | 		}
127 | 	}
128 | }
129 | 
130 | ?>


--------------------------------------------------------------------------------
/lib/conc.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | /***********************************************************************************/
 3 | // Change the values for these variables according to the naming scheme of your own
 4 | // SQL copy of Wiktionary.
 5 | /***********************************************************************************/
 6 | 	$dbHost = 'hostname'; // e.g. localhost
 7 | 	$dbUser = 'username';
 8 | 	$dbPass = 'password';
 9 | 	$dbName = 'database';
10 | 	
11 | /***********************************************************************************/
12 | 	$conn = new mysqli($dbHost, $dbUser, $dbPass, $dbName);
13 | 	
14 | 	if ($conn->connect_errno > 0) {
15 | 		die("Unable to connect to database [".$conn->connect_error."].");
16 | 	}
17 | 	
18 | 	mysqli_set_charset($conn, 'utf8');
19 | /***********************************************************************************/
20 | ?>


--------------------------------------------------------------------------------
/tests/test.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | use ybourque\Wikparser\Wikparser;
 3 | 
 4 | require dirname(__DIR__) . "/vendor/autoload.php";
 5 | 
 6 | $wik = new WikParser();
 7 | $queries = ['def', 'pos', 'syn', 'hyper', 'gender'];
 8 | $parsed = $wik->getWordDefiniton('bon', $queries, 'fr');
 9 | var_dump($parsed);
10 | 
11 | /*
12 | requires a separate call for each element. modify to make one call and return
13 | all parsed elements?
14 | */
15 | 


--------------------------------------------------------------------------------