├── .gitignore ├── LICENSE ├── README.md └── RIBES.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RIBES: Rank-based Intuitive Bilingual Evaluation Score 2 | 3 | ## What's RIBES? 4 | 5 | RIBES is an automatic evaluation metric for machine translation, developed in NTT Communication Science Labs. 6 | This website distributes its implementation in Python. 7 | The program is distributed based on GNU General Public License (ver.2)． 8 | You have to confirm the agreement with the license terms before download. 9 | 10 | 11 | ## How to Use 12 | 13 | ```bash 14 | $ python RIBES.py -r REFERENCE_TRANSLATION EVALUATING_TRANSLATION 15 | ``` 16 | Also you can see other options by the following: 17 | ``` 18 | $ python RIBES.py -h 19 | ``` 20 | 21 | ## File Format 22 | 23 | * Reference and evaluating files are plain text files, not SGML or XML. 24 | * Reference and evaluating files must have the same number of lines. 25 | * Each line must be tokenized including punctuations (The program does not change tokenization). 26 | 27 | 28 | ## References 29 | 30 | * PDF http://www.aclweb.org/anthology/D/D10/D10-1092.pdf 31 | * bib http://www.aclweb.org/anthology/D/D10/D10-1092.bib 32 | 33 | Hideki Isozaki, Tsutomu Hirao, Kevin Duh, Katsuhito Sudoh, Hajime Tsukada 34 | Automatic Evaluation of Translation Quality for Distant Language Pairs 35 | Conference on Empirical Methods on Natural Language Processing (EMNLP), Oct. 2010. 36 | 37 | 38 | ## Versions 39 | 40 | * Latest version 1.03.1 (2014/9/8) -- fixed a compatibility problem of Python 2's split / introduced a new option -z/--emptryref to allow blank lines in references 41 | * Version 1.03 (2014/8/13) -- supports Python 2.6 or later / only supports utf-8 / fixed the multibyte white space issue (Thanks to Graham Neubig) 42 | * Version 1.02.4 (2013/12/18) -- fixed a problem in word alignment 43 | * Version 1.02.3 (2012/2/23) 44 | * Version 1.01 (2011/8/10) 45 | 46 | Requires Python 2.6 or later (Python 3.0 is not supported. RIBES-1.02.4 or earlier only works with Python 3.1 or later) 47 | 48 | ## Original download website 49 | 50 | * http://www.kecl.ntt.co.jp/icl/lirg/ribes/index.html 51 | 52 | ## Contact 53 | 54 | (the name of our evaluation measure, lowercased)@lab.ntt.co.jp 55 | -------------------------------------------------------------------------------- /RIBES.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8-unix -*- 3 | ### 4 | ### RIBES.py - RIBES (Rank-based Intuitive Bilingual Evaluation Score) scorer 5 | ### Copyright (C) 2011-2014 Nippon Telegraph and Telephone Corporation 6 | ### 7 | ### This program is free software; you can redistribute it and/or 8 | ### modify it under the terms of the GNU General Public License 9 | ### as published by the Free Software Foundation; either version 2 10 | ### of the License, or (at your option) any later version. 11 | ### 12 | ### This program is distributed in the hope that it will be useful, 13 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | ### GNU General Public License for more details. 16 | ### 17 | ### You should have received a copy of the GNU General Public License 18 | ### along with this program; if not, write to the Free Software 19 | ### Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | ### 21 | ## History 22 | ## version 1.03.1 (2014/9/8) Fixed a compatibility problem of "split", which allows zero-length references wrongly with Python 2. 23 | ## Introduced a new option "-z/--emptyref" to allow zero-length references, which would be helpful for evaluation on data with blank lines. 24 | ## version 1.03 (2014/8/13) Supports Python 2.6 or higher 25 | ## Eliminated encoding option (now RIBES.py only supports utf-8) 26 | ## Limits word delimiters to ASCII white spaces (now multibyte spaces cannot be used as word delimiters) 27 | ## version 1.02.4 (2013/12/17) Fixed a problem in word alignment 28 | ## version 1.02.3 (2012/2/23) Fixed a problem in output 29 | ## version 1.02.2 (2011/10/25) Fixed a problem without -o option (in systems without /dev/stdout) 30 | ## version 1.02.1 (2011/8/18) Fixed bug on bytes.decode 31 | ## version 1.02 (2011/8/16) Improved distinguishment of same words, with a little code refactoring 32 | ## version 1.01 (2011/8/10) Fixed bug on empty lines 33 | ## version 1.0 (2011/8/1) Initial release 34 | # 35 | # Reference: 36 | # Tsutomu Hirao, Hideki Isozaki, Katsuhito Sudoh, Kevin Duh, Hajime Tsukada, and Masaaki Nagata, 37 | # "Evaluating Translation Quality with Word Order Correlations," 38 | # Journal of Natural Language Processing, Vol. 21, No. 3, pp. 421-444, June, 2014 (in Japanese). 39 | # 40 | # Hideki Isozaki, Tsutomu Hirao, Kevin Duh, Katsuhito Sudoh, and Hajime Tsukada, 41 | # "Automatic Evaluation of Translation Quality for Distant Language Pairs," 42 | # Proceedings of the 2010 Conference on Empirical Methods in Natural Language Processing (EMNLP), 43 | # pp. 944--952 Cambridge MA, October, 2010 44 | # -- http://aclweb.org/anthology-new/D/D10/D10-1092.pdf 45 | # 46 | 47 | from __future__ import print_function 48 | import sys 49 | if type(sys.version_info) is not tuple and sys.version_info.major != 3: 50 | reload(sys) 51 | sys.setdefaultencoding("utf-8") 52 | import os,re 53 | import datetime 54 | import traceback 55 | from optparse import OptionParser 56 | from math import exp 57 | 58 | _RIBES_VERSION = '1.03' 59 | debug = 0 60 | 61 | multiws_pattern = re.compile(r'\s+') 62 | 63 | ### "overlapping" substring counts ( string.count(x) returns "non-overlapping" counts... ) 64 | def overlapping_count (pattern, string): 65 | pos = string.find(pattern) 66 | if pos > -1: 67 | return 1 + overlapping_count (pattern, string[pos+1:]) 68 | else: 69 | return 0 70 | 71 | ### calculate Kendall's tau 72 | def kendall(ref, hyp, emptyref=False): 73 | """Calculates Kendall's tau between a reference and a hypothesis 74 | 75 | Calculates Kendall's tau (also unigram precision and brevity penalty (BP)) 76 | between a reference word list and a system output (hypothesis) word list. 77 | 78 | Arguments: 79 | ref : list of reference words 80 | sub : list of system output (hypothesis) words 81 | (optional) emptyref : allow empty reference translations (ignored in the evaluation) 82 | 83 | Returns: 84 | A tuple (nkt, precision, bp) 85 | - nkt : normalized Kendall's tau 86 | - precision : unigram precision 87 | - bp : brevity penalty 88 | 89 | Raises: 90 | RuntimeError: reference has no words, possibly due to a format violation 91 | """ 92 | 93 | # check reference length, raise RuntimeError if no words are found. 94 | if len(ref) == 0: 95 | if emptyref == True: 96 | return (None, None, None) 97 | else: 98 | raise RuntimeError ("Reference has no words") 99 | # check hypothesis length, return "zeros" if no words are found 100 | elif len(hyp) == 0: 101 | if debug > 1: print ("nkt=%g, precision=%g, bp=%g" % (0.0, 0.0, 0.0), file=sys.stderr) 102 | return (0.0, 0.0, 0.0) 103 | # bypass -- return 1.0 for identical hypothesis 104 | #elif ref == hyp: 105 | # if debug > 1: print ("nkt=%g, precision=%g, bp=%g" % (nkt, precision, bp), file=sys.stderr) 106 | # return (1.0, 1.0, 1.0) 107 | 108 | # calculate brevity penalty (BP), not exceeding 1.0 109 | bp = min(1.0, exp(1.0 - 1.0 * len(ref)/len(hyp))) 110 | 111 | 112 | ### determine which ref. word corresponds to each hypothesis word 113 | # list for ref. word indices 114 | intlist = [] 115 | 116 | 117 | ### prepare helper pseudo-string representing ref. and hyp. word sequences as strings, 118 | ### by mapping each word into non-overlapping Unicode characters 119 | # Word ID (dictionary) 120 | worddict = {} 121 | # Unicode hexadecimal sequences for ref. and words 122 | _ref = "" 123 | _hyp = "" 124 | for w in ref: 125 | # if w is not found in dictironary "worddict", add it. 126 | if w not in worddict: 127 | worddict[w] = len(worddict) 128 | # append Unicode hexadecimal for word w (with offset of 0x4e00 -- CJK character range) 129 | _ref += str(hex(worddict[w] + 0x4e00)).replace('0x', '', 1) 130 | # decode Unicode (UTF-16 BigEndian) sequences to UTF-8 131 | if type(sys.version_info) is not tuple and sys.version_info.major == 3: 132 | if sys.version_info.minor > 1: 133 | mapped_ref = bytes.fromhex(_ref).decode(encoding="utf_16_be") 134 | else: 135 | mapped_ref = bytes.fromhex(_ref).decode("utf_16_be") 136 | else: 137 | mapped_ref = _ref.decode("hex").decode("utf_16_be") 138 | 139 | for w in hyp: 140 | # if w is not found in dictironary "worddict", add it. 141 | if w not in worddict: 142 | worddict[w] = len(worddict) 143 | # append Unicode hexadecimal for word w (with offset of 0x4e00 -- CJK character range) 144 | _hyp += str(hex(worddict[w] + 0x4e00)).replace('0x', '', 1) 145 | # decode Unicode (UTF-16 BigEndian) sequences to UTF-8 146 | if type(sys.version_info) is not tuple and sys.version_info.major == 3: 147 | if sys.version_info.minor > 1: 148 | mapped_hyp = bytes.fromhex(_hyp).decode(encoding="utf_16_be") 149 | else: 150 | mapped_hyp = bytes.fromhex(_hyp).decode("utf_16_be") 151 | else: 152 | mapped_hyp = _hyp.decode("hex").decode("utf_16_be") 153 | 154 | for i in range(len(hyp)): 155 | ### i-th hypthesis word hyp[i] 156 | if not hyp[i] in ref: 157 | ### hyp[i] doesn't exist in reference 158 | pass 159 | # go on to the next hyp. word 160 | elif ref.count(hyp[i]) == 1 and hyp.count(hyp[i]) == 1: 161 | ### if we can determine one-to-one word correspondence by only unigram 162 | ### one-to-one correspondence 163 | # append the index in reference 164 | intlist.append(ref.index(hyp[i])) 165 | # go on to the next hyp. word 166 | else: 167 | ### if not, we consider context words... 168 | # use Unicode-mapped string for efficiency 169 | for window in range (1, max(i+1, len(hyp)-i+1)): 170 | if window <= i: 171 | ngram = mapped_hyp[i-window:i+1] 172 | if overlapping_count(ngram, mapped_ref) == 1 and overlapping_count(ngram, mapped_hyp) == 1: 173 | intlist.append(mapped_ref.index(ngram) + len(ngram) -1) 174 | break 175 | if i+window < len(hyp): 176 | ngram = mapped_hyp[i:i+window+1] 177 | if overlapping_count(ngram, mapped_ref) == 1 and overlapping_count(ngram, mapped_hyp) == 1: 178 | intlist.append(mapped_ref.index(ngram)) 179 | break 180 | 181 | ### At least two word correspondences are needed for rank correlation 182 | n = len(intlist) 183 | if n == 1 and len(ref) == 1: 184 | if debug > 1: print ("nkt=%g, precision=%g, bp=%g" % (1.0, 1.0/len(hyp), bp), file=sys.stderr) 185 | return (1.0, 1.0/len(hyp), bp) 186 | elif n < 2: 187 | # if not, return score 0.0 188 | if debug > 1: print ("nkt=%g, precision=%g, bp=%g" % (0.0, 0.0, bp), file=sys.stderr) 189 | return (0.0, 0.0, bp) 190 | 191 | ### calculation of rank correlation coefficient 192 | # count "ascending pairs" (intlist[i] < intlist[j]) 193 | ascending = 0.0 194 | for i in range(len(intlist)-1): 195 | for j in range(i+1,len(intlist)): 196 | if intlist[i] < intlist[j]: 197 | ascending += 1 198 | 199 | # normalize Kendall's tau 200 | nkt = ascending / ((n * (n - 1))/2) 201 | 202 | # calculate unigram precision 203 | precision = 1.0 * n / len(hyp) 204 | 205 | # return tuple (Normalized Kendall's tau, Unigram Precision, and Brevity Penalty) 206 | if debug > 1: print ("nkt=%g, precision=%g, bp=%g" % (nkt, precision, bp), file=sys.stderr) 207 | return (nkt, precision, bp) 208 | 209 | 210 | class RIBESevaluator: 211 | """RIBES evaluator class. 212 | 213 | Receives "Corpus" instances and score them with hyperparameters alpha and beta. 214 | 215 | Attributes (private): 216 | __sent : show sentence-level scores or not 217 | __alpha : hyperparameter alpha, for (unigram_precision)**alpha 218 | __beta : hyperparameter beta, for (brevity_penalty)**beta 219 | __output : output file name 220 | """ 221 | def __init__ (self, sent=False, alpha=0.25, beta=0.10, output=sys.stdout): 222 | """Constructor. 223 | 224 | Initialize a RIBESevaluator instance with four attributes. All attributes have their default values. 225 | 226 | Arguments (Keywords): 227 | - sent : for attribute __sent, default False 228 | - alpha : for attribute __alpha, default 0.25 229 | - beta : for attribute __beta, default 0.10 230 | - output : for attribute __output, default sys.stdout 231 | """ 232 | self.__sent = sent 233 | self.__alpha = alpha 234 | self.__beta = beta 235 | self.__output = output 236 | 237 | 238 | def eval (self, hyp, REFS, emptyref=False): 239 | """Evaluate a system output with multiple references. 240 | 241 | Calculates RIBES for a system output (hypothesis) with multiple references, 242 | and returns "best" score among multi-references and individual scores. 243 | The scores are corpus-wise, i.e., averaged by the number of sentences. 244 | 245 | Arguments: 246 | hyp : "Corpus" instance of hypothesis 247 | REFS : list of "Corpus" instances of references 248 | (optional) emptyref : allow empty reference translations (default: False; ignored in the evaluation) 249 | 250 | Returns: 251 | A floating point value _best_ribes_acc 252 | - _best_ribes_acc : best corpus-wise RIBES among multi-reference 253 | 254 | Raises: 255 | RuntimeError : #sentences of hypothesis and reference doesn't match 256 | RuntimeError : from the function "kendall" 257 | """ 258 | 259 | for ref in REFS: 260 | # check #sentences of hypothesis and each of the multi-references 261 | if len(hyp) != len(ref): 262 | raise RuntimeError ( "Different #sentences between " + hyp.filename + " (" + str(len(hyp)) + "sents.) and " + ref.filename + "( " + str(len(ref)) + "sents.)") 263 | 264 | # initialize "best" corpus-wise score 265 | _best_ribes_acc = 0.0 266 | # the number of valid sentences with at least one non-empty reference translations 267 | _num_valid_refs = 0 268 | 269 | # scores each hypothesis 270 | for i in range (len(hyp)): 271 | # initialize "best" sentence-wise score 272 | _best_ribes = -1.0 273 | 274 | # for each reference 275 | for r in range(len(REFS)): 276 | try: 277 | # calculate Kendall's tau, unigram precision, and brevity penalty. 278 | (nkt, precision, bp) = kendall(REFS[r][i], hyp[i], emptyref=emptyref) 279 | except Exception as e: 280 | # if the function "kendall" raises an exception, throw toward the main function 281 | print ("Error in " + REFS[r].filename + " line " + str(i), file=sys.stderr) 282 | raise e 283 | 284 | # in case of an empty reference, ignore this 285 | if nkt != None: 286 | # RIBES = (normalized Kendall's tau) * (unigram_precision ** alpha) * (brevity_penalty ** beta) 287 | _ribes = nkt * (precision ** self.__alpha) * (bp ** self.__beta) 288 | 289 | # maintain the best sentence-wise score 290 | if _ribes > _best_ribes: 291 | _best_ribes = _ribes 292 | 293 | if _best_ribes > -1.0: 294 | # found a non-empty reference translation 295 | _num_valid_refs += 1 296 | 297 | # accumulate the "best" sentence-wise score for the "best" corpus-wise score 298 | _best_ribes_acc += _best_ribes 299 | 300 | # print "best" sentence-wise score if __sent is True 301 | if self.__sent and self.__output != None: 302 | print ("%.6f alpha=%f beta=%f %s sentence %d" % (_best_ribes, self.__alpha, self.__beta, hyp.filename, i), file=self.__output) 303 | elif self.__sent and self.__output != None: 304 | print ("%.6f alpha=%f beta=%f %s sentence %d" % (-float("inf"), self.__alpha, self.__beta, hyp.filename, i), file=self.__output) 305 | 306 | # returns the "best" corpus-wise RIBES 307 | return _best_ribes_acc / _num_valid_refs 308 | 309 | 310 | class Corpus: 311 | """Corpus class. 312 | 313 | Stores sentences and is used for evaluation. 314 | 315 | Attributes (private): 316 | __sentence : list of sentences (word lists) 317 | __numwords : #words in the corpus (currently not used but can be used for corpus statistics.) 318 | 319 | Attributes (public): 320 | filename : corpus file name (set as public for error messages about the corpus) 321 | """ 322 | def __init__ (self, _file, case=False): 323 | """Constructor. 324 | 325 | Initialize a Corpus instance by a corpus file with a utf-8 encoding. 326 | 327 | Argument: 328 | _file : corpus file of "sentence-per-line" format 329 | Keyword: 330 | case : preserve uppercase letters or not, default: False 331 | """ 332 | 333 | # initialize contents 334 | self.__sentence = [] 335 | self.__numwords = 0 336 | 337 | # set file name 338 | self.filename = _file 339 | 340 | # read corpus 341 | with open (_file) as fp: 342 | for line in fp: 343 | # eliminates unnecessary spaces (white spaces and tabs) in each sentence 344 | line = multiws_pattern.sub(r' ', line.strip()) 345 | 346 | # lowercasing if case is False 347 | if not case: 348 | line = line.lower() 349 | 350 | # split the sentence to a word list and append it to the corpus sentence list 351 | if len(line) == 0: 352 | self.__sentence.append( [] ) 353 | else: 354 | self.__sentence.append( line.split(" ") ) 355 | 356 | # count words 357 | self.__numwords += len(self.__sentence[-1]) 358 | 359 | def __len__ (self): 360 | """Corpus size. 361 | 362 | Returns: 363 | len(self.__sentence) : corpus size (#sentences) 364 | """ 365 | return len(self.__sentence) 366 | 367 | def __getitem__ (self, index): 368 | """Pick up a sentence in the corpus 369 | 370 | Argument: 371 | index : index of the sentence to pick up 372 | 373 | Returns: 374 | self.__sentence[index] : (index+1)-th sentence in the corpus 375 | 376 | Raises: 377 | IndexError : index exceeds the size of the corpus 378 | """ 379 | if len(self.__sentence)-1 < index: 380 | raise IndexError ( "Invalid index " + str(index) + " for list of " + str(len(self.__sentence)) + " sentences" ) 381 | else: 382 | return self.__sentence[index] 383 | 384 | ### 385 | ### wrapper function for output 386 | ### 387 | def outputRIBES (options, args, file=sys.stdout): 388 | # print start time 389 | print ("# RIBES evaluation start at " + str(datetime.datetime.today()), file=sys.stderr) 390 | 391 | # initialize "RIBESevaluator" instance 392 | evaluator = RIBESevaluator (sent=options.sent, alpha=options.alpha, beta=options.beta, output=file) 393 | 394 | # REFS : list of "Corpus" instance (for multi reference) 395 | REFS = [] 396 | 397 | for _ref in options.ref: 398 | if debug > 0: 399 | # print reference file name (if debug > 0) 400 | print ("# reference file [" + str(len(REFS)) + "] : " + _ref, file=file) 401 | 402 | # read multi references, construct and store "Corpus" instance 403 | REFS.append( Corpus(_ref, case=options.case) ) 404 | 405 | for i in range(len(args)): 406 | if debug > 0: 407 | # print system output file name (if debug > 0) 408 | print ("# system output file [" + str(i) + "] : " + args[i], file=file) 409 | 410 | # read system output and construct "Corpus" instance 411 | result = Corpus(args[i], case=options.case) 412 | 413 | # evaluate by RIBES -- "best_ribes" stands for the best score by multi-references, RIBESs stands for the score list for each references 414 | best_ribes = evaluator.eval (result, REFS, emptyref=options.emptyref) 415 | 416 | # print resutls 417 | print ("%.6f alpha=%f beta=%f %s" % (best_ribes, options.alpha, options.beta, args[i]), file=file) 418 | 419 | # print end time 420 | print ("# RIBES evaluation done at " + str(datetime.datetime.today()), file=sys.stderr) 421 | 422 | 423 | ### 424 | ### main function 425 | ### 426 | def main (): 427 | # variable "debug" is global... 428 | global debug 429 | 430 | usage = "%prog [options] system_outputs" 431 | optparser = OptionParser(usage) 432 | 433 | ### option definitions 434 | # -d/--debug : debug level (0: scores and start/end time, 1: +ref/hyp files) 435 | optparser.add_option("-d", "--debug", dest="debug", default=0, type="int", help="debug level", metavar="INT") 436 | 437 | # -r/--ref : reference (multiple references available, repeat "-r REF" in arguments) 438 | optparser.add_option("-r", "--ref", dest="ref", default=[], action="append", type="string", help="reference translation file (use multiple \"-r REF\" for multi-references)", metavar="FILE") 439 | 440 | # -c/--case : preserve uppercase letters 441 | optparser.add_option("-c", "--case", dest="case", default=False, action="store_true", help="preserve uppercase letters in evaluation (default: False -- lowercasing all words)") 442 | 443 | # -s/--sentence : show scores for every sentences 444 | optparser.add_option("-s", "--sentence", dest="sent", default=False, action="store_true", help="output scores for every sentences") 445 | 446 | # -a/--alpha : "Unigram Precison" to the {alpha}-th power 447 | optparser.add_option("-a", "--alpha", dest="alpha", default=0.25, type="float", help="hyperparameter alpha (default=0.25)", metavar="FLOAT") 448 | 449 | # -b/--beta : "Brevity Penalty" to the {beta}-th power 450 | optparser.add_option("-b", "--beta", dest="beta", default=0.10, type="float", help="hyperparameter beta (default=0.10)", metavar="FLOAT") 451 | 452 | # -o/--output : output file 453 | optparser.add_option("-o", "--output", dest="output", default="", type="string", help="log output file", metavar="FILE") 454 | 455 | # -z/--emptyref : allow empty reference translations (ignored in the evaluation) 456 | optparser.add_option("-z", "--emptyref", dest="emptyref", default=False, action="store_true", help="allow empty reference translations (default: False -- raise RuntimeError in that case)") 457 | 458 | # args : system outputs 459 | 460 | # parse options 461 | (options, args) = optparser.parse_args() 462 | 463 | # set debug level (global) 464 | debug = options.debug 465 | 466 | if len(options.output) == 0: 467 | # output to stdout 468 | outputRIBES (options, args) 469 | else: 470 | # output file is automatically closed ... 471 | with open (options.output, 'w') as ofp: 472 | outputRIBES (options, args, file=ofp) 473 | 474 | 475 | 476 | if __name__ == "__main__": 477 | try: 478 | main() 479 | except Exception as err: 480 | traceback.print_exc(file=sys.stderr) 481 | sys.exit(255) 482 | --------------------------------------------------------------------------------