├── .gitignore ├── CMakeLists.txt ├── LICENSE.txt ├── README.md ├── cmake └── FindSparseHash.cmake └── src ├── alignment_io.cc ├── alignment_io.h ├── array2d.h ├── atools.cc ├── corpus.h ├── da.h ├── fast_align.cc ├── force_align.py ├── hashtables.h ├── port.h ├── ttables.cc └── ttables.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | fast_align 6 | Makefile 7 | build/ 8 | 9 | # Compiled Dynamic libraries 10 | *.so 11 | *.dylib 12 | 13 | # Compiled Static libraries 14 | *.lai 15 | *.la 16 | *.a 17 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(fast_align) 2 | cmake_minimum_required(VERSION 2.8 FATAL_ERROR) 3 | set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++11 -O3 -g") 5 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 6 | 7 | find_package(SparseHash) 8 | if(SPARSEHASH_FOUND) 9 | add_definitions(-DHAVE_SPARSEHASH) 10 | endif(SPARSEHASH_FOUND) 11 | 12 | find_package(OpenMP QUIET) 13 | if (OPENMP_FOUND) 14 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 15 | endif(OPENMP_FOUND) 16 | 17 | add_executable(fast_align src/fast_align.cc src/ttables.cc) 18 | add_executable(atools src/alignment_io.cc src/atools.cc) 19 | configure_file(src/force_align.py force_align.py COPYONLY) 20 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | fast_align 2 | ========== 3 | 4 | `fast_align` is a simple, fast, unsupervised word aligner. 5 | 6 | If you use this software, please cite: 7 | * [Chris Dyer](http://www.cs.cmu.edu/~cdyer), [Victor Chahuneau](http://victor.chahuneau.fr), and [Noah A. Smith](http://www.cs.cmu.edu/~nasmith). (2013). [A Simple, Fast, and Effective Reparameterization of IBM Model 2](http://www.ark.cs.cmu.edu/cdyer/fast_valign.pdf). In *Proc. of NAACL*. 8 | 9 | The source code in this repository is provided under the terms of the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 10 | 11 | ## Input format 12 | 13 | Input to `fast_align` must be tokenized and aligned into parallel sentences. Each line is a source language sentence and its target language translation, separated by a triple pipe symbol with leading and trailing white space (` ||| `). An example 3-sentence German–English parallel corpus is: 14 | 15 | doch jetzt ist der Held gefallen . ||| but now the hero has fallen . 16 | neue Modelle werden erprobt . ||| new models are being tested . 17 | doch fehlen uns neue Ressourcen . ||| but we lack new resources . 18 | 19 | ## Compiling and using `fast_align` 20 | 21 | Building `fast_align` requires a modern C++ compiler and the [CMake]() build system. Additionally, the following libraries can be used to obtain better performance 22 | 23 | * OpenMP (included with some compilers, such as GCC) 24 | * libtcmalloc (part of Google's perftools) 25 | * libsparsehash 26 | 27 | To install these on Ubuntu: 28 | 29 | sudo apt-get install libgoogle-perftools-dev libsparsehash-dev 30 | 31 | To compile, do the following 32 | 33 | mkdir build 34 | cd build 35 | cmake .. 36 | make 37 | 38 | Run `fast_align` to see a list of command line options. 39 | 40 | `fast_align` generates *asymmetric* alignments (i.e., by treating either the left or right language in the parallel corpus as primary language being modeled, slightly different alignments will be generated). The usually recommended way to generate *source–target* (left language–right language) alignments is: 41 | 42 | ./fast_align -i text.fr-en -d -o -v > forward.align 43 | 44 | The usually recommended way to generate *target–source* alignments is to just add the `-r` (“reverse”) option: 45 | 46 | ./fast_align -i text.fr-en -d -o -v -r > reverse.align 47 | 48 | These can be symmetrized using the included `atools` command using a variety of standard symmetrization heuristics, for example: 49 | 50 | ./atools -i forward.align -j reverse.align -c grow-diag-final-and 51 | 52 | ## Output 53 | 54 | `fast_align` produces outputs in the widely-used `i-j` “Pharaoh format,” where a pair `i-j` indicates that the ith word (zero-indexed) of the left language (by convention, the *source* language) is aligned to the jth word of the right sentence (by convention, the *target* language). For example, a good alignment of the above German–English corpus would be: 55 | 56 | 0-0 1-1 2-4 3-2 4-3 5-5 6-6 57 | 0-0 1-1 2-2 2-3 3-4 4-5 58 | 0-0 1-2 2-1 3-3 4-4 5-5 59 | 60 | ## Acknowledgements 61 | 62 | The development of this software was sponsored in part by the U.S. Army Research Laboratory and the U.S. Army Research Office under contract/grant number W911NF-10-1-0533. 63 | 64 | -------------------------------------------------------------------------------- /cmake/FindSparseHash.cmake: -------------------------------------------------------------------------------- 1 | if(SPARSEHASH_INCLUDE_DIR) 2 | set(SPARSEHASH_FIND_QUIETLY TRUE) 3 | endif(SPARSEHASH_INCLUDE_DIR) 4 | 5 | find_path(SPARSEHASH_INCLUDE_DIR google/sparse_hash_map) 6 | 7 | # handle the QUIETLY and REQUIRED arguments and set SPARSEHASH_FOUND to TRUE if 8 | # all listed variables are TRUE 9 | include(FindPackageHandleStandardArgs) 10 | find_package_handle_standard_args(SparseHash DEFAULT_MSG SPARSEHASH_INCLUDE_DIR) 11 | 12 | mark_as_advanced(SPARSEHASH_INCLUDE_DIR) 13 | -------------------------------------------------------------------------------- /src/alignment_io.cc: -------------------------------------------------------------------------------- 1 | #include "src/alignment_io.h" 2 | 3 | using namespace std; 4 | 5 | static bool is_digit(char x) { return x >= '0' && x <= '9'; } 6 | 7 | std::shared_ptr > AlignmentIO::ReadPharaohAlignmentGrid(const string& al) { 8 | int max_x = 0; 9 | int max_y = 0; 10 | unsigned i = 0; 11 | size_t pos = al.rfind(" ||| "); 12 | if (pos != string::npos) { i = pos + 5; } 13 | while (i < al.size()) { 14 | if (al[i] == '\n' || al[i] == '\r') break; 15 | int x = 0; 16 | while(i < al.size() && is_digit(al[i])) { 17 | x *= 10; 18 | x += al[i] - '0'; 19 | ++i; 20 | } 21 | if (x > max_x) max_x = x; 22 | assert(i < al.size()); 23 | if(al[i] != '-') { 24 | cerr << "BAD ALIGNMENT: " << al << endl; 25 | abort(); 26 | } 27 | ++i; 28 | int y = 0; 29 | while(i < al.size() && is_digit(al[i])) { 30 | y *= 10; 31 | y += al[i] - '0'; 32 | ++i; 33 | } 34 | if (y > max_y) max_y = y; 35 | while(i < al.size() && al[i] == ' ') { ++i; } 36 | } 37 | 38 | std::shared_ptr > grid(new Array2D(max_x + 1, max_y + 1)); 39 | i = 0; 40 | if (pos != string::npos) { i = pos + 5; } 41 | while (i < al.size()) { 42 | if (al[i] == '\n' || al[i] == '\r') break; 43 | int x = 0; 44 | while(i < al.size() && is_digit(al[i])) { 45 | x *= 10; 46 | x += al[i] - '0'; 47 | ++i; 48 | } 49 | assert(i < al.size()); 50 | assert(al[i] == '-'); 51 | ++i; 52 | int y = 0; 53 | while(i < al.size() && is_digit(al[i])) { 54 | y *= 10; 55 | y += al[i] - '0'; 56 | ++i; 57 | } 58 | (*grid)(x, y) = true; 59 | while(i < al.size() && al[i] == ' ') { ++i; } 60 | } 61 | // cerr << *grid << endl; 62 | return grid; 63 | } 64 | 65 | void AlignmentIO::SerializePharaohFormat(const Array2D& alignment, ostream* o) { 66 | ostream& out = *o; 67 | bool need_space = false; 68 | for (unsigned i = 0; i < alignment.width(); ++i) 69 | for (unsigned j = 0; j < alignment.height(); ++j) 70 | if (alignment(i,j)) { 71 | if (need_space) out << ' '; else need_space = true; 72 | out << i << '-' << j; 73 | } 74 | out << endl; 75 | } 76 | 77 | void AlignmentIO::SerializeTypedAlignment(const Array2D& alignment, ostream* o) { 78 | ostream& out = *o; 79 | bool need_space = false; 80 | for (unsigned i = 0; i < alignment.width(); ++i) 81 | for (unsigned j = 0; j < alignment.height(); ++j) { 82 | const AlignmentType& aij = alignment(i,j); 83 | if (aij != kNONE) { 84 | if (need_space) out << ' '; else need_space = true; 85 | if (aij == kTRANSLATION) {} 86 | else if (aij == kTRANSLITERATION) { 87 | out << 'T' << ':'; 88 | } else { 89 | cerr << "\nUnexpected alignment point type: " << static_cast(aij) << endl; 90 | abort(); 91 | } 92 | out << i << '-' << j; 93 | } 94 | } 95 | out << endl; 96 | } 97 | 98 | -------------------------------------------------------------------------------- /src/alignment_io.h: -------------------------------------------------------------------------------- 1 | #ifndef ALIGNMENT_IO_H_ 2 | #define ALIGNMENT_IO_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "array2d.h" 8 | 9 | struct AlignmentIO { 10 | enum AlignmentType { kNONE = 0, kTRANSLATION = 1, kTRANSLITERATION = 2 }; 11 | 12 | static std::shared_ptr > ReadPharaohAlignmentGrid(const std::string& al); 13 | static void SerializePharaohFormat(const Array2D& alignment, std::ostream* out); 14 | static void SerializeTypedAlignment(const Array2D& alignment, std::ostream* out); 15 | }; 16 | 17 | inline std::ostream& operator<<(std::ostream& os, const Array2D& m) { 18 | os << ' '; 19 | for (unsigned j=0; j 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | template 11 | class Array2D { 12 | public: 13 | typedef typename std::vector::reference reference; 14 | typedef typename std::vector::const_reference const_reference; 15 | typedef typename std::vector::iterator iterator; 16 | typedef typename std::vector::const_iterator const_iterator; 17 | Array2D() : width_(0), height_(0) {} 18 | Array2D(unsigned w, unsigned h, const T& d = T()) : 19 | width_(w), height_(h), data_(w*h, d) {} 20 | Array2D(const Array2D& rhs) : 21 | width_(rhs.width_), height_(rhs.height_), data_(rhs.data_) {} 22 | bool empty() const { return data_.empty(); } 23 | void resize(unsigned w, unsigned h, const T& d = T()) { 24 | data_.resize(w * h, d); 25 | width_ = w; 26 | height_ = h; 27 | } 28 | const Array2D& operator=(const Array2D& rhs) { 29 | data_ = rhs.data_; 30 | width_ = rhs.width_; 31 | height_ = rhs.height_; 32 | return *this; 33 | } 34 | void fill(const T& v) { data_.assign(data_.size(), v); } 35 | unsigned width() const { return width_; } 36 | unsigned height() const { return height_; } 37 | reference operator()(unsigned i, unsigned j) { 38 | return data_[offset(i, j)]; 39 | } 40 | void clear() { data_.clear(); width_=0; height_=0; } 41 | const_reference operator()(unsigned i, unsigned j) const { 42 | return data_[offset(i, j)]; 43 | } 44 | iterator begin_col(unsigned j) { 45 | return data_.begin() + offset(0,j); 46 | } 47 | const_iterator begin_col(unsigned j) const { 48 | return data_.begin() + offset(0,j); 49 | } 50 | iterator end_col(unsigned j) { 51 | return data_.begin() + offset(0,j) + width_; 52 | } 53 | const_iterator end_col(unsigned j) const { 54 | return data_.begin() + offset(0,j) + width_; 55 | } 56 | iterator end() { return data_.end(); } 57 | const_iterator end() const { return data_.end(); } 58 | const Array2D& operator*=(const T& x) { 59 | std::transform(data_.begin(), data_.end(), data_.begin(), 60 | std::bind2nd(std::multiplies(), x)); 61 | } 62 | const Array2D& operator/=(const T& x) { 63 | std::transform(data_.begin(), data_.end(), data_.begin(), 64 | std::bind2nd(std::divides(), x)); 65 | } 66 | const Array2D& operator+=(const Array2D& m) { 67 | std::transform(m.data_.begin(), m.data_.end(), data_.begin(), data_.begin(), std::plus()); 68 | } 69 | const Array2D& operator-=(const Array2D& m) { 70 | std::transform(m.data_.begin(), m.data_.end(), data_.begin(), data_.begin(), std::minus()); 71 | } 72 | 73 | private: 74 | inline unsigned offset(unsigned i, unsigned j) const { 75 | assert(i data_; 84 | }; 85 | 86 | template 87 | Array2D operator*(const Array2D& l, const T& scalar) { 88 | Array2D res(l); 89 | res *= scalar; 90 | return res; 91 | } 92 | 93 | template 94 | Array2D operator*(const T& scalar, const Array2D& l) { 95 | Array2D res(l); 96 | res *= scalar; 97 | return res; 98 | } 99 | 100 | template 101 | Array2D operator/(const Array2D& l, const T& scalar) { 102 | Array2D res(l); 103 | res /= scalar; 104 | return res; 105 | } 106 | 107 | template 108 | Array2D operator+(const Array2D& l, const Array2D& r) { 109 | Array2D res(l); 110 | res += r; 111 | return res; 112 | } 113 | 114 | template 115 | Array2D operator-(const Array2D& l, const Array2D& r) { 116 | Array2D res(l); 117 | res -= r; 118 | return res; 119 | } 120 | 121 | template 122 | inline std::ostream& operator<<(std::ostream& os, const Array2D& m) { 123 | for (unsigned i=0; i& m) { 132 | os << ' '; 133 | for (unsigned j=0; j >& m) { 150 | os << ' '; 151 | for (unsigned j=0; j& ar = m(i,j); 158 | for (unsigned k=0; k 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "alignment_io.h" 12 | 13 | using namespace std; 14 | 15 | struct option options[] = { 16 | {"input_1", required_argument, 0, 'i'}, 17 | {"input_2", required_argument, 0, 'j'}, 18 | {"command", required_argument, 0, 'c'}, 19 | {0,0,0,0} 20 | }; 21 | 22 | string input_1; 23 | string input_2; 24 | string command; 25 | 26 | bool InitCommandLine(int argc, char** argv) { 27 | while (1) { 28 | int oi; 29 | int c = getopt_long(argc, argv, "i:j:c:", options, &oi); 30 | if (c == -1) break; 31 | switch(c) { 32 | case 'i': input_1 = optarg; break; 33 | case 'j': input_2 = optarg; break; 34 | case 'c': command = optarg; break; 35 | default: return false; 36 | } 37 | } 38 | if (input_1.size() == 0) return false; 39 | if (command.size() == 0) return false; 40 | return true; 41 | } 42 | 43 | struct Command { 44 | virtual ~Command() {} 45 | virtual string Name() const = 0; 46 | 47 | // returns 1 for alignment grid output [default] 48 | // returns 2 if Summary() should be called [for AER, etc] 49 | virtual int Result() const { return 1; } 50 | 51 | virtual bool RequiresTwoOperands() const { return true; } 52 | virtual void Apply(const Array2D& a, const Array2D& b, Array2D* x) = 0; 53 | void EnsureSize(const Array2D& a, const Array2D& b, Array2D* x) { 54 | x->resize(max(a.width(), b.width()), max(a.height(), b.height())); 55 | } 56 | static bool Safe(const Array2D& a, int i, int j) { 57 | if (i >= 0 && j >= 0 && i < static_cast(a.width()) && j < static_cast(a.height())) 58 | return a(i,j); 59 | else 60 | return false; 61 | } 62 | virtual void Summary() { assert(!"Summary should have been overridden"); } 63 | }; 64 | 65 | // compute fmeasure, second alignment is reference, first is hyp 66 | struct FMeasureCommand : public Command { 67 | FMeasureCommand() : matches(), num_predicted(), num_in_ref() {} 68 | int Result() const { return 2; } 69 | string Name() const { return "fmeasure"; } 70 | bool RequiresTwoOperands() const { return true; } 71 | void Apply(const Array2D& hyp, const Array2D& ref, Array2D* x) { 72 | (void) x; // AER just computes statistics, not an alignment 73 | unsigned i_len = ref.width(); 74 | unsigned j_len = ref.height(); 75 | for (unsigned i = 0; i < i_len; ++i) { 76 | for (unsigned j = 0; j < j_len; ++j) { 77 | if (ref(i,j)) { 78 | ++num_in_ref; 79 | if (Safe(hyp, i, j)) ++matches; 80 | } 81 | } 82 | } 83 | for (unsigned i = 0; i < hyp.width(); ++i) 84 | for (unsigned j = 0; j < hyp.height(); ++j) 85 | if (hyp(i,j)) ++num_predicted; 86 | } 87 | void Summary() { 88 | if (num_predicted == 0 || num_in_ref == 0) { 89 | cerr << "Insufficient statistics to compute f-measure!\n"; 90 | abort(); 91 | } 92 | const double prec = static_cast(matches) / num_predicted; 93 | const double rec = static_cast(matches) / num_in_ref; 94 | cout << "P: " << prec << endl; 95 | cout << "R: " << rec << endl; 96 | const double f = (2.0 * prec * rec) / (rec + prec); 97 | cout << "F: " << f << endl; 98 | } 99 | int matches; 100 | int num_predicted; 101 | int num_in_ref; 102 | }; 103 | 104 | struct DisplayCommand : public Command { 105 | string Name() const { return "display"; } 106 | bool RequiresTwoOperands() const { return false; } 107 | void Apply(const Array2D& in, const Array2D&, Array2D* x) { 108 | *x = in; 109 | cout << *x << endl; 110 | } 111 | }; 112 | 113 | struct ConvertCommand : public Command { 114 | string Name() const { return "convert"; } 115 | bool RequiresTwoOperands() const { return false; } 116 | void Apply(const Array2D& in, const Array2D&, Array2D* x) { 117 | *x = in; 118 | } 119 | }; 120 | 121 | struct InvertCommand : public Command { 122 | string Name() const { return "invert"; } 123 | bool RequiresTwoOperands() const { return false; } 124 | void Apply(const Array2D& in, const Array2D&, Array2D* x) { 125 | Array2D& res = *x; 126 | res.resize(in.height(), in.width()); 127 | for (unsigned i = 0; i < in.height(); ++i) 128 | for (unsigned j = 0; j < in.width(); ++j) 129 | res(i, j) = in(j, i); 130 | } 131 | }; 132 | 133 | struct IntersectCommand : public Command { 134 | string Name() const { return "intersect"; } 135 | bool RequiresTwoOperands() const { return true; } 136 | void Apply(const Array2D& a, const Array2D& b, Array2D* x) { 137 | EnsureSize(a, b, x); 138 | Array2D& res = *x; 139 | for (unsigned i = 0; i < a.width(); ++i) 140 | for (unsigned j = 0; j < a.height(); ++j) 141 | res(i, j) = Safe(a, i, j) && Safe(b, i, j); 142 | } 143 | }; 144 | 145 | struct UnionCommand : public Command { 146 | string Name() const { return "union"; } 147 | bool RequiresTwoOperands() const { return true; } 148 | void Apply(const Array2D& a, const Array2D& b, Array2D* x) { 149 | EnsureSize(a, b, x); 150 | Array2D& res = *x; 151 | for (unsigned i = 0; i < res.width(); ++i) 152 | for (unsigned j = 0; j < res.height(); ++j) 153 | res(i, j) = Safe(a, i, j) || Safe(b, i, j); 154 | } 155 | }; 156 | 157 | struct RefineCommand : public Command { 158 | RefineCommand() { 159 | neighbors_.push_back(make_pair(1,0)); 160 | neighbors_.push_back(make_pair(-1,0)); 161 | neighbors_.push_back(make_pair(0,1)); 162 | neighbors_.push_back(make_pair(0,-1)); 163 | } 164 | bool RequiresTwoOperands() const { return true; } 165 | 166 | void Align(unsigned i, unsigned j) { 167 | res_(i, j) = true; 168 | is_i_aligned_[i] = true; 169 | is_j_aligned_[j] = true; 170 | } 171 | 172 | bool IsNeighborAligned(int i, int j) const { 173 | for (unsigned k = 0; k < neighbors_.size(); ++k) { 174 | const int di = neighbors_[k].first; 175 | const int dj = neighbors_[k].second; 176 | if (Safe(res_, i + di, j + dj)) 177 | return true; 178 | } 179 | return false; 180 | } 181 | 182 | bool IsNeitherAligned(int i, int j) const { 183 | return !(is_i_aligned_[i] || is_j_aligned_[j]); 184 | } 185 | 186 | bool IsOneOrBothUnaligned(int i, int j) const { 187 | return !(is_i_aligned_[i] && is_j_aligned_[j]); 188 | } 189 | 190 | bool KoehnAligned(int i, int j) const { 191 | return IsOneOrBothUnaligned(i, j) && IsNeighborAligned(i, j); 192 | } 193 | 194 | typedef bool (RefineCommand::*Predicate)(int i, int j) const; 195 | 196 | protected: 197 | void InitRefine( 198 | const Array2D& a, 199 | const Array2D& b) { 200 | res_.clear(); 201 | EnsureSize(a, b, &res_); 202 | in_.clear(); un_.clear(); is_i_aligned_.clear(); is_j_aligned_.clear(); 203 | EnsureSize(a, b, &in_); 204 | EnsureSize(a, b, &un_); 205 | is_i_aligned_.resize(res_.width(), false); 206 | is_j_aligned_.resize(res_.height(), false); 207 | for (unsigned i = 0; i < in_.width(); ++i) 208 | for (unsigned j = 0; j < in_.height(); ++j) { 209 | un_(i, j) = Safe(a, i, j) || Safe(b, i, j); 210 | in_(i, j) = Safe(a, i, j) && Safe(b, i, j); 211 | if (in_(i, j)) Align(i, j); 212 | } 213 | } 214 | // "grow" the resulting alignment using the points in adds 215 | // if they match the constraints determined by pred 216 | void Grow(Predicate pred, bool idempotent, const Array2D& adds) { 217 | if (idempotent) { 218 | for (unsigned i = 0; i < adds.width(); ++i) 219 | for (unsigned j = 0; j < adds.height(); ++j) { 220 | if (adds(i, j) && !res_(i, j) && 221 | (this->*pred)(i, j)) Align(i, j); 222 | } 223 | return; 224 | } 225 | set > p; 226 | for (unsigned i = 0; i < adds.width(); ++i) 227 | for (unsigned j = 0; j < adds.height(); ++j) 228 | if (adds(i, j) && !res_(i, j)) 229 | p.insert(make_pair(i, j)); 230 | bool keep_going = !p.empty(); 231 | while (keep_going) { 232 | keep_going = false; 233 | set > added; 234 | for (set >::iterator pi = p.begin(); pi != p.end(); ++pi) { 235 | if ((this->*pred)(pi->first, pi->second)) { 236 | Align(pi->first, pi->second); 237 | added.insert(make_pair(pi->first, pi->second)); 238 | keep_going = true; 239 | } 240 | } 241 | for (set >::iterator ai = added.begin(); ai != added.end(); ++ai) 242 | p.erase(*ai); 243 | } 244 | } 245 | Array2D res_; // refined alignment 246 | Array2D in_; // intersection alignment 247 | Array2D un_; // union alignment 248 | vector is_i_aligned_; 249 | vector is_j_aligned_; 250 | vector > neighbors_; 251 | }; 252 | 253 | struct DiagCommand : public RefineCommand { 254 | DiagCommand() { 255 | neighbors_.push_back(make_pair(1,1)); 256 | neighbors_.push_back(make_pair(-1,1)); 257 | neighbors_.push_back(make_pair(1,-1)); 258 | neighbors_.push_back(make_pair(-1,-1)); 259 | } 260 | }; 261 | 262 | struct GDCommand : public DiagCommand { 263 | string Name() const { return "grow-diag"; } 264 | void Apply(const Array2D& a, const Array2D& b, Array2D* x) { 265 | InitRefine(a, b); 266 | Grow(&RefineCommand::KoehnAligned, false, un_); 267 | *x = res_; 268 | } 269 | }; 270 | 271 | struct GDFCommand : public DiagCommand { 272 | string Name() const { return "grow-diag-final"; } 273 | void Apply(const Array2D& a, const Array2D& b, Array2D* x) { 274 | InitRefine(a, b); 275 | Grow(&RefineCommand::KoehnAligned, false, un_); 276 | Grow(&RefineCommand::IsOneOrBothUnaligned, true, a); 277 | Grow(&RefineCommand::IsOneOrBothUnaligned, true, b); 278 | *x = res_; 279 | } 280 | }; 281 | 282 | struct GDFACommand : public DiagCommand { 283 | string Name() const { return "grow-diag-final-and"; } 284 | void Apply(const Array2D& a, const Array2D& b, Array2D* x) { 285 | InitRefine(a, b); 286 | Grow(&RefineCommand::KoehnAligned, false, un_); 287 | Grow(&RefineCommand::IsNeitherAligned, true, a); 288 | Grow(&RefineCommand::IsNeitherAligned, true, b); 289 | *x = res_; 290 | } 291 | }; 292 | 293 | map > commands; 294 | 295 | template static void AddCommand() { 296 | C* c = new C; 297 | commands[c->Name()].reset(c); 298 | } 299 | 300 | int main(int argc, char **argv) { 301 | AddCommand(); 302 | AddCommand(); 303 | AddCommand(); 304 | AddCommand(); 305 | AddCommand(); 306 | AddCommand(); 307 | AddCommand(); 308 | AddCommand(); 309 | AddCommand(); 310 | if (!InitCommandLine(argc, argv)) { 311 | cerr << "Usage: " << argv[0] << " -c COMMAND -i FILE1.AL [-j FILE2.AL]\n"; 312 | cerr << "Valid options for COMMAND:"; 313 | for (auto it : commands) 314 | cerr << ' ' << it.first; 315 | cerr << endl; 316 | return 1; 317 | } 318 | if (commands.count(command) == 0) { 319 | cerr << "Don't understand command: " << command << endl; 320 | return 1; 321 | } 322 | if (commands[command]->RequiresTwoOperands()) { 323 | if (input_2.size() == 0) { 324 | cerr << "Command '" << command << "' requires two alignment files\n"; 325 | return 1; 326 | } 327 | } else { 328 | if (input_2.size() != 0) { 329 | cerr << "Command '" << command << "' requires only one alignment file\n"; 330 | return 1; 331 | } 332 | } 333 | Command& cmd = *commands[command]; 334 | istream* in1 = NULL; 335 | if (input_1 == "-") { 336 | in1 = &cin; 337 | } else { 338 | in1 = new ifstream(input_1.c_str()); 339 | } 340 | istream* in2 = NULL; 341 | if (cmd.RequiresTwoOperands()) { 342 | if (input_2 == "-") { 343 | in2 = &cin; 344 | } else { 345 | in2 = new ifstream(input_2.c_str()); 346 | } 347 | } 348 | string line1; 349 | string line2; 350 | while(*in1) { 351 | getline(*in1, line1); 352 | if (in2) { 353 | getline(*in2, line2); 354 | if ((*in1 && !*in2) || (*in2 && !*in1)) { 355 | cerr << "Mismatched number of lines!\n"; 356 | exit(1); 357 | } 358 | } 359 | if (line1.empty() && !*in1) break; 360 | shared_ptr > out(new Array2D); 361 | shared_ptr > a1 = AlignmentIO::ReadPharaohAlignmentGrid(line1); 362 | if (in2) { 363 | shared_ptr > a2 = AlignmentIO::ReadPharaohAlignmentGrid(line2); 364 | cmd.Apply(*a1, *a2, out.get()); 365 | } else { 366 | Array2D dummy; 367 | cmd.Apply(*a1, dummy, out.get()); 368 | } 369 | 370 | if (cmd.Result() == 1) { 371 | AlignmentIO::SerializePharaohFormat(*out, &cout); 372 | } 373 | } 374 | if (cmd.Result() == 2) 375 | cmd.Summary(); 376 | return 0; 377 | } 378 | 379 | -------------------------------------------------------------------------------- /src/corpus.h: -------------------------------------------------------------------------------- 1 | #ifndef CPYPDICT_H_ 2 | #define CPYPDICT_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "src/hashtables.h" 11 | #include "src/port.h" 12 | 13 | class Dict { 14 | public: 15 | Dict() : b0_("") { 16 | words_.reserve(1000); 17 | } 18 | 19 | inline unsigned max() const { return words_.size(); } 20 | 21 | static bool is_ws(char x) { 22 | return (x == ' ' || x == '\t'); 23 | } 24 | 25 | inline void ConvertWhitespaceDelimitedLine( 26 | const std::string& line, 27 | const unsigned separator_id, 28 | std::vector* out) { 29 | size_t cur = 0; 30 | size_t last = 0; 31 | int state = 0; 32 | out->clear(); 33 | while(cur < line.size()) { 34 | const char cur_char = line[cur++]; 35 | if (is_ws(cur_char)) { 36 | if (state == 1) { 37 | out->push_back(Convert(line.substr(last, cur - last - 1))); 38 | state = 0; 39 | } 40 | if (cur_char == '\t') out->push_back(separator_id); 41 | } else { 42 | if (state == 1) continue; 43 | last = cur - 1; 44 | state = 1; 45 | } 46 | } 47 | if (state == 1) 48 | out->push_back(Convert(line.substr(last, cur - last))); 49 | } 50 | 51 | inline unsigned Convert(const std::string& word, bool frozen = false) { 52 | MAP_TYPE::iterator i = d_.find(word); 53 | if (i == d_.end()) { 54 | if (frozen) 55 | return 0; 56 | words_.push_back(word); 57 | d_[word] = words_.size(); 58 | return words_.size(); 59 | } else { 60 | return i->second; 61 | } 62 | } 63 | 64 | inline const std::string& Convert(const unsigned id) const { 65 | if (id == 0) return b0_; 66 | return words_[id-1]; 67 | } 68 | private: 69 | std::string b0_; 70 | std::vector words_; 71 | MAP_TYPE d_; 72 | }; 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/da.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 by Chris Dyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #ifndef _DA_H_ 16 | #define _DA_H_ 17 | 18 | #include 19 | #include 20 | 21 | // m = trg len 22 | // n = src len 23 | // i = trg index 24 | // j = src index 25 | struct DiagonalAlignment { 26 | 27 | static double UnnormalizedProb(const unsigned i, const unsigned j, const unsigned m, const unsigned n, const double alpha) { 28 | #if 0 29 | assert(i > 0); 30 | assert(n > 0); 31 | assert(m >= i); 32 | assert(n >= j); 33 | #endif 34 | return exp(Feature(i, j, m, n) * alpha); 35 | } 36 | 37 | static double ComputeZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) { 38 | #if 0 39 | assert(i > 0); 40 | assert(n > 0); 41 | assert(m >= i); 42 | #endif 43 | const double split = double(i) * n / m; 44 | const unsigned floor = static_cast(split); 45 | unsigned ceil = floor + 1; 46 | const double ratio = exp(-alpha / n); 47 | const unsigned num_top = n - floor; 48 | double ezt = 0; 49 | double ezb = 0; 50 | if (num_top) 51 | ezt = UnnormalizedProb(i, ceil, m, n, alpha) * (1.0 - pow(ratio, num_top)) / (1.0 - ratio); 52 | if (floor) 53 | ezb = UnnormalizedProb(i, floor, m, n, alpha) * (1.0 - pow(ratio, floor)) / (1.0 - ratio); 54 | return ezb + ezt; 55 | } 56 | 57 | static double ComputeDLogZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) { 58 | const double z = ComputeZ(i, n, m, alpha); 59 | const double split = double(i) * n / m; 60 | const unsigned floor = static_cast(split); 61 | const unsigned ceil = floor + 1; 62 | const double ratio = exp(-alpha / n); 63 | const double d = -1.0 / n; 64 | const unsigned num_top = n - floor; 65 | double pct = 0; 66 | double pcb = 0; 67 | if (num_top) { 68 | pct = arithmetico_geometric_series(Feature(i, ceil, m, n), UnnormalizedProb(i, ceil, m, n, alpha), ratio, d, num_top); 69 | //cerr << "PCT = " << pct << endl; 70 | } 71 | if (floor) { 72 | pcb = arithmetico_geometric_series(Feature(i, floor, m, n), UnnormalizedProb(i, floor, m, n, alpha), ratio, d, floor); 73 | //cerr << "PCB = " << pcb << endl; 74 | } 75 | return (pct + pcb) / z; 76 | } 77 | 78 | inline static double Feature(const unsigned i, const unsigned j, const unsigned m, const unsigned n) { 79 | return -fabs(double(j) / n - double(i) / m); 80 | } 81 | 82 | private: 83 | inline static double arithmetico_geometric_series(const double a_1, const double g_1, const double r, const double d, const unsigned n) { 84 | const double g_np1 = g_1 * pow(r, n); 85 | const double a_n = d * (n - 1) + a_1; 86 | const double x_1 = a_1 * g_1; 87 | const double g_2 = g_1 * r; 88 | const double rm1 = r - 1; 89 | return (a_n * g_np1 - x_1) / rm1 - d*(g_np1 - g_2) / (rm1 * rm1); 90 | } 91 | }; 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /src/fast_align.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 by Chris Dyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "src/corpus.h" 25 | #include "src/ttables.h" 26 | #include "src/da.h" 27 | 28 | using namespace std; 29 | 30 | struct PairHash { 31 | size_t operator()(const pair& x) const { 32 | return (unsigned short) x.first << 16 | (unsigned) x.second; 33 | } 34 | }; 35 | 36 | Dict d; // integerization map 37 | 38 | void ParseLine(const string& line, 39 | vector* src, 40 | vector* trg) { 41 | static const unsigned kDIV = d.Convert("|||"); 42 | vector tmp; 43 | src->clear(); 44 | trg->clear(); 45 | d.ConvertWhitespaceDelimitedLine(line, kDIV, &tmp); 46 | unsigned i = 0; 47 | while (i < tmp.size() && tmp[i] != kDIV) { 48 | src->push_back(tmp[i]); 49 | ++i; 50 | } 51 | if (i < tmp.size() && tmp[i] == kDIV) { 52 | ++i; 53 | for (; i < tmp.size(); ++i) 54 | trg->push_back(tmp[i]); 55 | } 56 | } 57 | 58 | string input; 59 | string conditional_probability_filename = ""; 60 | string input_model_file = ""; 61 | double mean_srclen_multiplier = 1.0; 62 | int is_reverse = 0; 63 | int ITERATIONS = 5; 64 | int favor_diagonal = 0; 65 | double beam_threshold = -4.0; 66 | double prob_align_null = 0.08; 67 | double diagonal_tension = 4.0; 68 | int optimize_tension = 0; 69 | int variational_bayes = 0; 70 | double alpha = 0.01; 71 | int no_null_word = 0; 72 | size_t thread_buffer_size = 10000; 73 | bool force_align = false; 74 | int print_scores = 0; 75 | struct option options[] = { 76 | {"input", required_argument, 0, 'i'}, 77 | {"reverse", no_argument, &is_reverse, 1 }, 78 | {"iterations", required_argument, 0, 'I'}, 79 | {"favor_diagonal", no_argument, &favor_diagonal, 1 }, 80 | {"force_align", required_argument, 0, 'f'}, 81 | {"mean_srclen_multiplier", required_argument, 0, 'm'}, 82 | {"beam_threshold", required_argument, 0, 't'}, 83 | {"p0", required_argument, 0, 'q'}, 84 | {"diagonal_tension", required_argument, 0, 'T'}, 85 | {"optimize_tension", no_argument, &optimize_tension, 1 }, 86 | {"variational_bayes", no_argument, &variational_bayes, 1 }, 87 | {"alpha", required_argument, 0, 'a'}, 88 | {"no_null_word", no_argument, &no_null_word, 1 }, 89 | {"conditional_probabilities", required_argument, 0, 'p'}, 90 | {"thread_buffer_size", required_argument, 0, 'b'}, 91 | {0,0,0,0} 92 | }; 93 | 94 | bool InitCommandLine(int argc, char** argv) { 95 | while (1) { 96 | int oi; 97 | int c = getopt_long(argc, 98 | argv, 99 | "i:rI:df:m:t:q:T:ova:Np:b:s", 100 | options, 101 | &oi); 102 | if (c == -1) break; 103 | cerr << "ARG=" << (char)c << endl; 104 | switch(c) { 105 | case 'i': input = optarg; break; 106 | case 'r': is_reverse = 1; break; 107 | case 'I': ITERATIONS = atoi(optarg); break; 108 | case 'd': favor_diagonal = 1; break; 109 | case 'f': force_align = 1; conditional_probability_filename = optarg; break; 110 | case 'm': mean_srclen_multiplier = atof(optarg); break; 111 | case 't': beam_threshold = atof(optarg); break; 112 | case 'q': prob_align_null = atof(optarg); break; 113 | case 'T': favor_diagonal = 1; diagonal_tension = atof(optarg); break; 114 | case 'o': optimize_tension = 1; break; 115 | case 'v': variational_bayes = 1; break; 116 | case 'a': alpha = atof(optarg); break; 117 | case 'N': no_null_word = 1; break; 118 | case 'p': conditional_probability_filename = optarg; break; 119 | case 'b': thread_buffer_size = atoi(optarg); break; 120 | case 's': print_scores = 1; break; 121 | default: return false; 122 | } 123 | } 124 | if (input.size() == 0) return false; 125 | return true; 126 | } 127 | 128 | void UpdateFromPairs(const vector& lines, const int lc, const int iter, 129 | const bool final_iteration, const bool use_null, const unsigned kNULL, 130 | const double prob_align_not_null, double* c0, double* emp_feat, 131 | double* likelihood, TTable* s2t, vector* outputs) { 132 | if (final_iteration) { 133 | outputs->clear(); 134 | outputs->resize(lines.size()); 135 | } 136 | double emp_feat_ = 0.0; 137 | double c0_ = 0.0; 138 | double likelihood_ = 0.0; 139 | #pragma omp parallel for schedule(dynamic) reduction(+:emp_feat_,c0_,likelihood_) 140 | for (int line_idx = 0; line_idx < static_cast(lines.size()); 141 | ++line_idx) { 142 | vector src, trg; 143 | ParseLine(lines[line_idx], &src, &trg); 144 | if (is_reverse) 145 | swap(src, trg); 146 | if (src.size() == 0 || trg.size() == 0) { 147 | cerr << "Error in line " << lc << "\n" << lines[line_idx] << endl; 148 | //return 1; 149 | } 150 | ostringstream oss; // collect output in last iteration 151 | vector probs(src.size() + 1); 152 | bool first_al = true; // used when printing alignments 153 | double local_likelihood = 0.0; 154 | for (unsigned j = 0; j < trg.size(); ++j) { 155 | const unsigned& f_j = trg[j]; 156 | double sum = 0; 157 | double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1) 158 | if (use_null) { 159 | if (favor_diagonal) 160 | prob_a_i = prob_align_null; 161 | probs[0] = s2t->prob(kNULL, f_j) * prob_a_i; 162 | sum += probs[0]; 163 | } 164 | double az = 0; 165 | if (favor_diagonal) 166 | az = DiagonalAlignment::ComputeZ(j + 1, trg.size(), src.size(), 167 | diagonal_tension) / prob_align_not_null; 168 | for (unsigned i = 1; i <= src.size(); ++i) { 169 | if (favor_diagonal) 170 | prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), 171 | src.size(), diagonal_tension) / az; 172 | probs[i] = s2t->prob(src[i - 1], f_j) * prob_a_i; 173 | sum += probs[i]; 174 | } 175 | if (final_iteration) { 176 | double max_p = -1; 177 | int max_index = -1; 178 | if (use_null) { 179 | max_index = 0; 180 | max_p = probs[0]; 181 | } 182 | for (unsigned i = 1; i <= src.size(); ++i) { 183 | if (probs[i] > max_p) { 184 | max_index = i; 185 | max_p = probs[i]; 186 | } 187 | } 188 | if (max_index > 0) { 189 | if (first_al) 190 | first_al = false; 191 | else 192 | oss << ' '; 193 | if (is_reverse) 194 | oss << j << '-' << (max_index - 1); 195 | else 196 | oss << (max_index - 1) << '-' << j; 197 | } 198 | } else { 199 | if (use_null) { 200 | double count = probs[0] / sum; 201 | c0_ += count; 202 | s2t->Increment(kNULL, f_j, count); 203 | } 204 | for (unsigned i = 1; i <= src.size(); ++i) { 205 | const double p = probs[i] / sum; 206 | s2t->Increment(src[i - 1], f_j, p); 207 | emp_feat_ += DiagonalAlignment::Feature(j, i, trg.size(), src.size()) * p; 208 | } 209 | } 210 | local_likelihood += log(sum); 211 | } 212 | likelihood_ += local_likelihood; 213 | if (final_iteration) { 214 | if (print_scores) { 215 | double log_prob = Md::log_poisson(trg.size(), 0.05 + src.size() * mean_srclen_multiplier); 216 | log_prob += local_likelihood; 217 | oss << " ||| " << log_prob; 218 | } 219 | oss << endl; 220 | (*outputs)[line_idx] = oss.str(); 221 | } 222 | } 223 | *emp_feat += emp_feat_; 224 | *c0 += c0_; 225 | *likelihood += likelihood_; 226 | } 227 | 228 | inline void AddTranslationOptions(vector >& insert_buffer, 229 | TTable* s2t) { 230 | s2t->SetMaxE(insert_buffer.size()-1); 231 | #pragma omp parallel for schedule(dynamic) 232 | for (unsigned e = 0; e < insert_buffer.size(); ++e) { 233 | for (unsigned f : insert_buffer[e]) { 234 | s2t->Insert(e, f); 235 | } 236 | insert_buffer[e].clear(); 237 | } 238 | } 239 | 240 | void InitialPass(const unsigned kNULL, const bool use_null, TTable* s2t, 241 | double* n_target_tokens, double* tot_len_ratio, 242 | vector, unsigned>>* size_counts) { 243 | ifstream in(input.c_str()); 244 | if (!in) { 245 | cerr << "Can't read " << input << endl; 246 | } 247 | unordered_map, unsigned, PairHash> size_counts_; 248 | vector> insert_buffer; 249 | size_t insert_buffer_items = 0; 250 | vector src, trg; 251 | string line; 252 | bool flag = false; 253 | int lc = 0; 254 | cerr << "INITIAL PASS " << endl; 255 | while (true) { 256 | getline(in, line); 257 | if (!in) 258 | break; 259 | lc++; 260 | if (lc % 1000 == 0) { cerr << '.'; flag = true; } 261 | if (lc %50000 == 0) { cerr << " [" << lc << "]\n" << flush; flag = false; } 262 | ParseLine(line, &src, &trg); 263 | if (is_reverse) 264 | swap(src, trg); 265 | if (src.size() == 0 || trg.size() == 0) { 266 | cerr << "Error in line " << lc << "\n" << line << endl; 267 | } 268 | *tot_len_ratio += static_cast(trg.size()) / static_cast(src.size()); 269 | *n_target_tokens += trg.size(); 270 | if (use_null) { 271 | for (const unsigned f : trg) { 272 | s2t->Insert(kNULL, f); 273 | } 274 | } 275 | for (const unsigned e : src) { 276 | if (e >= insert_buffer.size()) { 277 | insert_buffer.resize(e+1); 278 | } 279 | for (const unsigned f : trg) { 280 | insert_buffer[e].push_back(f); 281 | } 282 | insert_buffer_items += trg.size(); 283 | } 284 | if (insert_buffer_items > thread_buffer_size * 100) { 285 | insert_buffer_items = 0; 286 | AddTranslationOptions(insert_buffer, s2t); 287 | } 288 | ++size_counts_[make_pair(trg.size(), src.size())]; 289 | } 290 | for (const auto& p : size_counts_) { 291 | size_counts->push_back(p); 292 | } 293 | AddTranslationOptions(insert_buffer, s2t); 294 | 295 | mean_srclen_multiplier = (*tot_len_ratio) / lc; 296 | if (flag) { 297 | cerr << endl; 298 | } 299 | cerr << "expected target length = source length * " << mean_srclen_multiplier << endl; 300 | } 301 | 302 | int main(int argc, char** argv) { 303 | if (!InitCommandLine(argc, argv)) { 304 | cerr << "Usage: " << argv[0] << " -i file.fr-en\n" 305 | << " Standard options ([USE] = strongly recommended):\n" 306 | << " -i: [REQ] Input parallel corpus\n" 307 | << " -v: [USE] Use Dirichlet prior on lexical translation distributions\n" 308 | << " -d: [USE] Favor alignment points close to the monotonic diagonoal\n" 309 | << " -o: [USE] Optimize how close to the diagonal alignment points should be\n" 310 | << " -r: Run alignment in reverse (condition on target and predict source)\n" 311 | << " -c: Output conditional probability table\n" 312 | << " Advanced options:\n" 313 | << " -I: number of iterations in EM training (default = 5)\n" 314 | << " -q: p_null parameter (default = 0.08)\n" 315 | << " -N: No null word\n" 316 | << " -a: alpha parameter for optional Dirichlet prior (default = 0.01)\n" 317 | << " -T: starting lambda for diagonal distance parameter (default = 4)\n" 318 | << " -s: print alignment scores (alignment ||| score, disabled by default)\n"; 319 | return 1; 320 | } 321 | const bool use_null = !no_null_word; 322 | if (variational_bayes && alpha <= 0.0) { 323 | cerr << "--alpha must be > 0\n"; 324 | return 1; 325 | } 326 | const double prob_align_not_null = 1.0 - prob_align_null; 327 | const unsigned kNULL = d.Convert(""); 328 | TTable s2t, t2s; 329 | vector, unsigned>> size_counts; 330 | double tot_len_ratio = 0; 331 | double n_target_tokens = 0; 332 | 333 | if (force_align) { 334 | ifstream in(conditional_probability_filename.c_str()); 335 | s2t.DeserializeLogProbsFromText(&in, d); 336 | ITERATIONS = 0; // don't do any learning 337 | } else { 338 | InitialPass(kNULL, use_null, &s2t, &n_target_tokens, &tot_len_ratio, &size_counts); 339 | s2t.Freeze(); 340 | } 341 | 342 | for (int iter = 0; iter < ITERATIONS; ++iter) { 343 | const bool final_iteration = (iter == (ITERATIONS - 1)); 344 | cerr << "ITERATION " << (iter + 1) << (final_iteration ? " (FINAL)" : "") << endl; 345 | ifstream in(input.c_str()); 346 | if (!in) { 347 | cerr << "Can't read " << input << endl; 348 | return 1; 349 | } 350 | double likelihood = 0; 351 | const double denom = n_target_tokens; 352 | int lc = 0; 353 | bool flag = false; 354 | string line; 355 | double c0 = 0; 356 | double emp_feat = 0; 357 | vector buffer; 358 | vector outputs; 359 | while(true) { 360 | getline(in, line); 361 | if (!in) break; 362 | ++lc; 363 | if (lc % 1000 == 0) { cerr << '.'; flag = true; } 364 | if (lc %50000 == 0) { cerr << " [" << lc << "]\n" << flush; flag = false; } 365 | buffer.push_back(line); 366 | 367 | if (buffer.size() >= thread_buffer_size) { 368 | UpdateFromPairs(buffer, lc, iter, final_iteration, use_null, kNULL, 369 | prob_align_not_null, &c0, &emp_feat, &likelihood, &s2t, &outputs); 370 | if (final_iteration) { 371 | for (const string& output : outputs) { 372 | cout << output; 373 | } 374 | } 375 | buffer.clear(); 376 | } 377 | } // end data loop 378 | if (buffer.size() > 0) { 379 | UpdateFromPairs(buffer, lc, iter, final_iteration, use_null, kNULL, 380 | prob_align_not_null, &c0, &emp_feat, &likelihood, &s2t, &outputs); 381 | if (final_iteration) { 382 | for (const string& output : outputs) { 383 | cout << output; 384 | } 385 | } 386 | buffer.clear(); 387 | } 388 | 389 | // log(e) = 1.0 390 | double base2_likelihood = likelihood / log(2); 391 | 392 | if (flag) { 393 | cerr << endl; 394 | } 395 | emp_feat /= n_target_tokens; 396 | cerr << " log_e likelihood: " << likelihood << endl; 397 | cerr << " log_2 likelihood: " << base2_likelihood << endl; 398 | cerr << " cross entropy: " << (-base2_likelihood / denom) << endl; 399 | cerr << " perplexity: " << pow(2.0, -base2_likelihood / denom) << endl; 400 | cerr << " posterior p0: " << c0 / n_target_tokens << endl; 401 | cerr << " posterior al-feat: " << emp_feat << endl; 402 | //cerr << " model tension: " << mod_feat / toks << endl; 403 | cerr << " size counts: " << size_counts.size() << endl; 404 | if (!final_iteration) { 405 | if (favor_diagonal && optimize_tension && iter > 0) { 406 | for (int ii = 0; ii < 8; ++ii) { 407 | double mod_feat = 0; 408 | #pragma omp parallel for reduction(+:mod_feat) 409 | for(size_t i = 0; i < size_counts.size(); ++i) { 410 | const pair& p = size_counts[i].first; 411 | for (short j = 1; j <= p.first; ++j) 412 | mod_feat += size_counts[i].second * DiagonalAlignment::ComputeDLogZ(j, p.first, p.second, diagonal_tension); 413 | } 414 | mod_feat /= n_target_tokens; 415 | cerr << " " << ii + 1 << " model al-feat: " << mod_feat << " (tension=" << diagonal_tension << ")\n"; 416 | diagonal_tension += (emp_feat - mod_feat) * 20.0; 417 | if (diagonal_tension <= 0.1) diagonal_tension = 0.1; 418 | if (diagonal_tension > 14) diagonal_tension = 14; 419 | } 420 | cerr << " final tension: " << diagonal_tension << endl; 421 | } 422 | if (variational_bayes) 423 | s2t.NormalizeVB(alpha); 424 | else 425 | s2t.Normalize(); 426 | } 427 | } 428 | if (!force_align && !conditional_probability_filename.empty()) { 429 | cerr << "conditional probabilities: " << conditional_probability_filename << endl; 430 | s2t.ExportToFile(conditional_probability_filename.c_str(), d, beam_threshold); 431 | } 432 | if (force_align) { 433 | istream* pin = &cin; 434 | if (input != "-" && !input.empty()) 435 | pin = new ifstream(input.c_str()); 436 | istream& in = *pin; 437 | string line; 438 | vector src, trg; 439 | int lc = 0; 440 | double tlp = 0; 441 | while(getline(in, line)) { 442 | ++lc; 443 | ParseLine(line, &src, &trg); 444 | for (auto s : src) cout << d.Convert(s) << ' '; 445 | cout << "|||"; 446 | for (auto t : trg) cout << ' ' << d.Convert(t); 447 | cout << " |||"; 448 | if (is_reverse) 449 | swap(src, trg); 450 | if (src.size() == 0 || trg.size() == 0) { 451 | cerr << "Error in line " << lc << endl; 452 | return 1; 453 | } 454 | double log_prob = Md::log_poisson(trg.size(), 0.05 + src.size() * mean_srclen_multiplier); 455 | 456 | // compute likelihood 457 | for (unsigned j = 0; j < trg.size(); ++j) { 458 | unsigned f_j = trg[j]; 459 | double sum = 0; 460 | int a_j = 0; 461 | double max_pat = 0; 462 | double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1) 463 | if (use_null) { 464 | if (favor_diagonal) prob_a_i = prob_align_null; 465 | max_pat = s2t.safe_prob(kNULL, f_j) * prob_a_i; 466 | sum += max_pat; 467 | } 468 | double az = 0; 469 | if (favor_diagonal) 470 | az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null; 471 | for (unsigned i = 1; i <= src.size(); ++i) { 472 | if (favor_diagonal) 473 | prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az; 474 | double pat = s2t.safe_prob(src[i-1], f_j) * prob_a_i; 475 | if (pat > max_pat) { max_pat = pat; a_j = i; } 476 | sum += pat; 477 | } 478 | log_prob += log(sum); 479 | if (true) { 480 | if (a_j > 0) { 481 | cout << ' '; 482 | if (is_reverse) 483 | cout << j << '-' << (a_j - 1); 484 | else 485 | cout << (a_j - 1) << '-' << j; 486 | } 487 | } 488 | } 489 | tlp += log_prob; 490 | cout << " ||| " << log_prob << endl << flush; 491 | } // loop over test set sentences 492 | cerr << "TOTAL LOG PROB " << tlp << endl; 493 | } 494 | return 0; 495 | } 496 | -------------------------------------------------------------------------------- /src/force_align.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | import threading 7 | 8 | # Simplified, non-threadsafe version for force_align.py 9 | # Use the version in realtime for development 10 | class Aligner: 11 | 12 | def __init__(self, fwd_params, fwd_err, rev_params, rev_err, heuristic='grow-diag-final-and'): 13 | 14 | build_root = os.path.dirname(os.path.abspath(__file__)) 15 | fast_align = os.path.join(build_root, 'fast_align') 16 | atools = os.path.join(build_root, 'atools') 17 | 18 | (fwd_T, fwd_m) = self.read_err(fwd_err) 19 | (rev_T, rev_m) = self.read_err(rev_err) 20 | 21 | fwd_cmd = [fast_align, '-i', '-', '-d', '-T', fwd_T, '-m', fwd_m, '-f', fwd_params] 22 | rev_cmd = [fast_align, '-i', '-', '-d', '-T', rev_T, '-m', rev_m, '-f', rev_params, '-r'] 23 | tools_cmd = [atools, '-i', '-', '-j', '-', '-c', heuristic] 24 | 25 | self.fwd_align = popen_io(fwd_cmd) 26 | self.rev_align = popen_io(rev_cmd) 27 | self.tools = popen_io(tools_cmd) 28 | 29 | def align(self, line): 30 | self.fwd_align.stdin.write('{}\n'.format(line)) 31 | self.rev_align.stdin.write('{}\n'.format(line)) 32 | # f words ||| e words ||| links ||| score 33 | fwd_line = self.fwd_align.stdout.readline().split('|||')[2].strip() 34 | rev_line = self.rev_align.stdout.readline().split('|||')[2].strip() 35 | self.tools.stdin.write('{}\n'.format(fwd_line)) 36 | self.tools.stdin.write('{}\n'.format(rev_line)) 37 | al_line = self.tools.stdout.readline().strip() 38 | return al_line 39 | 40 | def close(self): 41 | self.fwd_align.stdin.close() 42 | self.fwd_align.wait() 43 | self.rev_align.stdin.close() 44 | self.rev_align.wait() 45 | self.tools.stdin.close() 46 | self.tools.wait() 47 | 48 | def read_err(self, err): 49 | (T, m) = ('', '') 50 | for line in open(err): 51 | # expected target length = source length * N 52 | if 'expected target length' in line: 53 | m = line.split()[-1] 54 | # final tension: N 55 | elif 'final tension' in line: 56 | T = line.split()[-1] 57 | return (T, m) 58 | 59 | def popen_io(cmd): 60 | p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 61 | def consume(s): 62 | for _ in s: 63 | pass 64 | threading.Thread(target=consume, args=(p.stderr,)).start() 65 | return p 66 | 67 | def main(): 68 | 69 | if len(sys.argv[1:]) < 4: 70 | sys.stderr.write('run:\n') 71 | sys.stderr.write(' fast_align -i corpus.f-e -d -v -o -p fwd_params >fwd_align 2>fwd_err\n') 72 | sys.stderr.write(' fast_align -i corpus.f-e -r -d -v -o -p rev_params >rev_align 2>rev_err\n') 73 | sys.stderr.write('\n') 74 | sys.stderr.write('then run:\n') 75 | sys.stderr.write(' {} fwd_params fwd_err rev_params rev_err [heuristic] out.f-e.gdfa\n'.format(sys.argv[0])) 76 | sys.stderr.write('\n') 77 | sys.stderr.write('where heuristic is one of: (intersect union grow-diag grow-diag-final grow-diag-final-and) default=grow-diag-final-and\n') 78 | sys.exit(2) 79 | 80 | aligner = Aligner(*sys.argv[1:]) 81 | 82 | while True: 83 | line = sys.stdin.readline() 84 | if not line: 85 | break 86 | sys.stdout.write('{}\n'.format(aligner.align(line.strip()))) 87 | sys.stdout.flush() 88 | 89 | aligner.close() 90 | 91 | if __name__ == '__main__': 92 | main() 93 | 94 | 95 | -------------------------------------------------------------------------------- /src/hashtables.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_HASHTABLES_H 2 | #define SRC_HASHTABLES_H 3 | 4 | #ifdef HAVE_SPARSEHASH 5 | #include 6 | typedef google::sparse_hash_map > MAP_TYPE; 7 | typedef google::sparse_hash_map Word2Double; 8 | #else 9 | #include 10 | typedef std::unordered_map > MAP_TYPE; 11 | typedef std::unordered_map Word2Double; 12 | #endif 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /src/port.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 by Tetsuo Kiso 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef FAST_ALIGN_PORT_H_ 16 | #define FAST_ALIGN_PORT_H_ 17 | 18 | // As of OS X 10.9, it looks like C++ TR1 headers are removed from the 19 | // search paths. Instead, we can include C++11 headers. 20 | #if defined(__APPLE__) 21 | #include 22 | #endif 23 | 24 | #if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_9) && \ 25 | MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_9 26 | #include 27 | #include 28 | #else // Assuming older OS X, Linux or similar platforms 29 | #include 30 | #include 31 | //#include 32 | //#include 33 | //namespace std { 34 | //using tr1::unordered_map; 35 | //using tr1::hash; 36 | //} // namespace std 37 | #endif 38 | 39 | #endif // FAST_ALIGN_PORT_H_ 40 | -------------------------------------------------------------------------------- /src/ttables.cc: -------------------------------------------------------------------------------- 1 | #include "src/ttables.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "src/corpus.h" 8 | 9 | void TTable::DeserializeLogProbsFromText(std::istream* in, Dict& d) { 10 | int c = 0; 11 | std::string e, f; 12 | double p; 13 | while(*in) { 14 | (*in) >> e >> f >> p; 15 | if (e.empty()) break; 16 | ++c; 17 | unsigned ie = d.Convert(e); 18 | if (ie >= ttable.size()) ttable.resize(ie + 1); 19 | ttable[ie][d.Convert(f)] = std::exp(p); 20 | } 21 | std::cerr << "Loaded " << c << " translation parameters.\n"; 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/ttables.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 by Chris Dyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #ifndef _TTABLES_H_ 16 | #define _TTABLES_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "src/hashtables.h" 25 | #include "src/corpus.h" 26 | 27 | struct Md { 28 | static double digamma(double x) { 29 | double result = 0, xx, xx2, xx4; 30 | for ( ; x < 7; ++x) 31 | result -= 1/x; 32 | x -= 1.0/2.0; 33 | xx = 1.0/x; 34 | xx2 = xx*xx; 35 | xx4 = xx2*xx2; 36 | result += log(x)+(1./24.)*xx2-(7.0/960.0)*xx4+(31.0/8064.0)*xx4*xx2-(127.0/30720.0)*xx4*xx4; 37 | return result; 38 | } 39 | static inline double log_poisson(unsigned x, const double& lambda) { 40 | assert(lambda > 0.0); 41 | return std::log(lambda) * x - lgamma(x + 1) - lambda; 42 | } 43 | }; 44 | 45 | class TTable { 46 | public: 47 | TTable() : frozen_(false), probs_initialized_(false) {} 48 | // typedef std::unordered_map Word2Double; 49 | 50 | typedef std::vector Word2Word2Double; 51 | 52 | inline double prob(const unsigned e, const unsigned f) const { 53 | return probs_initialized_ ? ttable[e].find(f)->second : 1e-9; 54 | } 55 | 56 | inline double safe_prob(const int& e, const int& f) const { 57 | if (e < static_cast(ttable.size())) { 58 | const Word2Double& cpd = ttable[e]; 59 | const Word2Double::const_iterator it = cpd.find(f); 60 | if (it == cpd.end()) return 1e-9; 61 | return it->second; 62 | } else { 63 | return 1e-9; 64 | } 65 | } 66 | 67 | inline void SetMaxE(const unsigned e) { 68 | // NOT thread safe 69 | if (e >= counts.size()) 70 | counts.resize(e + 1); 71 | } 72 | 73 | inline void Insert(const unsigned e, const unsigned f) { 74 | // NOT thread safe 75 | if (e >= counts.size()) 76 | counts.resize(e + 1); 77 | counts[e][f] = 0; 78 | } 79 | 80 | inline void Increment(const unsigned e, const unsigned f, const double x) { 81 | counts[e].find(f)->second += x; // Ignore race conditions here. 82 | } 83 | 84 | void NormalizeVB(const double alpha) { 85 | ttable.swap(counts); 86 | #pragma omp parallel for schedule(dynamic) 87 | for (unsigned i = 0; i < ttable.size(); ++i) { 88 | double tot = 0; 89 | Word2Double& cpd = ttable[i]; 90 | for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) 91 | tot += it->second + alpha; 92 | if (!tot) tot = 1; 93 | const double digamma_tot = Md::digamma(tot); 94 | for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) 95 | it->second = exp(Md::digamma(it->second + alpha) - digamma_tot); 96 | } 97 | ClearCounts(); 98 | probs_initialized_ = true; 99 | } 100 | 101 | void Normalize() { 102 | ttable.swap(counts); 103 | #pragma omp parallel for schedule(dynamic) 104 | for (unsigned i = 0; i < ttable.size(); ++i) { 105 | double tot = 0; 106 | Word2Double& cpd = ttable[i]; 107 | for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) 108 | tot += it->second; 109 | if (!tot) tot = 1; 110 | for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) 111 | it->second /= tot; 112 | } 113 | ClearCounts(); 114 | probs_initialized_ = true; 115 | } 116 | 117 | void Freeze() { 118 | // duplicate all values in counts into ttable 119 | // later updates to both are semi-threadsafe 120 | assert (!frozen_); 121 | if (!frozen_) { 122 | ttable.resize(counts.size()); 123 | for (unsigned i = 0; i < counts.size(); ++i) { 124 | ttable[i] = counts[i]; 125 | } 126 | } 127 | frozen_ = true; 128 | } 129 | // adds counts from another TTable - probabilities remain unchanged 130 | TTable& operator+=(const TTable& rhs) { 131 | if (rhs.counts.size() > counts.size()) counts.resize(rhs.counts.size()); 132 | for (unsigned i = 0; i < rhs.counts.size(); ++i) { 133 | const Word2Double& cpd = rhs.counts[i]; 134 | Word2Double& tgt = counts[i]; 135 | for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) { 136 | tgt[j->first] += j->second; 137 | } 138 | } 139 | return *this; 140 | } 141 | void ExportToFile(const char* filename, Dict& d, double BEAM_THRESHOLD) const { 142 | std::ofstream file(filename); 143 | for (unsigned i = 0; i < ttable.size(); ++i) { 144 | const std::string& a = d.Convert(i); 145 | const Word2Double& cpd = ttable[i]; 146 | double max_p = -1; 147 | for (auto& it : cpd) 148 | if (it.second > max_p) max_p = it.second; 149 | const double threshold = - log(max_p) * BEAM_THRESHOLD; 150 | for (auto& it : cpd) { 151 | const std::string& b = d.Convert(it.first); 152 | double c = log(it.second); 153 | if (c >= threshold) 154 | file << a << '\t' << b << '\t' << c << std::endl; 155 | } 156 | } 157 | file.close(); 158 | } 159 | private: 160 | void ClearCounts() { 161 | #pragma omp parallel for schedule(dynamic) 162 | for (size_t i=0; i