├── .gitignore
├── .idea
    ├── dataSources.local.xml
    ├── dataSources.xml
    ├── dataSources
    │   ├── 0e2e93c2-ec73-47f1-a3f3-868befde3f72.xml
    │   └── eb023117-f844-4ce6-9dbd-5a03c863f17a.xml
    ├── encodings.xml
    ├── inspectionProfiles
    │   └── Project_Default.xml
    ├── misc.xml
    ├── modules.xml
    ├── other.xml
    ├── transformer-master.iml
    ├── vcs.xml
    └── workspace.xml
├── LICENSE
├── README.md
├── __pycache__
    ├── beam_search.cpython-36.pyc
    ├── data_load.cpython-36.pyc
    ├── hparams.cpython-36.pyc
    ├── model.cpython-36.pyc
    ├── modules.cpython-36.pyc
    └── utils.cpython-36.pyc
├── beam_search.py
├── data_load.py
├── fig
    ├── structure.jpg
    ├── transformer-loss.png
    └── transformer-pointer gererator-loss.png
├── hparams.py
├── model.py
├── modules.py
├── pred.py
├── requirements.txt
├── train.py
├── utils.py
└── vocab


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 


--------------------------------------------------------------------------------
/.idea/dataSources.local.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project version="4">
 3 |   <component name="dataSourceStorageLocal">
 4 |     <data-source name="awake_test" uuid="eb023117-f844-4ce6-9dbd-5a03c863f17a">
 5 |       <database-info product="MySQL" version="5.6.43" jdbc-version="4.2" driver-name="MySQL Connector/J" driver-version="mysql-connector-java-8.0.15 (Revision: 79a4336f140499bd22dd07f02b708e163844e3d5)" dbms="MYSQL" exact-version="5.6.43" exact-driver-version="8.0">
 6 |         <extra-name-characters>#@</extra-name-characters>
 7 |         <identifier-quote-string>`</identifier-quote-string>
 8 |       </database-info>
 9 |       <case-sensitivity plain-identifiers="exact" quoted-identifiers="exact" />
10 |       <secret-storage>master_key</secret-storage>
11 |       <user-name>juexiaotest</user-name>
12 |     </data-source>
13 |     <data-source name="awake_official" uuid="0e2e93c2-ec73-47f1-a3f3-868befde3f72">
14 |       <database-info product="MySQL" version="5.6.16-log" jdbc-version="4.2" driver-name="MySQL Connector/J" driver-version="mysql-connector-java-8.0.15 (Revision: 79a4336f140499bd22dd07f02b708e163844e3d5)" dbms="MYSQL" exact-version="5.6.16" exact-driver-version="8.0">
15 |         <extra-name-characters>#@</extra-name-characters>
16 |         <identifier-quote-string>`</identifier-quote-string>
17 |       </database-info>
18 |       <case-sensitivity plain-identifiers="lower" quoted-identifiers="lower" />
19 |       <secret-storage>master_key</secret-storage>
20 |       <user-name>juexiaotimered</user-name>
21 |     </data-source>
22 |   </component>
23 | </project>


--------------------------------------------------------------------------------
/.idea/dataSources.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project version="4">
 3 |   <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
 4 |     <data-source source="LOCAL" name="awake_test" uuid="eb023117-f844-4ce6-9dbd-5a03c863f17a">
 5 |       <driver-ref>mysql.8</driver-ref>
 6 |       <synchronize>true</synchronize>
 7 |       <jdbc-driver>com.mysql.cj.jdbc.Driver</jdbc-driver>
 8 |       <jdbc-url>jdbc:mysql://47.102.145.57:3306/jx</jdbc-url>
 9 |       <driver-properties>
10 |         <property name="autoReconnect" value="true" />
11 |         <property name="zeroDateTimeBehavior" value="CONVERT_TO_NULL" />
12 |         <property name="tinyInt1isBit" value="false" />
13 |         <property name="characterEncoding" value="utf8" />
14 |         <property name="characterSetResults" value="utf8" />
15 |         <property name="yearIsDateType" value="false" />
16 |       </driver-properties>
17 |     </data-source>
18 |     <data-source source="LOCAL" name="awake_official" uuid="0e2e93c2-ec73-47f1-a3f3-868befde3f72">
19 |       <driver-ref>mysql.8</driver-ref>
20 |       <synchronize>true</synchronize>
21 |       <jdbc-driver>com.mysql.cj.jdbc.Driver</jdbc-driver>
22 |       <jdbc-url>jdbc:mysql://juexiaotime.rwlb.rds.aliyuncs.com:3306/jx</jdbc-url>
23 |       <driver-properties>
24 |         <property name="autoReconnect" value="true" />
25 |         <property name="zeroDateTimeBehavior" value="CONVERT_TO_NULL" />
26 |         <property name="tinyInt1isBit" value="false" />
27 |         <property name="characterEncoding" value="utf8" />
28 |         <property name="characterSetResults" value="utf8" />
29 |         <property name="yearIsDateType" value="false" />
30 |       </driver-properties>
31 |     </data-source>
32 |   </component>
33 | </project>


--------------------------------------------------------------------------------
/.idea/dataSources/0e2e93c2-ec73-47f1-a3f3-868befde3f72.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <dataSource name="awake_official">
  3 |   <database-model serializer="dbm" dbms="MYSQL" family-id="MYSQL" format-version="4.15">
  4 |     <root id="1">
  5 |       <ServerVersion>5.6.16</ServerVersion>
  6 |       <DefaultEngine>InnoDB</DefaultEngine>
  7 |       <DefaultTmpEngine>InnoDB</DefaultTmpEngine>
  8 |       <DefaultCasing>lower/lower</DefaultCasing>
  9 |     </root>
 10 |     <schema id="2" parent="1" name="information_schema">
 11 |       <Collation>utf8_general_ci</Collation>
 12 |     </schema>
 13 |     <schema id="3" parent="1" name="jx">
 14 |       <Current>1</Current>
 15 |       <Collation>utf8_general_ci</Collation>
 16 |     </schema>
 17 |     <schema id="4" parent="1" name="mysql">
 18 |       <Collation>utf8_general_ci</Collation>
 19 |     </schema>
 20 |     <schema id="5" parent="1" name="test">
 21 |       <Collation>utf8_general_ci</Collation>
 22 |     </schema>
 23 |     <collation id="6" parent="1" name="armscii8_bin">
 24 |       <Charset>armscii8</Charset>
 25 |       <DefaultForCharset>0</DefaultForCharset>
 26 |     </collation>
 27 |     <collation id="7" parent="1" name="armscii8_general_ci">
 28 |       <Charset>armscii8</Charset>
 29 |       <DefaultForCharset>1</DefaultForCharset>
 30 |     </collation>
 31 |     <collation id="8" parent="1" name="ascii_bin">
 32 |       <Charset>ascii</Charset>
 33 |       <DefaultForCharset>0</DefaultForCharset>
 34 |     </collation>
 35 |     <collation id="9" parent="1" name="ascii_general_ci">
 36 |       <Charset>ascii</Charset>
 37 |       <DefaultForCharset>1</DefaultForCharset>
 38 |     </collation>
 39 |     <collation id="10" parent="1" name="big5_bin">
 40 |       <Charset>big5</Charset>
 41 |       <DefaultForCharset>0</DefaultForCharset>
 42 |     </collation>
 43 |     <collation id="11" parent="1" name="big5_chinese_ci">
 44 |       <Charset>big5</Charset>
 45 |       <DefaultForCharset>1</DefaultForCharset>
 46 |     </collation>
 47 |     <collation id="12" parent="1" name="binary">
 48 |       <Charset>binary</Charset>
 49 |       <DefaultForCharset>1</DefaultForCharset>
 50 |     </collation>
 51 |     <collation id="13" parent="1" name="cp1250_bin">
 52 |       <Charset>cp1250</Charset>
 53 |       <DefaultForCharset>0</DefaultForCharset>
 54 |     </collation>
 55 |     <collation id="14" parent="1" name="cp1250_croatian_ci">
 56 |       <Charset>cp1250</Charset>
 57 |       <DefaultForCharset>0</DefaultForCharset>
 58 |     </collation>
 59 |     <collation id="15" parent="1" name="cp1250_czech_cs">
 60 |       <Charset>cp1250</Charset>
 61 |       <DefaultForCharset>0</DefaultForCharset>
 62 |     </collation>
 63 |     <collation id="16" parent="1" name="cp1250_general_ci">
 64 |       <Charset>cp1250</Charset>
 65 |       <DefaultForCharset>1</DefaultForCharset>
 66 |     </collation>
 67 |     <collation id="17" parent="1" name="cp1250_polish_ci">
 68 |       <Charset>cp1250</Charset>
 69 |       <DefaultForCharset>0</DefaultForCharset>
 70 |     </collation>
 71 |     <collation id="18" parent="1" name="cp1251_bin">
 72 |       <Charset>cp1251</Charset>
 73 |       <DefaultForCharset>0</DefaultForCharset>
 74 |     </collation>
 75 |     <collation id="19" parent="1" name="cp1251_bulgarian_ci">
 76 |       <Charset>cp1251</Charset>
 77 |       <DefaultForCharset>0</DefaultForCharset>
 78 |     </collation>
 79 |     <collation id="20" parent="1" name="cp1251_general_ci">
 80 |       <Charset>cp1251</Charset>
 81 |       <DefaultForCharset>1</DefaultForCharset>
 82 |     </collation>
 83 |     <collation id="21" parent="1" name="cp1251_general_cs">
 84 |       <Charset>cp1251</Charset>
 85 |       <DefaultForCharset>0</DefaultForCharset>
 86 |     </collation>
 87 |     <collation id="22" parent="1" name="cp1251_ukrainian_ci">
 88 |       <Charset>cp1251</Charset>
 89 |       <DefaultForCharset>0</DefaultForCharset>
 90 |     </collation>
 91 |     <collation id="23" parent="1" name="cp1256_bin">
 92 |       <Charset>cp1256</Charset>
 93 |       <DefaultForCharset>0</DefaultForCharset>
 94 |     </collation>
 95 |     <collation id="24" parent="1" name="cp1256_general_ci">
 96 |       <Charset>cp1256</Charset>
 97 |       <DefaultForCharset>1</DefaultForCharset>
 98 |     </collation>
 99 |     <collation id="25" parent="1" name="cp1257_bin">
100 |       <Charset>cp1257</Charset>
101 |       <DefaultForCharset>0</DefaultForCharset>
102 |     </collation>
103 |     <collation id="26" parent="1" name="cp1257_general_ci">
104 |       <Charset>cp1257</Charset>
105 |       <DefaultForCharset>1</DefaultForCharset>
106 |     </collation>
107 |     <collation id="27" parent="1" name="cp1257_lithuanian_ci">
108 |       <Charset>cp1257</Charset>
109 |       <DefaultForCharset>0</DefaultForCharset>
110 |     </collation>
111 |     <collation id="28" parent="1" name="cp850_bin">
112 |       <Charset>cp850</Charset>
113 |       <DefaultForCharset>0</DefaultForCharset>
114 |     </collation>
115 |     <collation id="29" parent="1" name="cp850_general_ci">
116 |       <Charset>cp850</Charset>
117 |       <DefaultForCharset>1</DefaultForCharset>
118 |     </collation>
119 |     <collation id="30" parent="1" name="cp852_bin">
120 |       <Charset>cp852</Charset>
121 |       <DefaultForCharset>0</DefaultForCharset>
122 |     </collation>
123 |     <collation id="31" parent="1" name="cp852_general_ci">
124 |       <Charset>cp852</Charset>
125 |       <DefaultForCharset>1</DefaultForCharset>
126 |     </collation>
127 |     <collation id="32" parent="1" name="cp866_bin">
128 |       <Charset>cp866</Charset>
129 |       <DefaultForCharset>0</DefaultForCharset>
130 |     </collation>
131 |     <collation id="33" parent="1" name="cp866_general_ci">
132 |       <Charset>cp866</Charset>
133 |       <DefaultForCharset>1</DefaultForCharset>
134 |     </collation>
135 |     <collation id="34" parent="1" name="cp932_bin">
136 |       <Charset>cp932</Charset>
137 |       <DefaultForCharset>0</DefaultForCharset>
138 |     </collation>
139 |     <collation id="35" parent="1" name="cp932_japanese_ci">
140 |       <Charset>cp932</Charset>
141 |       <DefaultForCharset>1</DefaultForCharset>
142 |     </collation>
143 |     <collation id="36" parent="1" name="dec8_bin">
144 |       <Charset>dec8</Charset>
145 |       <DefaultForCharset>0</DefaultForCharset>
146 |     </collation>
147 |     <collation id="37" parent="1" name="dec8_swedish_ci">
148 |       <Charset>dec8</Charset>
149 |       <DefaultForCharset>1</DefaultForCharset>
150 |     </collation>
151 |     <collation id="38" parent="1" name="eucjpms_bin">
152 |       <Charset>eucjpms</Charset>
153 |       <DefaultForCharset>0</DefaultForCharset>
154 |     </collation>
155 |     <collation id="39" parent="1" name="eucjpms_japanese_ci">
156 |       <Charset>eucjpms</Charset>
157 |       <DefaultForCharset>1</DefaultForCharset>
158 |     </collation>
159 |     <collation id="40" parent="1" name="euckr_bin">
160 |       <Charset>euckr</Charset>
161 |       <DefaultForCharset>0</DefaultForCharset>
162 |     </collation>
163 |     <collation id="41" parent="1" name="euckr_korean_ci">
164 |       <Charset>euckr</Charset>
165 |       <DefaultForCharset>1</DefaultForCharset>
166 |     </collation>
167 |     <collation id="42" parent="1" name="gb2312_bin">
168 |       <Charset>gb2312</Charset>
169 |       <DefaultForCharset>0</DefaultForCharset>
170 |     </collation>
171 |     <collation id="43" parent="1" name="gb2312_chinese_ci">
172 |       <Charset>gb2312</Charset>
173 |       <DefaultForCharset>1</DefaultForCharset>
174 |     </collation>
175 |     <collation id="44" parent="1" name="gbk_bin">
176 |       <Charset>gbk</Charset>
177 |       <DefaultForCharset>0</DefaultForCharset>
178 |     </collation>
179 |     <collation id="45" parent="1" name="gbk_chinese_ci">
180 |       <Charset>gbk</Charset>
181 |       <DefaultForCharset>1</DefaultForCharset>
182 |     </collation>
183 |     <collation id="46" parent="1" name="geostd8_bin">
184 |       <Charset>geostd8</Charset>
185 |       <DefaultForCharset>0</DefaultForCharset>
186 |     </collation>
187 |     <collation id="47" parent="1" name="geostd8_general_ci">
188 |       <Charset>geostd8</Charset>
189 |       <DefaultForCharset>1</DefaultForCharset>
190 |     </collation>
191 |     <collation id="48" parent="1" name="greek_bin">
192 |       <Charset>greek</Charset>
193 |       <DefaultForCharset>0</DefaultForCharset>
194 |     </collation>
195 |     <collation id="49" parent="1" name="greek_general_ci">
196 |       <Charset>greek</Charset>
197 |       <DefaultForCharset>1</DefaultForCharset>
198 |     </collation>
199 |     <collation id="50" parent="1" name="hebrew_bin">
200 |       <Charset>hebrew</Charset>
201 |       <DefaultForCharset>0</DefaultForCharset>
202 |     </collation>
203 |     <collation id="51" parent="1" name="hebrew_general_ci">
204 |       <Charset>hebrew</Charset>
205 |       <DefaultForCharset>1</DefaultForCharset>
206 |     </collation>
207 |     <collation id="52" parent="1" name="hp8_bin">
208 |       <Charset>hp8</Charset>
209 |       <DefaultForCharset>0</DefaultForCharset>
210 |     </collation>
211 |     <collation id="53" parent="1" name="hp8_english_ci">
212 |       <Charset>hp8</Charset>
213 |       <DefaultForCharset>1</DefaultForCharset>
214 |     </collation>
215 |     <collation id="54" parent="1" name="keybcs2_bin">
216 |       <Charset>keybcs2</Charset>
217 |       <DefaultForCharset>0</DefaultForCharset>
218 |     </collation>
219 |     <collation id="55" parent="1" name="keybcs2_general_ci">
220 |       <Charset>keybcs2</Charset>
221 |       <DefaultForCharset>1</DefaultForCharset>
222 |     </collation>
223 |     <collation id="56" parent="1" name="koi8r_bin">
224 |       <Charset>koi8r</Charset>
225 |       <DefaultForCharset>0</DefaultForCharset>
226 |     </collation>
227 |     <collation id="57" parent="1" name="koi8r_general_ci">
228 |       <Charset>koi8r</Charset>
229 |       <DefaultForCharset>1</DefaultForCharset>
230 |     </collation>
231 |     <collation id="58" parent="1" name="koi8u_bin">
232 |       <Charset>koi8u</Charset>
233 |       <DefaultForCharset>0</DefaultForCharset>
234 |     </collation>
235 |     <collation id="59" parent="1" name="koi8u_general_ci">
236 |       <Charset>koi8u</Charset>
237 |       <DefaultForCharset>1</DefaultForCharset>
238 |     </collation>
239 |     <collation id="60" parent="1" name="latin1_bin">
240 |       <Charset>latin1</Charset>
241 |       <DefaultForCharset>0</DefaultForCharset>
242 |     </collation>
243 |     <collation id="61" parent="1" name="latin1_danish_ci">
244 |       <Charset>latin1</Charset>
245 |       <DefaultForCharset>0</DefaultForCharset>
246 |     </collation>
247 |     <collation id="62" parent="1" name="latin1_general_ci">
248 |       <Charset>latin1</Charset>
249 |       <DefaultForCharset>0</DefaultForCharset>
250 |     </collation>
251 |     <collation id="63" parent="1" name="latin1_general_cs">
252 |       <Charset>latin1</Charset>
253 |       <DefaultForCharset>0</DefaultForCharset>
254 |     </collation>
255 |     <collation id="64" parent="1" name="latin1_german1_ci">
256 |       <Charset>latin1</Charset>
257 |       <DefaultForCharset>0</DefaultForCharset>
258 |     </collation>
259 |     <collation id="65" parent="1" name="latin1_german2_ci">
260 |       <Charset>latin1</Charset>
261 |       <DefaultForCharset>0</DefaultForCharset>
262 |     </collation>
263 |     <collation id="66" parent="1" name="latin1_spanish_ci">
264 |       <Charset>latin1</Charset>
265 |       <DefaultForCharset>0</DefaultForCharset>
266 |     </collation>
267 |     <collation id="67" parent="1" name="latin1_swedish_ci">
268 |       <Charset>latin1</Charset>
269 |       <DefaultForCharset>1</DefaultForCharset>
270 |     </collation>
271 |     <collation id="68" parent="1" name="latin2_bin">
272 |       <Charset>latin2</Charset>
273 |       <DefaultForCharset>0</DefaultForCharset>
274 |     </collation>
275 |     <collation id="69" parent="1" name="latin2_croatian_ci">
276 |       <Charset>latin2</Charset>
277 |       <DefaultForCharset>0</DefaultForCharset>
278 |     </collation>
279 |     <collation id="70" parent="1" name="latin2_czech_cs">
280 |       <Charset>latin2</Charset>
281 |       <DefaultForCharset>0</DefaultForCharset>
282 |     </collation>
283 |     <collation id="71" parent="1" name="latin2_general_ci">
284 |       <Charset>latin2</Charset>
285 |       <DefaultForCharset>1</DefaultForCharset>
286 |     </collation>
287 |     <collation id="72" parent="1" name="latin2_hungarian_ci">
288 |       <Charset>latin2</Charset>
289 |       <DefaultForCharset>0</DefaultForCharset>
290 |     </collation>
291 |     <collation id="73" parent="1" name="latin5_bin">
292 |       <Charset>latin5</Charset>
293 |       <DefaultForCharset>0</DefaultForCharset>
294 |     </collation>
295 |     <collation id="74" parent="1" name="latin5_turkish_ci">
296 |       <Charset>latin5</Charset>
297 |       <DefaultForCharset>1</DefaultForCharset>
298 |     </collation>
299 |     <collation id="75" parent="1" name="latin7_bin">
300 |       <Charset>latin7</Charset>
301 |       <DefaultForCharset>0</DefaultForCharset>
302 |     </collation>
303 |     <collation id="76" parent="1" name="latin7_estonian_cs">
304 |       <Charset>latin7</Charset>
305 |       <DefaultForCharset>0</DefaultForCharset>
306 |     </collation>
307 |     <collation id="77" parent="1" name="latin7_general_ci">
308 |       <Charset>latin7</Charset>
309 |       <DefaultForCharset>1</DefaultForCharset>
310 |     </collation>
311 |     <collation id="78" parent="1" name="latin7_general_cs">
312 |       <Charset>latin7</Charset>
313 |       <DefaultForCharset>0</DefaultForCharset>
314 |     </collation>
315 |     <collation id="79" parent="1" name="macce_bin">
316 |       <Charset>macce</Charset>
317 |       <DefaultForCharset>0</DefaultForCharset>
318 |     </collation>
319 |     <collation id="80" parent="1" name="macce_general_ci">
320 |       <Charset>macce</Charset>
321 |       <DefaultForCharset>1</DefaultForCharset>
322 |     </collation>
323 |     <collation id="81" parent="1" name="macroman_bin">
324 |       <Charset>macroman</Charset>
325 |       <DefaultForCharset>0</DefaultForCharset>
326 |     </collation>
327 |     <collation id="82" parent="1" name="macroman_general_ci">
328 |       <Charset>macroman</Charset>
329 |       <DefaultForCharset>1</DefaultForCharset>
330 |     </collation>
331 |     <collation id="83" parent="1" name="sjis_bin">
332 |       <Charset>sjis</Charset>
333 |       <DefaultForCharset>0</DefaultForCharset>
334 |     </collation>
335 |     <collation id="84" parent="1" name="sjis_japanese_ci">
336 |       <Charset>sjis</Charset>
337 |       <DefaultForCharset>1</DefaultForCharset>
338 |     </collation>
339 |     <collation id="85" parent="1" name="swe7_bin">
340 |       <Charset>swe7</Charset>
341 |       <DefaultForCharset>0</DefaultForCharset>
342 |     </collation>
343 |     <collation id="86" parent="1" name="swe7_swedish_ci">
344 |       <Charset>swe7</Charset>
345 |       <DefaultForCharset>1</DefaultForCharset>
346 |     </collation>
347 |     <collation id="87" parent="1" name="tis620_bin">
348 |       <Charset>tis620</Charset>
349 |       <DefaultForCharset>0</DefaultForCharset>
350 |     </collation>
351 |     <collation id="88" parent="1" name="tis620_thai_ci">
352 |       <Charset>tis620</Charset>
353 |       <DefaultForCharset>1</DefaultForCharset>
354 |     </collation>
355 |     <collation id="89" parent="1" name="ucs2_bin">
356 |       <Charset>ucs2</Charset>
357 |       <DefaultForCharset>0</DefaultForCharset>
358 |     </collation>
359 |     <collation id="90" parent="1" name="ucs2_croatian_ci">
360 |       <Charset>ucs2</Charset>
361 |       <DefaultForCharset>0</DefaultForCharset>
362 |     </collation>
363 |     <collation id="91" parent="1" name="ucs2_czech_ci">
364 |       <Charset>ucs2</Charset>
365 |       <DefaultForCharset>0</DefaultForCharset>
366 |     </collation>
367 |     <collation id="92" parent="1" name="ucs2_danish_ci">
368 |       <Charset>ucs2</Charset>
369 |       <DefaultForCharset>0</DefaultForCharset>
370 |     </collation>
371 |     <collation id="93" parent="1" name="ucs2_esperanto_ci">
372 |       <Charset>ucs2</Charset>
373 |       <DefaultForCharset>0</DefaultForCharset>
374 |     </collation>
375 |     <collation id="94" parent="1" name="ucs2_estonian_ci">
376 |       <Charset>ucs2</Charset>
377 |       <DefaultForCharset>0</DefaultForCharset>
378 |     </collation>
379 |     <collation id="95" parent="1" name="ucs2_general_ci">
380 |       <Charset>ucs2</Charset>
381 |       <DefaultForCharset>1</DefaultForCharset>
382 |     </collation>
383 |     <collation id="96" parent="1" name="ucs2_general_mysql500_ci">
384 |       <Charset>ucs2</Charset>
385 |       <DefaultForCharset>0</DefaultForCharset>
386 |     </collation>
387 |     <collation id="97" parent="1" name="ucs2_german2_ci">
388 |       <Charset>ucs2</Charset>
389 |       <DefaultForCharset>0</DefaultForCharset>
390 |     </collation>
391 |     <collation id="98" parent="1" name="ucs2_hungarian_ci">
392 |       <Charset>ucs2</Charset>
393 |       <DefaultForCharset>0</DefaultForCharset>
394 |     </collation>
395 |     <collation id="99" parent="1" name="ucs2_icelandic_ci">
396 |       <Charset>ucs2</Charset>
397 |       <DefaultForCharset>0</DefaultForCharset>
398 |     </collation>
399 |     <collation id="100" parent="1" name="ucs2_latvian_ci">
400 |       <Charset>ucs2</Charset>
401 |       <DefaultForCharset>0</DefaultForCharset>
402 |     </collation>
403 |     <collation id="101" parent="1" name="ucs2_lithuanian_ci">
404 |       <Charset>ucs2</Charset>
405 |       <DefaultForCharset>0</DefaultForCharset>
406 |     </collation>
407 |     <collation id="102" parent="1" name="ucs2_persian_ci">
408 |       <Charset>ucs2</Charset>
409 |       <DefaultForCharset>0</DefaultForCharset>
410 |     </collation>
411 |     <collation id="103" parent="1" name="ucs2_polish_ci">
412 |       <Charset>ucs2</Charset>
413 |       <DefaultForCharset>0</DefaultForCharset>
414 |     </collation>
415 |     <collation id="104" parent="1" name="ucs2_roman_ci">
416 |       <Charset>ucs2</Charset>
417 |       <DefaultForCharset>0</DefaultForCharset>
418 |     </collation>
419 |     <collation id="105" parent="1" name="ucs2_romanian_ci">
420 |       <Charset>ucs2</Charset>
421 |       <DefaultForCharset>0</DefaultForCharset>
422 |     </collation>
423 |     <collation id="106" parent="1" name="ucs2_sinhala_ci">
424 |       <Charset>ucs2</Charset>
425 |       <DefaultForCharset>0</DefaultForCharset>
426 |     </collation>
427 |     <collation id="107" parent="1" name="ucs2_slovak_ci">
428 |       <Charset>ucs2</Charset>
429 |       <DefaultForCharset>0</DefaultForCharset>
430 |     </collation>
431 |     <collation id="108" parent="1" name="ucs2_slovenian_ci">
432 |       <Charset>ucs2</Charset>
433 |       <DefaultForCharset>0</DefaultForCharset>
434 |     </collation>
435 |     <collation id="109" parent="1" name="ucs2_spanish2_ci">
436 |       <Charset>ucs2</Charset>
437 |       <DefaultForCharset>0</DefaultForCharset>
438 |     </collation>
439 |     <collation id="110" parent="1" name="ucs2_spanish_ci">
440 |       <Charset>ucs2</Charset>
441 |       <DefaultForCharset>0</DefaultForCharset>
442 |     </collation>
443 |     <collation id="111" parent="1" name="ucs2_swedish_ci">
444 |       <Charset>ucs2</Charset>
445 |       <DefaultForCharset>0</DefaultForCharset>
446 |     </collation>
447 |     <collation id="112" parent="1" name="ucs2_turkish_ci">
448 |       <Charset>ucs2</Charset>
449 |       <DefaultForCharset>0</DefaultForCharset>
450 |     </collation>
451 |     <collation id="113" parent="1" name="ucs2_unicode_520_ci">
452 |       <Charset>ucs2</Charset>
453 |       <DefaultForCharset>0</DefaultForCharset>
454 |     </collation>
455 |     <collation id="114" parent="1" name="ucs2_unicode_ci">
456 |       <Charset>ucs2</Charset>
457 |       <DefaultForCharset>0</DefaultForCharset>
458 |     </collation>
459 |     <collation id="115" parent="1" name="ucs2_vietnamese_ci">
460 |       <Charset>ucs2</Charset>
461 |       <DefaultForCharset>0</DefaultForCharset>
462 |     </collation>
463 |     <collation id="116" parent="1" name="ujis_bin">
464 |       <Charset>ujis</Charset>
465 |       <DefaultForCharset>0</DefaultForCharset>
466 |     </collation>
467 |     <collation id="117" parent="1" name="ujis_japanese_ci">
468 |       <Charset>ujis</Charset>
469 |       <DefaultForCharset>1</DefaultForCharset>
470 |     </collation>
471 |     <collation id="118" parent="1" name="utf16_bin">
472 |       <Charset>utf16</Charset>
473 |       <DefaultForCharset>0</DefaultForCharset>
474 |     </collation>
475 |     <collation id="119" parent="1" name="utf16_croatian_ci">
476 |       <Charset>utf16</Charset>
477 |       <DefaultForCharset>0</DefaultForCharset>
478 |     </collation>
479 |     <collation id="120" parent="1" name="utf16_czech_ci">
480 |       <Charset>utf16</Charset>
481 |       <DefaultForCharset>0</DefaultForCharset>
482 |     </collation>
483 |     <collation id="121" parent="1" name="utf16_danish_ci">
484 |       <Charset>utf16</Charset>
485 |       <DefaultForCharset>0</DefaultForCharset>
486 |     </collation>
487 |     <collation id="122" parent="1" name="utf16_esperanto_ci">
488 |       <Charset>utf16</Charset>
489 |       <DefaultForCharset>0</DefaultForCharset>
490 |     </collation>
491 |     <collation id="123" parent="1" name="utf16_estonian_ci">
492 |       <Charset>utf16</Charset>
493 |       <DefaultForCharset>0</DefaultForCharset>
494 |     </collation>
495 |     <collation id="124" parent="1" name="utf16_general_ci">
496 |       <Charset>utf16</Charset>
497 |       <DefaultForCharset>1</DefaultForCharset>
498 |     </collation>
499 |     <collation id="125" parent="1" name="utf16_german2_ci">
500 |       <Charset>utf16</Charset>
501 |       <DefaultForCharset>0</DefaultForCharset>
502 |     </collation>
503 |     <collation id="126" parent="1" name="utf16_hungarian_ci">
504 |       <Charset>utf16</Charset>
505 |       <DefaultForCharset>0</DefaultForCharset>
506 |     </collation>
507 |     <collation id="127" parent="1" name="utf16_icelandic_ci">
508 |       <Charset>utf16</Charset>
509 |       <DefaultForCharset>0</DefaultForCharset>
510 |     </collation>
511 |     <collation id="128" parent="1" name="utf16_latvian_ci">
512 |       <Charset>utf16</Charset>
513 |       <DefaultForCharset>0</DefaultForCharset>
514 |     </collation>
515 |     <collation id="129" parent="1" name="utf16_lithuanian_ci">
516 |       <Charset>utf16</Charset>
517 |       <DefaultForCharset>0</DefaultForCharset>
518 |     </collation>
519 |     <collation id="130" parent="1" name="utf16_persian_ci">
520 |       <Charset>utf16</Charset>
521 |       <DefaultForCharset>0</DefaultForCharset>
522 |     </collation>
523 |     <collation id="131" parent="1" name="utf16_polish_ci">
524 |       <Charset>utf16</Charset>
525 |       <DefaultForCharset>0</DefaultForCharset>
526 |     </collation>
527 |     <collation id="132" parent="1" name="utf16_roman_ci">
528 |       <Charset>utf16</Charset>
529 |       <DefaultForCharset>0</DefaultForCharset>
530 |     </collation>
531 |     <collation id="133" parent="1" name="utf16_romanian_ci">
532 |       <Charset>utf16</Charset>
533 |       <DefaultForCharset>0</DefaultForCharset>
534 |     </collation>
535 |     <collation id="134" parent="1" name="utf16_sinhala_ci">
536 |       <Charset>utf16</Charset>
537 |       <DefaultForCharset>0</DefaultForCharset>
538 |     </collation>
539 |     <collation id="135" parent="1" name="utf16_slovak_ci">
540 |       <Charset>utf16</Charset>
541 |       <DefaultForCharset>0</DefaultForCharset>
542 |     </collation>
543 |     <collation id="136" parent="1" name="utf16_slovenian_ci">
544 |       <Charset>utf16</Charset>
545 |       <DefaultForCharset>0</DefaultForCharset>
546 |     </collation>
547 |     <collation id="137" parent="1" name="utf16_spanish2_ci">
548 |       <Charset>utf16</Charset>
549 |       <DefaultForCharset>0</DefaultForCharset>
550 |     </collation>
551 |     <collation id="138" parent="1" name="utf16_spanish_ci">
552 |       <Charset>utf16</Charset>
553 |       <DefaultForCharset>0</DefaultForCharset>
554 |     </collation>
555 |     <collation id="139" parent="1" name="utf16_swedish_ci">
556 |       <Charset>utf16</Charset>
557 |       <DefaultForCharset>0</DefaultForCharset>
558 |     </collation>
559 |     <collation id="140" parent="1" name="utf16_turkish_ci">
560 |       <Charset>utf16</Charset>
561 |       <DefaultForCharset>0</DefaultForCharset>
562 |     </collation>
563 |     <collation id="141" parent="1" name="utf16_unicode_520_ci">
564 |       <Charset>utf16</Charset>
565 |       <DefaultForCharset>0</DefaultForCharset>
566 |     </collation>
567 |     <collation id="142" parent="1" name="utf16_unicode_ci">
568 |       <Charset>utf16</Charset>
569 |       <DefaultForCharset>0</DefaultForCharset>
570 |     </collation>
571 |     <collation id="143" parent="1" name="utf16_vietnamese_ci">
572 |       <Charset>utf16</Charset>
573 |       <DefaultForCharset>0</DefaultForCharset>
574 |     </collation>
575 |     <collation id="144" parent="1" name="utf16le_bin">
576 |       <Charset>utf16le</Charset>
577 |       <DefaultForCharset>0</DefaultForCharset>
578 |     </collation>
579 |     <collation id="145" parent="1" name="utf16le_general_ci">
580 |       <Charset>utf16le</Charset>
581 |       <DefaultForCharset>1</DefaultForCharset>
582 |     </collation>
583 |     <collation id="146" parent="1" name="utf32_bin">
584 |       <Charset>utf32</Charset>
585 |       <DefaultForCharset>0</DefaultForCharset>
586 |     </collation>
587 |     <collation id="147" parent="1" name="utf32_croatian_ci">
588 |       <Charset>utf32</Charset>
589 |       <DefaultForCharset>0</DefaultForCharset>
590 |     </collation>
591 |     <collation id="148" parent="1" name="utf32_czech_ci">
592 |       <Charset>utf32</Charset>
593 |       <DefaultForCharset>0</DefaultForCharset>
594 |     </collation>
595 |     <collation id="149" parent="1" name="utf32_danish_ci">
596 |       <Charset>utf32</Charset>
597 |       <DefaultForCharset>0</DefaultForCharset>
598 |     </collation>
599 |     <collation id="150" parent="1" name="utf32_esperanto_ci">
600 |       <Charset>utf32</Charset>
601 |       <DefaultForCharset>0</DefaultForCharset>
602 |     </collation>
603 |     <collation id="151" parent="1" name="utf32_estonian_ci">
604 |       <Charset>utf32</Charset>
605 |       <DefaultForCharset>0</DefaultForCharset>
606 |     </collation>
607 |     <collation id="152" parent="1" name="utf32_general_ci">
608 |       <Charset>utf32</Charset>
609 |       <DefaultForCharset>1</DefaultForCharset>
610 |     </collation>
611 |     <collation id="153" parent="1" name="utf32_german2_ci">
612 |       <Charset>utf32</Charset>
613 |       <DefaultForCharset>0</DefaultForCharset>
614 |     </collation>
615 |     <collation id="154" parent="1" name="utf32_hungarian_ci">
616 |       <Charset>utf32</Charset>
617 |       <DefaultForCharset>0</DefaultForCharset>
618 |     </collation>
619 |     <collation id="155" parent="1" name="utf32_icelandic_ci">
620 |       <Charset>utf32</Charset>
621 |       <DefaultForCharset>0</DefaultForCharset>
622 |     </collation>
623 |     <collation id="156" parent="1" name="utf32_latvian_ci">
624 |       <Charset>utf32</Charset>
625 |       <DefaultForCharset>0</DefaultForCharset>
626 |     </collation>
627 |     <collation id="157" parent="1" name="utf32_lithuanian_ci">
628 |       <Charset>utf32</Charset>
629 |       <DefaultForCharset>0</DefaultForCharset>
630 |     </collation>
631 |     <collation id="158" parent="1" name="utf32_persian_ci">
632 |       <Charset>utf32</Charset>
633 |       <DefaultForCharset>0</DefaultForCharset>
634 |     </collation>
635 |     <collation id="159" parent="1" name="utf32_polish_ci">
636 |       <Charset>utf32</Charset>
637 |       <DefaultForCharset>0</DefaultForCharset>
638 |     </collation>
639 |     <collation id="160" parent="1" name="utf32_roman_ci">
640 |       <Charset>utf32</Charset>
641 |       <DefaultForCharset>0</DefaultForCharset>
642 |     </collation>
643 |     <collation id="161" parent="1" name="utf32_romanian_ci">
644 |       <Charset>utf32</Charset>
645 |       <DefaultForCharset>0</DefaultForCharset>
646 |     </collation>
647 |     <collation id="162" parent="1" name="utf32_sinhala_ci">
648 |       <Charset>utf32</Charset>
649 |       <DefaultForCharset>0</DefaultForCharset>
650 |     </collation>
651 |     <collation id="163" parent="1" name="utf32_slovak_ci">
652 |       <Charset>utf32</Charset>
653 |       <DefaultForCharset>0</DefaultForCharset>
654 |     </collation>
655 |     <collation id="164" parent="1" name="utf32_slovenian_ci">
656 |       <Charset>utf32</Charset>
657 |       <DefaultForCharset>0</DefaultForCharset>
658 |     </collation>
659 |     <collation id="165" parent="1" name="utf32_spanish2_ci">
660 |       <Charset>utf32</Charset>
661 |       <DefaultForCharset>0</DefaultForCharset>
662 |     </collation>
663 |     <collation id="166" parent="1" name="utf32_spanish_ci">
664 |       <Charset>utf32</Charset>
665 |       <DefaultForCharset>0</DefaultForCharset>
666 |     </collation>
667 |     <collation id="167" parent="1" name="utf32_swedish_ci">
668 |       <Charset>utf32</Charset>
669 |       <DefaultForCharset>0</DefaultForCharset>
670 |     </collation>
671 |     <collation id="168" parent="1" name="utf32_turkish_ci">
672 |       <Charset>utf32</Charset>
673 |       <DefaultForCharset>0</DefaultForCharset>
674 |     </collation>
675 |     <collation id="169" parent="1" name="utf32_unicode_520_ci">
676 |       <Charset>utf32</Charset>
677 |       <DefaultForCharset>0</DefaultForCharset>
678 |     </collation>
679 |     <collation id="170" parent="1" name="utf32_unicode_ci">
680 |       <Charset>utf32</Charset>
681 |       <DefaultForCharset>0</DefaultForCharset>
682 |     </collation>
683 |     <collation id="171" parent="1" name="utf32_vietnamese_ci">
684 |       <Charset>utf32</Charset>
685 |       <DefaultForCharset>0</DefaultForCharset>
686 |     </collation>
687 |     <collation id="172" parent="1" name="utf8_bin">
688 |       <Charset>utf8</Charset>
689 |       <DefaultForCharset>0</DefaultForCharset>
690 |     </collation>
691 |     <collation id="173" parent="1" name="utf8_croatian_ci">
692 |       <Charset>utf8</Charset>
693 |       <DefaultForCharset>0</DefaultForCharset>
694 |     </collation>
695 |     <collation id="174" parent="1" name="utf8_czech_ci">
696 |       <Charset>utf8</Charset>
697 |       <DefaultForCharset>0</DefaultForCharset>
698 |     </collation>
699 |     <collation id="175" parent="1" name="utf8_danish_ci">
700 |       <Charset>utf8</Charset>
701 |       <DefaultForCharset>0</DefaultForCharset>
702 |     </collation>
703 |     <collation id="176" parent="1" name="utf8_esperanto_ci">
704 |       <Charset>utf8</Charset>
705 |       <DefaultForCharset>0</DefaultForCharset>
706 |     </collation>
707 |     <collation id="177" parent="1" name="utf8_estonian_ci">
708 |       <Charset>utf8</Charset>
709 |       <DefaultForCharset>0</DefaultForCharset>
710 |     </collation>
711 |     <collation id="178" parent="1" name="utf8_general_ci">
712 |       <Charset>utf8</Charset>
713 |       <DefaultForCharset>1</DefaultForCharset>
714 |     </collation>
715 |     <collation id="179" parent="1" name="utf8_general_mysql500_ci">
716 |       <Charset>utf8</Charset>
717 |       <DefaultForCharset>0</DefaultForCharset>
718 |     </collation>
719 |     <collation id="180" parent="1" name="utf8_german2_ci">
720 |       <Charset>utf8</Charset>
721 |       <DefaultForCharset>0</DefaultForCharset>
722 |     </collation>
723 |     <collation id="181" parent="1" name="utf8_hungarian_ci">
724 |       <Charset>utf8</Charset>
725 |       <DefaultForCharset>0</DefaultForCharset>
726 |     </collation>
727 |     <collation id="182" parent="1" name="utf8_icelandic_ci">
728 |       <Charset>utf8</Charset>
729 |       <DefaultForCharset>0</DefaultForCharset>
730 |     </collation>
731 |     <collation id="183" parent="1" name="utf8_latvian_ci">
732 |       <Charset>utf8</Charset>
733 |       <DefaultForCharset>0</DefaultForCharset>
734 |     </collation>
735 |     <collation id="184" parent="1" name="utf8_lithuanian_ci">
736 |       <Charset>utf8</Charset>
737 |       <DefaultForCharset>0</DefaultForCharset>
738 |     </collation>
739 |     <collation id="185" parent="1" name="utf8_persian_ci">
740 |       <Charset>utf8</Charset>
741 |       <DefaultForCharset>0</DefaultForCharset>
742 |     </collation>
743 |     <collation id="186" parent="1" name="utf8_polish_ci">
744 |       <Charset>utf8</Charset>
745 |       <DefaultForCharset>0</DefaultForCharset>
746 |     </collation>
747 |     <collation id="187" parent="1" name="utf8_roman_ci">
748 |       <Charset>utf8</Charset>
749 |       <DefaultForCharset>0</DefaultForCharset>
750 |     </collation>
751 |     <collation id="188" parent="1" name="utf8_romanian_ci">
752 |       <Charset>utf8</Charset>
753 |       <DefaultForCharset>0</DefaultForCharset>
754 |     </collation>
755 |     <collation id="189" parent="1" name="utf8_sinhala_ci">
756 |       <Charset>utf8</Charset>
757 |       <DefaultForCharset>0</DefaultForCharset>
758 |     </collation>
759 |     <collation id="190" parent="1" name="utf8_slovak_ci">
760 |       <Charset>utf8</Charset>
761 |       <DefaultForCharset>0</DefaultForCharset>
762 |     </collation>
763 |     <collation id="191" parent="1" name="utf8_slovenian_ci">
764 |       <Charset>utf8</Charset>
765 |       <DefaultForCharset>0</DefaultForCharset>
766 |     </collation>
767 |     <collation id="192" parent="1" name="utf8_spanish2_ci">
768 |       <Charset>utf8</Charset>
769 |       <DefaultForCharset>0</DefaultForCharset>
770 |     </collation>
771 |     <collation id="193" parent="1" name="utf8_spanish_ci">
772 |       <Charset>utf8</Charset>
773 |       <DefaultForCharset>0</DefaultForCharset>
774 |     </collation>
775 |     <collation id="194" parent="1" name="utf8_swedish_ci">
776 |       <Charset>utf8</Charset>
777 |       <DefaultForCharset>0</DefaultForCharset>
778 |     </collation>
779 |     <collation id="195" parent="1" name="utf8_turkish_ci">
780 |       <Charset>utf8</Charset>
781 |       <DefaultForCharset>0</DefaultForCharset>
782 |     </collation>
783 |     <collation id="196" parent="1" name="utf8_unicode_520_ci">
784 |       <Charset>utf8</Charset>
785 |       <DefaultForCharset>0</DefaultForCharset>
786 |     </collation>
787 |     <collation id="197" parent="1" name="utf8_unicode_ci">
788 |       <Charset>utf8</Charset>
789 |       <DefaultForCharset>0</DefaultForCharset>
790 |     </collation>
791 |     <collation id="198" parent="1" name="utf8_vietnamese_ci">
792 |       <Charset>utf8</Charset>
793 |       <DefaultForCharset>0</DefaultForCharset>
794 |     </collation>
795 |     <collation id="199" parent="1" name="utf8mb4_bin">
796 |       <Charset>utf8mb4</Charset>
797 |       <DefaultForCharset>0</DefaultForCharset>
798 |     </collation>
799 |     <collation id="200" parent="1" name="utf8mb4_croatian_ci">
800 |       <Charset>utf8mb4</Charset>
801 |       <DefaultForCharset>0</DefaultForCharset>
802 |     </collation>
803 |     <collation id="201" parent="1" name="utf8mb4_czech_ci">
804 |       <Charset>utf8mb4</Charset>
805 |       <DefaultForCharset>0</DefaultForCharset>
806 |     </collation>
807 |     <collation id="202" parent="1" name="utf8mb4_danish_ci">
808 |       <Charset>utf8mb4</Charset>
809 |       <DefaultForCharset>0</DefaultForCharset>
810 |     </collation>
811 |     <collation id="203" parent="1" name="utf8mb4_esperanto_ci">
812 |       <Charset>utf8mb4</Charset>
813 |       <DefaultForCharset>0</DefaultForCharset>
814 |     </collation>
815 |     <collation id="204" parent="1" name="utf8mb4_estonian_ci">
816 |       <Charset>utf8mb4</Charset>
817 |       <DefaultForCharset>0</DefaultForCharset>
818 |     </collation>
819 |     <collation id="205" parent="1" name="utf8mb4_general_ci">
820 |       <Charset>utf8mb4</Charset>
821 |       <DefaultForCharset>1</DefaultForCharset>
822 |     </collation>
823 |     <collation id="206" parent="1" name="utf8mb4_german2_ci">
824 |       <Charset>utf8mb4</Charset>
825 |       <DefaultForCharset>0</DefaultForCharset>
826 |     </collation>
827 |     <collation id="207" parent="1" name="utf8mb4_hungarian_ci">
828 |       <Charset>utf8mb4</Charset>
829 |       <DefaultForCharset>0</DefaultForCharset>
830 |     </collation>
831 |     <collation id="208" parent="1" name="utf8mb4_icelandic_ci">
832 |       <Charset>utf8mb4</Charset>
833 |       <DefaultForCharset>0</DefaultForCharset>
834 |     </collation>
835 |     <collation id="209" parent="1" name="utf8mb4_latvian_ci">
836 |       <Charset>utf8mb4</Charset>
837 |       <DefaultForCharset>0</DefaultForCharset>
838 |     </collation>
839 |     <collation id="210" parent="1" name="utf8mb4_lithuanian_ci">
840 |       <Charset>utf8mb4</Charset>
841 |       <DefaultForCharset>0</DefaultForCharset>
842 |     </collation>
843 |     <collation id="211" parent="1" name="utf8mb4_persian_ci">
844 |       <Charset>utf8mb4</Charset>
845 |       <DefaultForCharset>0</DefaultForCharset>
846 |     </collation>
847 |     <collation id="212" parent="1" name="utf8mb4_polish_ci">
848 |       <Charset>utf8mb4</Charset>
849 |       <DefaultForCharset>0</DefaultForCharset>
850 |     </collation>
851 |     <collation id="213" parent="1" name="utf8mb4_roman_ci">
852 |       <Charset>utf8mb4</Charset>
853 |       <DefaultForCharset>0</DefaultForCharset>
854 |     </collation>
855 |     <collation id="214" parent="1" name="utf8mb4_romanian_ci">
856 |       <Charset>utf8mb4</Charset>
857 |       <DefaultForCharset>0</DefaultForCharset>
858 |     </collation>
859 |     <collation id="215" parent="1" name="utf8mb4_sinhala_ci">
860 |       <Charset>utf8mb4</Charset>
861 |       <DefaultForCharset>0</DefaultForCharset>
862 |     </collation>
863 |     <collation id="216" parent="1" name="utf8mb4_slovak_ci">
864 |       <Charset>utf8mb4</Charset>
865 |       <DefaultForCharset>0</DefaultForCharset>
866 |     </collation>
867 |     <collation id="217" parent="1" name="utf8mb4_slovenian_ci">
868 |       <Charset>utf8mb4</Charset>
869 |       <DefaultForCharset>0</DefaultForCharset>
870 |     </collation>
871 |     <collation id="218" parent="1" name="utf8mb4_spanish2_ci">
872 |       <Charset>utf8mb4</Charset>
873 |       <DefaultForCharset>0</DefaultForCharset>
874 |     </collation>
875 |     <collation id="219" parent="1" name="utf8mb4_spanish_ci">
876 |       <Charset>utf8mb4</Charset>
877 |       <DefaultForCharset>0</DefaultForCharset>
878 |     </collation>
879 |     <collation id="220" parent="1" name="utf8mb4_swedish_ci">
880 |       <Charset>utf8mb4</Charset>
881 |       <DefaultForCharset>0</DefaultForCharset>
882 |     </collation>
883 |     <collation id="221" parent="1" name="utf8mb4_turkish_ci">
884 |       <Charset>utf8mb4</Charset>
885 |       <DefaultForCharset>0</DefaultForCharset>
886 |     </collation>
887 |     <collation id="222" parent="1" name="utf8mb4_unicode_520_ci">
888 |       <Charset>utf8mb4</Charset>
889 |       <DefaultForCharset>0</DefaultForCharset>
890 |     </collation>
891 |     <collation id="223" parent="1" name="utf8mb4_unicode_ci">
892 |       <Charset>utf8mb4</Charset>
893 |       <DefaultForCharset>0</DefaultForCharset>
894 |     </collation>
895 |     <collation id="224" parent="1" name="utf8mb4_vietnamese_ci">
896 |       <Charset>utf8mb4</Charset>
897 |       <DefaultForCharset>0</DefaultForCharset>
898 |     </collation>
899 |   </database-model>
900 | </dataSource>


--------------------------------------------------------------------------------
/.idea/dataSources/eb023117-f844-4ce6-9dbd-5a03c863f17a.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <dataSource name="awake_test">
  3 |   <database-model serializer="dbm" dbms="MYSQL" family-id="MYSQL" format-version="4.15">
  4 |     <root id="1">
  5 |       <ServerVersion>5.6.43</ServerVersion>
  6 |       <DefaultEngine>InnoDB</DefaultEngine>
  7 |       <DefaultTmpEngine>InnoDB</DefaultTmpEngine>
  8 |       <DefaultCasing>exact</DefaultCasing>
  9 |     </root>
 10 |     <schema id="2" parent="1" name="confluence">
 11 |       <Collation>utf8_bin</Collation>
 12 |     </schema>
 13 |     <schema id="3" parent="1" name="file">
 14 |       <Collation>utf8_general_ci</Collation>
 15 |     </schema>
 16 |     <schema id="4" parent="1" name="information_schema">
 17 |       <Collation>utf8_general_ci</Collation>
 18 |     </schema>
 19 |     <schema id="5" parent="1" name="jira">
 20 |       <Collation>utf8_bin</Collation>
 21 |     </schema>
 22 |     <schema id="6" parent="1" name="jx">
 23 |       <Current>1</Current>
 24 |       <Collation>utf8_general_ci</Collation>
 25 |     </schema>
 26 |     <schema id="7" parent="1" name="mysql">
 27 |       <Collation>utf8_general_ci</Collation>
 28 |     </schema>
 29 |     <schema id="8" parent="1" name="performance_schema">
 30 |       <Collation>utf8_general_ci</Collation>
 31 |     </schema>
 32 |     <collation id="9" parent="1" name="armscii8_bin">
 33 |       <Charset>armscii8</Charset>
 34 |       <DefaultForCharset>0</DefaultForCharset>
 35 |     </collation>
 36 |     <collation id="10" parent="1" name="armscii8_general_ci">
 37 |       <Charset>armscii8</Charset>
 38 |       <DefaultForCharset>1</DefaultForCharset>
 39 |     </collation>
 40 |     <collation id="11" parent="1" name="ascii_bin">
 41 |       <Charset>ascii</Charset>
 42 |       <DefaultForCharset>0</DefaultForCharset>
 43 |     </collation>
 44 |     <collation id="12" parent="1" name="ascii_general_ci">
 45 |       <Charset>ascii</Charset>
 46 |       <DefaultForCharset>1</DefaultForCharset>
 47 |     </collation>
 48 |     <collation id="13" parent="1" name="big5_bin">
 49 |       <Charset>big5</Charset>
 50 |       <DefaultForCharset>0</DefaultForCharset>
 51 |     </collation>
 52 |     <collation id="14" parent="1" name="big5_chinese_ci">
 53 |       <Charset>big5</Charset>
 54 |       <DefaultForCharset>1</DefaultForCharset>
 55 |     </collation>
 56 |     <collation id="15" parent="1" name="binary">
 57 |       <Charset>binary</Charset>
 58 |       <DefaultForCharset>1</DefaultForCharset>
 59 |     </collation>
 60 |     <collation id="16" parent="1" name="cp1250_bin">
 61 |       <Charset>cp1250</Charset>
 62 |       <DefaultForCharset>0</DefaultForCharset>
 63 |     </collation>
 64 |     <collation id="17" parent="1" name="cp1250_croatian_ci">
 65 |       <Charset>cp1250</Charset>
 66 |       <DefaultForCharset>0</DefaultForCharset>
 67 |     </collation>
 68 |     <collation id="18" parent="1" name="cp1250_czech_cs">
 69 |       <Charset>cp1250</Charset>
 70 |       <DefaultForCharset>0</DefaultForCharset>
 71 |     </collation>
 72 |     <collation id="19" parent="1" name="cp1250_general_ci">
 73 |       <Charset>cp1250</Charset>
 74 |       <DefaultForCharset>1</DefaultForCharset>
 75 |     </collation>
 76 |     <collation id="20" parent="1" name="cp1250_polish_ci">
 77 |       <Charset>cp1250</Charset>
 78 |       <DefaultForCharset>0</DefaultForCharset>
 79 |     </collation>
 80 |     <collation id="21" parent="1" name="cp1251_bin">
 81 |       <Charset>cp1251</Charset>
 82 |       <DefaultForCharset>0</DefaultForCharset>
 83 |     </collation>
 84 |     <collation id="22" parent="1" name="cp1251_bulgarian_ci">
 85 |       <Charset>cp1251</Charset>
 86 |       <DefaultForCharset>0</DefaultForCharset>
 87 |     </collation>
 88 |     <collation id="23" parent="1" name="cp1251_general_ci">
 89 |       <Charset>cp1251</Charset>
 90 |       <DefaultForCharset>1</DefaultForCharset>
 91 |     </collation>
 92 |     <collation id="24" parent="1" name="cp1251_general_cs">
 93 |       <Charset>cp1251</Charset>
 94 |       <DefaultForCharset>0</DefaultForCharset>
 95 |     </collation>
 96 |     <collation id="25" parent="1" name="cp1251_ukrainian_ci">
 97 |       <Charset>cp1251</Charset>
 98 |       <DefaultForCharset>0</DefaultForCharset>
 99 |     </collation>
100 |     <collation id="26" parent="1" name="cp1256_bin">
101 |       <Charset>cp1256</Charset>
102 |       <DefaultForCharset>0</DefaultForCharset>
103 |     </collation>
104 |     <collation id="27" parent="1" name="cp1256_general_ci">
105 |       <Charset>cp1256</Charset>
106 |       <DefaultForCharset>1</DefaultForCharset>
107 |     </collation>
108 |     <collation id="28" parent="1" name="cp1257_bin">
109 |       <Charset>cp1257</Charset>
110 |       <DefaultForCharset>0</DefaultForCharset>
111 |     </collation>
112 |     <collation id="29" parent="1" name="cp1257_general_ci">
113 |       <Charset>cp1257</Charset>
114 |       <DefaultForCharset>1</DefaultForCharset>
115 |     </collation>
116 |     <collation id="30" parent="1" name="cp1257_lithuanian_ci">
117 |       <Charset>cp1257</Charset>
118 |       <DefaultForCharset>0</DefaultForCharset>
119 |     </collation>
120 |     <collation id="31" parent="1" name="cp850_bin">
121 |       <Charset>cp850</Charset>
122 |       <DefaultForCharset>0</DefaultForCharset>
123 |     </collation>
124 |     <collation id="32" parent="1" name="cp850_general_ci">
125 |       <Charset>cp850</Charset>
126 |       <DefaultForCharset>1</DefaultForCharset>
127 |     </collation>
128 |     <collation id="33" parent="1" name="cp852_bin">
129 |       <Charset>cp852</Charset>
130 |       <DefaultForCharset>0</DefaultForCharset>
131 |     </collation>
132 |     <collation id="34" parent="1" name="cp852_general_ci">
133 |       <Charset>cp852</Charset>
134 |       <DefaultForCharset>1</DefaultForCharset>
135 |     </collation>
136 |     <collation id="35" parent="1" name="cp866_bin">
137 |       <Charset>cp866</Charset>
138 |       <DefaultForCharset>0</DefaultForCharset>
139 |     </collation>
140 |     <collation id="36" parent="1" name="cp866_general_ci">
141 |       <Charset>cp866</Charset>
142 |       <DefaultForCharset>1</DefaultForCharset>
143 |     </collation>
144 |     <collation id="37" parent="1" name="cp932_bin">
145 |       <Charset>cp932</Charset>
146 |       <DefaultForCharset>0</DefaultForCharset>
147 |     </collation>
148 |     <collation id="38" parent="1" name="cp932_japanese_ci">
149 |       <Charset>cp932</Charset>
150 |       <DefaultForCharset>1</DefaultForCharset>
151 |     </collation>
152 |     <collation id="39" parent="1" name="dec8_bin">
153 |       <Charset>dec8</Charset>
154 |       <DefaultForCharset>0</DefaultForCharset>
155 |     </collation>
156 |     <collation id="40" parent="1" name="dec8_swedish_ci">
157 |       <Charset>dec8</Charset>
158 |       <DefaultForCharset>1</DefaultForCharset>
159 |     </collation>
160 |     <collation id="41" parent="1" name="eucjpms_bin">
161 |       <Charset>eucjpms</Charset>
162 |       <DefaultForCharset>0</DefaultForCharset>
163 |     </collation>
164 |     <collation id="42" parent="1" name="eucjpms_japanese_ci">
165 |       <Charset>eucjpms</Charset>
166 |       <DefaultForCharset>1</DefaultForCharset>
167 |     </collation>
168 |     <collation id="43" parent="1" name="euckr_bin">
169 |       <Charset>euckr</Charset>
170 |       <DefaultForCharset>0</DefaultForCharset>
171 |     </collation>
172 |     <collation id="44" parent="1" name="euckr_korean_ci">
173 |       <Charset>euckr</Charset>
174 |       <DefaultForCharset>1</DefaultForCharset>
175 |     </collation>
176 |     <collation id="45" parent="1" name="gb2312_bin">
177 |       <Charset>gb2312</Charset>
178 |       <DefaultForCharset>0</DefaultForCharset>
179 |     </collation>
180 |     <collation id="46" parent="1" name="gb2312_chinese_ci">
181 |       <Charset>gb2312</Charset>
182 |       <DefaultForCharset>1</DefaultForCharset>
183 |     </collation>
184 |     <collation id="47" parent="1" name="gbk_bin">
185 |       <Charset>gbk</Charset>
186 |       <DefaultForCharset>0</DefaultForCharset>
187 |     </collation>
188 |     <collation id="48" parent="1" name="gbk_chinese_ci">
189 |       <Charset>gbk</Charset>
190 |       <DefaultForCharset>1</DefaultForCharset>
191 |     </collation>
192 |     <collation id="49" parent="1" name="geostd8_bin">
193 |       <Charset>geostd8</Charset>
194 |       <DefaultForCharset>0</DefaultForCharset>
195 |     </collation>
196 |     <collation id="50" parent="1" name="geostd8_general_ci">
197 |       <Charset>geostd8</Charset>
198 |       <DefaultForCharset>1</DefaultForCharset>
199 |     </collation>
200 |     <collation id="51" parent="1" name="greek_bin">
201 |       <Charset>greek</Charset>
202 |       <DefaultForCharset>0</DefaultForCharset>
203 |     </collation>
204 |     <collation id="52" parent="1" name="greek_general_ci">
205 |       <Charset>greek</Charset>
206 |       <DefaultForCharset>1</DefaultForCharset>
207 |     </collation>
208 |     <collation id="53" parent="1" name="hebrew_bin">
209 |       <Charset>hebrew</Charset>
210 |       <DefaultForCharset>0</DefaultForCharset>
211 |     </collation>
212 |     <collation id="54" parent="1" name="hebrew_general_ci">
213 |       <Charset>hebrew</Charset>
214 |       <DefaultForCharset>1</DefaultForCharset>
215 |     </collation>
216 |     <collation id="55" parent="1" name="hp8_bin">
217 |       <Charset>hp8</Charset>
218 |       <DefaultForCharset>0</DefaultForCharset>
219 |     </collation>
220 |     <collation id="56" parent="1" name="hp8_english_ci">
221 |       <Charset>hp8</Charset>
222 |       <DefaultForCharset>1</DefaultForCharset>
223 |     </collation>
224 |     <collation id="57" parent="1" name="keybcs2_bin">
225 |       <Charset>keybcs2</Charset>
226 |       <DefaultForCharset>0</DefaultForCharset>
227 |     </collation>
228 |     <collation id="58" parent="1" name="keybcs2_general_ci">
229 |       <Charset>keybcs2</Charset>
230 |       <DefaultForCharset>1</DefaultForCharset>
231 |     </collation>
232 |     <collation id="59" parent="1" name="koi8r_bin">
233 |       <Charset>koi8r</Charset>
234 |       <DefaultForCharset>0</DefaultForCharset>
235 |     </collation>
236 |     <collation id="60" parent="1" name="koi8r_general_ci">
237 |       <Charset>koi8r</Charset>
238 |       <DefaultForCharset>1</DefaultForCharset>
239 |     </collation>
240 |     <collation id="61" parent="1" name="koi8u_bin">
241 |       <Charset>koi8u</Charset>
242 |       <DefaultForCharset>0</DefaultForCharset>
243 |     </collation>
244 |     <collation id="62" parent="1" name="koi8u_general_ci">
245 |       <Charset>koi8u</Charset>
246 |       <DefaultForCharset>1</DefaultForCharset>
247 |     </collation>
248 |     <collation id="63" parent="1" name="latin1_bin">
249 |       <Charset>latin1</Charset>
250 |       <DefaultForCharset>0</DefaultForCharset>
251 |     </collation>
252 |     <collation id="64" parent="1" name="latin1_danish_ci">
253 |       <Charset>latin1</Charset>
254 |       <DefaultForCharset>0</DefaultForCharset>
255 |     </collation>
256 |     <collation id="65" parent="1" name="latin1_general_ci">
257 |       <Charset>latin1</Charset>
258 |       <DefaultForCharset>0</DefaultForCharset>
259 |     </collation>
260 |     <collation id="66" parent="1" name="latin1_general_cs">
261 |       <Charset>latin1</Charset>
262 |       <DefaultForCharset>0</DefaultForCharset>
263 |     </collation>
264 |     <collation id="67" parent="1" name="latin1_german1_ci">
265 |       <Charset>latin1</Charset>
266 |       <DefaultForCharset>0</DefaultForCharset>
267 |     </collation>
268 |     <collation id="68" parent="1" name="latin1_german2_ci">
269 |       <Charset>latin1</Charset>
270 |       <DefaultForCharset>0</DefaultForCharset>
271 |     </collation>
272 |     <collation id="69" parent="1" name="latin1_spanish_ci">
273 |       <Charset>latin1</Charset>
274 |       <DefaultForCharset>0</DefaultForCharset>
275 |     </collation>
276 |     <collation id="70" parent="1" name="latin1_swedish_ci">
277 |       <Charset>latin1</Charset>
278 |       <DefaultForCharset>1</DefaultForCharset>
279 |     </collation>
280 |     <collation id="71" parent="1" name="latin2_bin">
281 |       <Charset>latin2</Charset>
282 |       <DefaultForCharset>0</DefaultForCharset>
283 |     </collation>
284 |     <collation id="72" parent="1" name="latin2_croatian_ci">
285 |       <Charset>latin2</Charset>
286 |       <DefaultForCharset>0</DefaultForCharset>
287 |     </collation>
288 |     <collation id="73" parent="1" name="latin2_czech_cs">
289 |       <Charset>latin2</Charset>
290 |       <DefaultForCharset>0</DefaultForCharset>
291 |     </collation>
292 |     <collation id="74" parent="1" name="latin2_general_ci">
293 |       <Charset>latin2</Charset>
294 |       <DefaultForCharset>1</DefaultForCharset>
295 |     </collation>
296 |     <collation id="75" parent="1" name="latin2_hungarian_ci">
297 |       <Charset>latin2</Charset>
298 |       <DefaultForCharset>0</DefaultForCharset>
299 |     </collation>
300 |     <collation id="76" parent="1" name="latin5_bin">
301 |       <Charset>latin5</Charset>
302 |       <DefaultForCharset>0</DefaultForCharset>
303 |     </collation>
304 |     <collation id="77" parent="1" name="latin5_turkish_ci">
305 |       <Charset>latin5</Charset>
306 |       <DefaultForCharset>1</DefaultForCharset>
307 |     </collation>
308 |     <collation id="78" parent="1" name="latin7_bin">
309 |       <Charset>latin7</Charset>
310 |       <DefaultForCharset>0</DefaultForCharset>
311 |     </collation>
312 |     <collation id="79" parent="1" name="latin7_estonian_cs">
313 |       <Charset>latin7</Charset>
314 |       <DefaultForCharset>0</DefaultForCharset>
315 |     </collation>
316 |     <collation id="80" parent="1" name="latin7_general_ci">
317 |       <Charset>latin7</Charset>
318 |       <DefaultForCharset>1</DefaultForCharset>
319 |     </collation>
320 |     <collation id="81" parent="1" name="latin7_general_cs">
321 |       <Charset>latin7</Charset>
322 |       <DefaultForCharset>0</DefaultForCharset>
323 |     </collation>
324 |     <collation id="82" parent="1" name="macce_bin">
325 |       <Charset>macce</Charset>
326 |       <DefaultForCharset>0</DefaultForCharset>
327 |     </collation>
328 |     <collation id="83" parent="1" name="macce_general_ci">
329 |       <Charset>macce</Charset>
330 |       <DefaultForCharset>1</DefaultForCharset>
331 |     </collation>
332 |     <collation id="84" parent="1" name="macroman_bin">
333 |       <Charset>macroman</Charset>
334 |       <DefaultForCharset>0</DefaultForCharset>
335 |     </collation>
336 |     <collation id="85" parent="1" name="macroman_general_ci">
337 |       <Charset>macroman</Charset>
338 |       <DefaultForCharset>1</DefaultForCharset>
339 |     </collation>
340 |     <collation id="86" parent="1" name="sjis_bin">
341 |       <Charset>sjis</Charset>
342 |       <DefaultForCharset>0</DefaultForCharset>
343 |     </collation>
344 |     <collation id="87" parent="1" name="sjis_japanese_ci">
345 |       <Charset>sjis</Charset>
346 |       <DefaultForCharset>1</DefaultForCharset>
347 |     </collation>
348 |     <collation id="88" parent="1" name="swe7_bin">
349 |       <Charset>swe7</Charset>
350 |       <DefaultForCharset>0</DefaultForCharset>
351 |     </collation>
352 |     <collation id="89" parent="1" name="swe7_swedish_ci">
353 |       <Charset>swe7</Charset>
354 |       <DefaultForCharset>1</DefaultForCharset>
355 |     </collation>
356 |     <collation id="90" parent="1" name="tis620_bin">
357 |       <Charset>tis620</Charset>
358 |       <DefaultForCharset>0</DefaultForCharset>
359 |     </collation>
360 |     <collation id="91" parent="1" name="tis620_thai_ci">
361 |       <Charset>tis620</Charset>
362 |       <DefaultForCharset>1</DefaultForCharset>
363 |     </collation>
364 |     <collation id="92" parent="1" name="ucs2_bin">
365 |       <Charset>ucs2</Charset>
366 |       <DefaultForCharset>0</DefaultForCharset>
367 |     </collation>
368 |     <collation id="93" parent="1" name="ucs2_croatian_ci">
369 |       <Charset>ucs2</Charset>
370 |       <DefaultForCharset>0</DefaultForCharset>
371 |     </collation>
372 |     <collation id="94" parent="1" name="ucs2_czech_ci">
373 |       <Charset>ucs2</Charset>
374 |       <DefaultForCharset>0</DefaultForCharset>
375 |     </collation>
376 |     <collation id="95" parent="1" name="ucs2_danish_ci">
377 |       <Charset>ucs2</Charset>
378 |       <DefaultForCharset>0</DefaultForCharset>
379 |     </collation>
380 |     <collation id="96" parent="1" name="ucs2_esperanto_ci">
381 |       <Charset>ucs2</Charset>
382 |       <DefaultForCharset>0</DefaultForCharset>
383 |     </collation>
384 |     <collation id="97" parent="1" name="ucs2_estonian_ci">
385 |       <Charset>ucs2</Charset>
386 |       <DefaultForCharset>0</DefaultForCharset>
387 |     </collation>
388 |     <collation id="98" parent="1" name="ucs2_general_ci">
389 |       <Charset>ucs2</Charset>
390 |       <DefaultForCharset>1</DefaultForCharset>
391 |     </collation>
392 |     <collation id="99" parent="1" name="ucs2_general_mysql500_ci">
393 |       <Charset>ucs2</Charset>
394 |       <DefaultForCharset>0</DefaultForCharset>
395 |     </collation>
396 |     <collation id="100" parent="1" name="ucs2_german2_ci">
397 |       <Charset>ucs2</Charset>
398 |       <DefaultForCharset>0</DefaultForCharset>
399 |     </collation>
400 |     <collation id="101" parent="1" name="ucs2_hungarian_ci">
401 |       <Charset>ucs2</Charset>
402 |       <DefaultForCharset>0</DefaultForCharset>
403 |     </collation>
404 |     <collation id="102" parent="1" name="ucs2_icelandic_ci">
405 |       <Charset>ucs2</Charset>
406 |       <DefaultForCharset>0</DefaultForCharset>
407 |     </collation>
408 |     <collation id="103" parent="1" name="ucs2_latvian_ci">
409 |       <Charset>ucs2</Charset>
410 |       <DefaultForCharset>0</DefaultForCharset>
411 |     </collation>
412 |     <collation id="104" parent="1" name="ucs2_lithuanian_ci">
413 |       <Charset>ucs2</Charset>
414 |       <DefaultForCharset>0</DefaultForCharset>
415 |     </collation>
416 |     <collation id="105" parent="1" name="ucs2_persian_ci">
417 |       <Charset>ucs2</Charset>
418 |       <DefaultForCharset>0</DefaultForCharset>
419 |     </collation>
420 |     <collation id="106" parent="1" name="ucs2_polish_ci">
421 |       <Charset>ucs2</Charset>
422 |       <DefaultForCharset>0</DefaultForCharset>
423 |     </collation>
424 |     <collation id="107" parent="1" name="ucs2_roman_ci">
425 |       <Charset>ucs2</Charset>
426 |       <DefaultForCharset>0</DefaultForCharset>
427 |     </collation>
428 |     <collation id="108" parent="1" name="ucs2_romanian_ci">
429 |       <Charset>ucs2</Charset>
430 |       <DefaultForCharset>0</DefaultForCharset>
431 |     </collation>
432 |     <collation id="109" parent="1" name="ucs2_sinhala_ci">
433 |       <Charset>ucs2</Charset>
434 |       <DefaultForCharset>0</DefaultForCharset>
435 |     </collation>
436 |     <collation id="110" parent="1" name="ucs2_slovak_ci">
437 |       <Charset>ucs2</Charset>
438 |       <DefaultForCharset>0</DefaultForCharset>
439 |     </collation>
440 |     <collation id="111" parent="1" name="ucs2_slovenian_ci">
441 |       <Charset>ucs2</Charset>
442 |       <DefaultForCharset>0</DefaultForCharset>
443 |     </collation>
444 |     <collation id="112" parent="1" name="ucs2_spanish2_ci">
445 |       <Charset>ucs2</Charset>
446 |       <DefaultForCharset>0</DefaultForCharset>
447 |     </collation>
448 |     <collation id="113" parent="1" name="ucs2_spanish_ci">
449 |       <Charset>ucs2</Charset>
450 |       <DefaultForCharset>0</DefaultForCharset>
451 |     </collation>
452 |     <collation id="114" parent="1" name="ucs2_swedish_ci">
453 |       <Charset>ucs2</Charset>
454 |       <DefaultForCharset>0</DefaultForCharset>
455 |     </collation>
456 |     <collation id="115" parent="1" name="ucs2_turkish_ci">
457 |       <Charset>ucs2</Charset>
458 |       <DefaultForCharset>0</DefaultForCharset>
459 |     </collation>
460 |     <collation id="116" parent="1" name="ucs2_unicode_520_ci">
461 |       <Charset>ucs2</Charset>
462 |       <DefaultForCharset>0</DefaultForCharset>
463 |     </collation>
464 |     <collation id="117" parent="1" name="ucs2_unicode_ci">
465 |       <Charset>ucs2</Charset>
466 |       <DefaultForCharset>0</DefaultForCharset>
467 |     </collation>
468 |     <collation id="118" parent="1" name="ucs2_vietnamese_ci">
469 |       <Charset>ucs2</Charset>
470 |       <DefaultForCharset>0</DefaultForCharset>
471 |     </collation>
472 |     <collation id="119" parent="1" name="ujis_bin">
473 |       <Charset>ujis</Charset>
474 |       <DefaultForCharset>0</DefaultForCharset>
475 |     </collation>
476 |     <collation id="120" parent="1" name="ujis_japanese_ci">
477 |       <Charset>ujis</Charset>
478 |       <DefaultForCharset>1</DefaultForCharset>
479 |     </collation>
480 |     <collation id="121" parent="1" name="utf16_bin">
481 |       <Charset>utf16</Charset>
482 |       <DefaultForCharset>0</DefaultForCharset>
483 |     </collation>
484 |     <collation id="122" parent="1" name="utf16_croatian_ci">
485 |       <Charset>utf16</Charset>
486 |       <DefaultForCharset>0</DefaultForCharset>
487 |     </collation>
488 |     <collation id="123" parent="1" name="utf16_czech_ci">
489 |       <Charset>utf16</Charset>
490 |       <DefaultForCharset>0</DefaultForCharset>
491 |     </collation>
492 |     <collation id="124" parent="1" name="utf16_danish_ci">
493 |       <Charset>utf16</Charset>
494 |       <DefaultForCharset>0</DefaultForCharset>
495 |     </collation>
496 |     <collation id="125" parent="1" name="utf16_esperanto_ci">
497 |       <Charset>utf16</Charset>
498 |       <DefaultForCharset>0</DefaultForCharset>
499 |     </collation>
500 |     <collation id="126" parent="1" name="utf16_estonian_ci">
501 |       <Charset>utf16</Charset>
502 |       <DefaultForCharset>0</DefaultForCharset>
503 |     </collation>
504 |     <collation id="127" parent="1" name="utf16_general_ci">
505 |       <Charset>utf16</Charset>
506 |       <DefaultForCharset>1</DefaultForCharset>
507 |     </collation>
508 |     <collation id="128" parent="1" name="utf16_german2_ci">
509 |       <Charset>utf16</Charset>
510 |       <DefaultForCharset>0</DefaultForCharset>
511 |     </collation>
512 |     <collation id="129" parent="1" name="utf16_hungarian_ci">
513 |       <Charset>utf16</Charset>
514 |       <DefaultForCharset>0</DefaultForCharset>
515 |     </collation>
516 |     <collation id="130" parent="1" name="utf16_icelandic_ci">
517 |       <Charset>utf16</Charset>
518 |       <DefaultForCharset>0</DefaultForCharset>
519 |     </collation>
520 |     <collation id="131" parent="1" name="utf16_latvian_ci">
521 |       <Charset>utf16</Charset>
522 |       <DefaultForCharset>0</DefaultForCharset>
523 |     </collation>
524 |     <collation id="132" parent="1" name="utf16_lithuanian_ci">
525 |       <Charset>utf16</Charset>
526 |       <DefaultForCharset>0</DefaultForCharset>
527 |     </collation>
528 |     <collation id="133" parent="1" name="utf16_persian_ci">
529 |       <Charset>utf16</Charset>
530 |       <DefaultForCharset>0</DefaultForCharset>
531 |     </collation>
532 |     <collation id="134" parent="1" name="utf16_polish_ci">
533 |       <Charset>utf16</Charset>
534 |       <DefaultForCharset>0</DefaultForCharset>
535 |     </collation>
536 |     <collation id="135" parent="1" name="utf16_roman_ci">
537 |       <Charset>utf16</Charset>
538 |       <DefaultForCharset>0</DefaultForCharset>
539 |     </collation>
540 |     <collation id="136" parent="1" name="utf16_romanian_ci">
541 |       <Charset>utf16</Charset>
542 |       <DefaultForCharset>0</DefaultForCharset>
543 |     </collation>
544 |     <collation id="137" parent="1" name="utf16_sinhala_ci">
545 |       <Charset>utf16</Charset>
546 |       <DefaultForCharset>0</DefaultForCharset>
547 |     </collation>
548 |     <collation id="138" parent="1" name="utf16_slovak_ci">
549 |       <Charset>utf16</Charset>
550 |       <DefaultForCharset>0</DefaultForCharset>
551 |     </collation>
552 |     <collation id="139" parent="1" name="utf16_slovenian_ci">
553 |       <Charset>utf16</Charset>
554 |       <DefaultForCharset>0</DefaultForCharset>
555 |     </collation>
556 |     <collation id="140" parent="1" name="utf16_spanish2_ci">
557 |       <Charset>utf16</Charset>
558 |       <DefaultForCharset>0</DefaultForCharset>
559 |     </collation>
560 |     <collation id="141" parent="1" name="utf16_spanish_ci">
561 |       <Charset>utf16</Charset>
562 |       <DefaultForCharset>0</DefaultForCharset>
563 |     </collation>
564 |     <collation id="142" parent="1" name="utf16_swedish_ci">
565 |       <Charset>utf16</Charset>
566 |       <DefaultForCharset>0</DefaultForCharset>
567 |     </collation>
568 |     <collation id="143" parent="1" name="utf16_turkish_ci">
569 |       <Charset>utf16</Charset>
570 |       <DefaultForCharset>0</DefaultForCharset>
571 |     </collation>
572 |     <collation id="144" parent="1" name="utf16_unicode_520_ci">
573 |       <Charset>utf16</Charset>
574 |       <DefaultForCharset>0</DefaultForCharset>
575 |     </collation>
576 |     <collation id="145" parent="1" name="utf16_unicode_ci">
577 |       <Charset>utf16</Charset>
578 |       <DefaultForCharset>0</DefaultForCharset>
579 |     </collation>
580 |     <collation id="146" parent="1" name="utf16_vietnamese_ci">
581 |       <Charset>utf16</Charset>
582 |       <DefaultForCharset>0</DefaultForCharset>
583 |     </collation>
584 |     <collation id="147" parent="1" name="utf16le_bin">
585 |       <Charset>utf16le</Charset>
586 |       <DefaultForCharset>0</DefaultForCharset>
587 |     </collation>
588 |     <collation id="148" parent="1" name="utf16le_general_ci">
589 |       <Charset>utf16le</Charset>
590 |       <DefaultForCharset>1</DefaultForCharset>
591 |     </collation>
592 |     <collation id="149" parent="1" name="utf32_bin">
593 |       <Charset>utf32</Charset>
594 |       <DefaultForCharset>0</DefaultForCharset>
595 |     </collation>
596 |     <collation id="150" parent="1" name="utf32_croatian_ci">
597 |       <Charset>utf32</Charset>
598 |       <DefaultForCharset>0</DefaultForCharset>
599 |     </collation>
600 |     <collation id="151" parent="1" name="utf32_czech_ci">
601 |       <Charset>utf32</Charset>
602 |       <DefaultForCharset>0</DefaultForCharset>
603 |     </collation>
604 |     <collation id="152" parent="1" name="utf32_danish_ci">
605 |       <Charset>utf32</Charset>
606 |       <DefaultForCharset>0</DefaultForCharset>
607 |     </collation>
608 |     <collation id="153" parent="1" name="utf32_esperanto_ci">
609 |       <Charset>utf32</Charset>
610 |       <DefaultForCharset>0</DefaultForCharset>
611 |     </collation>
612 |     <collation id="154" parent="1" name="utf32_estonian_ci">
613 |       <Charset>utf32</Charset>
614 |       <DefaultForCharset>0</DefaultForCharset>
615 |     </collation>
616 |     <collation id="155" parent="1" name="utf32_general_ci">
617 |       <Charset>utf32</Charset>
618 |       <DefaultForCharset>1</DefaultForCharset>
619 |     </collation>
620 |     <collation id="156" parent="1" name="utf32_german2_ci">
621 |       <Charset>utf32</Charset>
622 |       <DefaultForCharset>0</DefaultForCharset>
623 |     </collation>
624 |     <collation id="157" parent="1" name="utf32_hungarian_ci">
625 |       <Charset>utf32</Charset>
626 |       <DefaultForCharset>0</DefaultForCharset>
627 |     </collation>
628 |     <collation id="158" parent="1" name="utf32_icelandic_ci">
629 |       <Charset>utf32</Charset>
630 |       <DefaultForCharset>0</DefaultForCharset>
631 |     </collation>
632 |     <collation id="159" parent="1" name="utf32_latvian_ci">
633 |       <Charset>utf32</Charset>
634 |       <DefaultForCharset>0</DefaultForCharset>
635 |     </collation>
636 |     <collation id="160" parent="1" name="utf32_lithuanian_ci">
637 |       <Charset>utf32</Charset>
638 |       <DefaultForCharset>0</DefaultForCharset>
639 |     </collation>
640 |     <collation id="161" parent="1" name="utf32_persian_ci">
641 |       <Charset>utf32</Charset>
642 |       <DefaultForCharset>0</DefaultForCharset>
643 |     </collation>
644 |     <collation id="162" parent="1" name="utf32_polish_ci">
645 |       <Charset>utf32</Charset>
646 |       <DefaultForCharset>0</DefaultForCharset>
647 |     </collation>
648 |     <collation id="163" parent="1" name="utf32_roman_ci">
649 |       <Charset>utf32</Charset>
650 |       <DefaultForCharset>0</DefaultForCharset>
651 |     </collation>
652 |     <collation id="164" parent="1" name="utf32_romanian_ci">
653 |       <Charset>utf32</Charset>
654 |       <DefaultForCharset>0</DefaultForCharset>
655 |     </collation>
656 |     <collation id="165" parent="1" name="utf32_sinhala_ci">
657 |       <Charset>utf32</Charset>
658 |       <DefaultForCharset>0</DefaultForCharset>
659 |     </collation>
660 |     <collation id="166" parent="1" name="utf32_slovak_ci">
661 |       <Charset>utf32</Charset>
662 |       <DefaultForCharset>0</DefaultForCharset>
663 |     </collation>
664 |     <collation id="167" parent="1" name="utf32_slovenian_ci">
665 |       <Charset>utf32</Charset>
666 |       <DefaultForCharset>0</DefaultForCharset>
667 |     </collation>
668 |     <collation id="168" parent="1" name="utf32_spanish2_ci">
669 |       <Charset>utf32</Charset>
670 |       <DefaultForCharset>0</DefaultForCharset>
671 |     </collation>
672 |     <collation id="169" parent="1" name="utf32_spanish_ci">
673 |       <Charset>utf32</Charset>
674 |       <DefaultForCharset>0</DefaultForCharset>
675 |     </collation>
676 |     <collation id="170" parent="1" name="utf32_swedish_ci">
677 |       <Charset>utf32</Charset>
678 |       <DefaultForCharset>0</DefaultForCharset>
679 |     </collation>
680 |     <collation id="171" parent="1" name="utf32_turkish_ci">
681 |       <Charset>utf32</Charset>
682 |       <DefaultForCharset>0</DefaultForCharset>
683 |     </collation>
684 |     <collation id="172" parent="1" name="utf32_unicode_520_ci">
685 |       <Charset>utf32</Charset>
686 |       <DefaultForCharset>0</DefaultForCharset>
687 |     </collation>
688 |     <collation id="173" parent="1" name="utf32_unicode_ci">
689 |       <Charset>utf32</Charset>
690 |       <DefaultForCharset>0</DefaultForCharset>
691 |     </collation>
692 |     <collation id="174" parent="1" name="utf32_vietnamese_ci">
693 |       <Charset>utf32</Charset>
694 |       <DefaultForCharset>0</DefaultForCharset>
695 |     </collation>
696 |     <collation id="175" parent="1" name="utf8_bin">
697 |       <Charset>utf8</Charset>
698 |       <DefaultForCharset>0</DefaultForCharset>
699 |     </collation>
700 |     <collation id="176" parent="1" name="utf8_croatian_ci">
701 |       <Charset>utf8</Charset>
702 |       <DefaultForCharset>0</DefaultForCharset>
703 |     </collation>
704 |     <collation id="177" parent="1" name="utf8_czech_ci">
705 |       <Charset>utf8</Charset>
706 |       <DefaultForCharset>0</DefaultForCharset>
707 |     </collation>
708 |     <collation id="178" parent="1" name="utf8_danish_ci">
709 |       <Charset>utf8</Charset>
710 |       <DefaultForCharset>0</DefaultForCharset>
711 |     </collation>
712 |     <collation id="179" parent="1" name="utf8_esperanto_ci">
713 |       <Charset>utf8</Charset>
714 |       <DefaultForCharset>0</DefaultForCharset>
715 |     </collation>
716 |     <collation id="180" parent="1" name="utf8_estonian_ci">
717 |       <Charset>utf8</Charset>
718 |       <DefaultForCharset>0</DefaultForCharset>
719 |     </collation>
720 |     <collation id="181" parent="1" name="utf8_general_ci">
721 |       <Charset>utf8</Charset>
722 |       <DefaultForCharset>1</DefaultForCharset>
723 |     </collation>
724 |     <collation id="182" parent="1" name="utf8_general_mysql500_ci">
725 |       <Charset>utf8</Charset>
726 |       <DefaultForCharset>0</DefaultForCharset>
727 |     </collation>
728 |     <collation id="183" parent="1" name="utf8_german2_ci">
729 |       <Charset>utf8</Charset>
730 |       <DefaultForCharset>0</DefaultForCharset>
731 |     </collation>
732 |     <collation id="184" parent="1" name="utf8_hungarian_ci">
733 |       <Charset>utf8</Charset>
734 |       <DefaultForCharset>0</DefaultForCharset>
735 |     </collation>
736 |     <collation id="185" parent="1" name="utf8_icelandic_ci">
737 |       <Charset>utf8</Charset>
738 |       <DefaultForCharset>0</DefaultForCharset>
739 |     </collation>
740 |     <collation id="186" parent="1" name="utf8_latvian_ci">
741 |       <Charset>utf8</Charset>
742 |       <DefaultForCharset>0</DefaultForCharset>
743 |     </collation>
744 |     <collation id="187" parent="1" name="utf8_lithuanian_ci">
745 |       <Charset>utf8</Charset>
746 |       <DefaultForCharset>0</DefaultForCharset>
747 |     </collation>
748 |     <collation id="188" parent="1" name="utf8_persian_ci">
749 |       <Charset>utf8</Charset>
750 |       <DefaultForCharset>0</DefaultForCharset>
751 |     </collation>
752 |     <collation id="189" parent="1" name="utf8_polish_ci">
753 |       <Charset>utf8</Charset>
754 |       <DefaultForCharset>0</DefaultForCharset>
755 |     </collation>
756 |     <collation id="190" parent="1" name="utf8_roman_ci">
757 |       <Charset>utf8</Charset>
758 |       <DefaultForCharset>0</DefaultForCharset>
759 |     </collation>
760 |     <collation id="191" parent="1" name="utf8_romanian_ci">
761 |       <Charset>utf8</Charset>
762 |       <DefaultForCharset>0</DefaultForCharset>
763 |     </collation>
764 |     <collation id="192" parent="1" name="utf8_sinhala_ci">
765 |       <Charset>utf8</Charset>
766 |       <DefaultForCharset>0</DefaultForCharset>
767 |     </collation>
768 |     <collation id="193" parent="1" name="utf8_slovak_ci">
769 |       <Charset>utf8</Charset>
770 |       <DefaultForCharset>0</DefaultForCharset>
771 |     </collation>
772 |     <collation id="194" parent="1" name="utf8_slovenian_ci">
773 |       <Charset>utf8</Charset>
774 |       <DefaultForCharset>0</DefaultForCharset>
775 |     </collation>
776 |     <collation id="195" parent="1" name="utf8_spanish2_ci">
777 |       <Charset>utf8</Charset>
778 |       <DefaultForCharset>0</DefaultForCharset>
779 |     </collation>
780 |     <collation id="196" parent="1" name="utf8_spanish_ci">
781 |       <Charset>utf8</Charset>
782 |       <DefaultForCharset>0</DefaultForCharset>
783 |     </collation>
784 |     <collation id="197" parent="1" name="utf8_swedish_ci">
785 |       <Charset>utf8</Charset>
786 |       <DefaultForCharset>0</DefaultForCharset>
787 |     </collation>
788 |     <collation id="198" parent="1" name="utf8_turkish_ci">
789 |       <Charset>utf8</Charset>
790 |       <DefaultForCharset>0</DefaultForCharset>
791 |     </collation>
792 |     <collation id="199" parent="1" name="utf8_unicode_520_ci">
793 |       <Charset>utf8</Charset>
794 |       <DefaultForCharset>0</DefaultForCharset>
795 |     </collation>
796 |     <collation id="200" parent="1" name="utf8_unicode_ci">
797 |       <Charset>utf8</Charset>
798 |       <DefaultForCharset>0</DefaultForCharset>
799 |     </collation>
800 |     <collation id="201" parent="1" name="utf8_vietnamese_ci">
801 |       <Charset>utf8</Charset>
802 |       <DefaultForCharset>0</DefaultForCharset>
803 |     </collation>
804 |     <collation id="202" parent="1" name="utf8mb4_bin">
805 |       <Charset>utf8mb4</Charset>
806 |       <DefaultForCharset>0</DefaultForCharset>
807 |     </collation>
808 |     <collation id="203" parent="1" name="utf8mb4_croatian_ci">
809 |       <Charset>utf8mb4</Charset>
810 |       <DefaultForCharset>0</DefaultForCharset>
811 |     </collation>
812 |     <collation id="204" parent="1" name="utf8mb4_czech_ci">
813 |       <Charset>utf8mb4</Charset>
814 |       <DefaultForCharset>0</DefaultForCharset>
815 |     </collation>
816 |     <collation id="205" parent="1" name="utf8mb4_danish_ci">
817 |       <Charset>utf8mb4</Charset>
818 |       <DefaultForCharset>0</DefaultForCharset>
819 |     </collation>
820 |     <collation id="206" parent="1" name="utf8mb4_esperanto_ci">
821 |       <Charset>utf8mb4</Charset>
822 |       <DefaultForCharset>0</DefaultForCharset>
823 |     </collation>
824 |     <collation id="207" parent="1" name="utf8mb4_estonian_ci">
825 |       <Charset>utf8mb4</Charset>
826 |       <DefaultForCharset>0</DefaultForCharset>
827 |     </collation>
828 |     <collation id="208" parent="1" name="utf8mb4_general_ci">
829 |       <Charset>utf8mb4</Charset>
830 |       <DefaultForCharset>1</DefaultForCharset>
831 |     </collation>
832 |     <collation id="209" parent="1" name="utf8mb4_german2_ci">
833 |       <Charset>utf8mb4</Charset>
834 |       <DefaultForCharset>0</DefaultForCharset>
835 |     </collation>
836 |     <collation id="210" parent="1" name="utf8mb4_hungarian_ci">
837 |       <Charset>utf8mb4</Charset>
838 |       <DefaultForCharset>0</DefaultForCharset>
839 |     </collation>
840 |     <collation id="211" parent="1" name="utf8mb4_icelandic_ci">
841 |       <Charset>utf8mb4</Charset>
842 |       <DefaultForCharset>0</DefaultForCharset>
843 |     </collation>
844 |     <collation id="212" parent="1" name="utf8mb4_latvian_ci">
845 |       <Charset>utf8mb4</Charset>
846 |       <DefaultForCharset>0</DefaultForCharset>
847 |     </collation>
848 |     <collation id="213" parent="1" name="utf8mb4_lithuanian_ci">
849 |       <Charset>utf8mb4</Charset>
850 |       <DefaultForCharset>0</DefaultForCharset>
851 |     </collation>
852 |     <collation id="214" parent="1" name="utf8mb4_persian_ci">
853 |       <Charset>utf8mb4</Charset>
854 |       <DefaultForCharset>0</DefaultForCharset>
855 |     </collation>
856 |     <collation id="215" parent="1" name="utf8mb4_polish_ci">
857 |       <Charset>utf8mb4</Charset>
858 |       <DefaultForCharset>0</DefaultForCharset>
859 |     </collation>
860 |     <collation id="216" parent="1" name="utf8mb4_roman_ci">
861 |       <Charset>utf8mb4</Charset>
862 |       <DefaultForCharset>0</DefaultForCharset>
863 |     </collation>
864 |     <collation id="217" parent="1" name="utf8mb4_romanian_ci">
865 |       <Charset>utf8mb4</Charset>
866 |       <DefaultForCharset>0</DefaultForCharset>
867 |     </collation>
868 |     <collation id="218" parent="1" name="utf8mb4_sinhala_ci">
869 |       <Charset>utf8mb4</Charset>
870 |       <DefaultForCharset>0</DefaultForCharset>
871 |     </collation>
872 |     <collation id="219" parent="1" name="utf8mb4_slovak_ci">
873 |       <Charset>utf8mb4</Charset>
874 |       <DefaultForCharset>0</DefaultForCharset>
875 |     </collation>
876 |     <collation id="220" parent="1" name="utf8mb4_slovenian_ci">
877 |       <Charset>utf8mb4</Charset>
878 |       <DefaultForCharset>0</DefaultForCharset>
879 |     </collation>
880 |     <collation id="221" parent="1" name="utf8mb4_spanish2_ci">
881 |       <Charset>utf8mb4</Charset>
882 |       <DefaultForCharset>0</DefaultForCharset>
883 |     </collation>
884 |     <collation id="222" parent="1" name="utf8mb4_spanish_ci">
885 |       <Charset>utf8mb4</Charset>
886 |       <DefaultForCharset>0</DefaultForCharset>
887 |     </collation>
888 |     <collation id="223" parent="1" name="utf8mb4_swedish_ci">
889 |       <Charset>utf8mb4</Charset>
890 |       <DefaultForCharset>0</DefaultForCharset>
891 |     </collation>
892 |     <collation id="224" parent="1" name="utf8mb4_turkish_ci">
893 |       <Charset>utf8mb4</Charset>
894 |       <DefaultForCharset>0</DefaultForCharset>
895 |     </collation>
896 |     <collation id="225" parent="1" name="utf8mb4_unicode_520_ci">
897 |       <Charset>utf8mb4</Charset>
898 |       <DefaultForCharset>0</DefaultForCharset>
899 |     </collation>
900 |     <collation id="226" parent="1" name="utf8mb4_unicode_ci">
901 |       <Charset>utf8mb4</Charset>
902 |       <DefaultForCharset>0</DefaultForCharset>
903 |     </collation>
904 |     <collation id="227" parent="1" name="utf8mb4_vietnamese_ci">
905 |       <Charset>utf8mb4</Charset>
906 |       <DefaultForCharset>0</DefaultForCharset>
907 |     </collation>
908 |   </database-model>
909 | </dataSource>


--------------------------------------------------------------------------------
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="Encoding" addBOMForNewFiles="with NO BOM" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
 1 | <component name="InspectionProjectProfileManager">
 2 |   <profile version="1.0">
 3 |     <option name="myName" value="Project Default" />
 4 |     <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
 5 |       <option name="ignoredPackages">
 6 |         <value>
 7 |           <list size="3">
 8 |             <item index="0" class="java.lang.String" itemvalue="tensorflow" />
 9 |             <item index="1" class="java.lang.String" itemvalue="sentencepiece" />
10 |             <item index="2" class="java.lang.String" itemvalue="jieba" />
11 |           </list>
12 |         </value>
13 |       </option>
14 |     </inspection_tool>
15 |   </profile>
16 | </component>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (transformer-cn)" project-jdk-type="Python SDK" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/transformer-master.iml" filepath="$PROJECT_DIR$/.idea/transformer-master.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/other.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="PySciProjectComponent">
4 |     <option name="PY_SCI_VIEW" value="true" />
5 |     <option name="PY_SCI_VIEW_SUGGESTED" value="true" />
6 |   </component>
7 | </project>


--------------------------------------------------------------------------------
/.idea/transformer-master.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="jdk" jdkName="Python 3.6 (transformer-cn)" jdkType="Python SDK" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="PyDocumentationSettings">
 9 |     <option name="renderExternalDocumentation" value="true" />
10 |   </component>
11 |   <component name="TestRunnerService">
12 |     <option name="projectConfiguration" value="pytest" />
13 |     <option name="PROJECT_TEST_RUNNER" value="pytest" />
14 |   </component>
15 | </module>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
   1 | <?xml version="1.0" encoding="UTF-8"?>
   2 | <project version="4">
   3 |   <component name="ChangeListManager">
   4 |     <list default="true" id="ecab60aa-5d06-4fd2-a363-9a3baaaaa418" name="Default Changelist" comment="fixed some bug">
   5 |       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
   6 |       <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
   7 |       <change beforePath="$PROJECT_DIR$/beam_search.py" beforeDir="false" afterPath="$PROJECT_DIR$/beam_search.py" afterDir="false" />
   8 |       <change beforePath="$PROJECT_DIR$/data_load.py" beforeDir="false" afterPath="$PROJECT_DIR$/data_load.py" afterDir="false" />
   9 |       <change beforePath="$PROJECT_DIR$/hparams.py" beforeDir="false" afterPath="$PROJECT_DIR$/hparams.py" afterDir="false" />
  10 |       <change beforePath="$PROJECT_DIR$/model.py" beforeDir="false" afterPath="$PROJECT_DIR$/model.py" afterDir="false" />
  11 |       <change beforePath="$PROJECT_DIR$/modules.py" beforeDir="false" afterPath="$PROJECT_DIR$/modules.py" afterDir="false" />
  12 |       <change beforePath="$PROJECT_DIR$/pred.py" beforeDir="false" afterPath="$PROJECT_DIR$/pred.py" afterDir="false" />
  13 |       <change beforePath="$PROJECT_DIR$/requirements.txt" beforeDir="false" afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
  14 |       <change beforePath="$PROJECT_DIR$/train.py" beforeDir="false" afterPath="$PROJECT_DIR$/train.py" afterDir="false" />
  15 |       <change beforePath="$PROJECT_DIR$/utils.py" beforeDir="false" afterPath="$PROJECT_DIR$/utils.py" afterDir="false" />
  16 |     </list>
  17 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  18 |     <option name="SHOW_DIALOG" value="false" />
  19 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
  20 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
  21 |     <option name="LAST_RESOLUTION" value="IGNORE" />
  22 |   </component>
  23 |   <component name="DatabaseView">
  24 |     <option name="SHOW_INTERMEDIATE" value="true" />
  25 |     <option name="GROUP_DATA_SOURCES" value="true" />
  26 |     <option name="GROUP_SCHEMA" value="true" />
  27 |     <option name="GROUP_CONTENTS" value="false" />
  28 |     <option name="SORT_POSITIONED" value="false" />
  29 |     <option name="SHOW_EMPTY_GROUPS" value="false" />
  30 |     <option name="AUTO_SCROLL_FROM_SOURCE" value="false" />
  31 |     <option name="HIDDEN_KINDS">
  32 |       <set />
  33 |     </option>
  34 |     <expand />
  35 |     <select />
  36 |   </component>
  37 |   <component name="FileEditorManager">
  38 |     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
  39 |       <file pinned="false" current-in-tab="false">
  40 |         <entry file="file://$PROJECT_DIR$/hparams.py">
  41 |           <provider selected="true" editor-type-id="text-editor">
  42 |             <state relative-caret-position="527">
  43 |               <caret line="44" column="56" lean-forward="true" selection-start-line="44" selection-start-column="56" selection-end-line="44" selection-end-column="56" />
  44 |             </state>
  45 |           </provider>
  46 |         </entry>
  47 |       </file>
  48 |       <file pinned="false" current-in-tab="false">
  49 |         <entry file="file://$PROJECT_DIR$/utils.py">
  50 |           <provider selected="true" editor-type-id="text-editor">
  51 |             <state relative-caret-position="289">
  52 |               <caret line="153" column="57" selection-start-line="153" selection-start-column="57" selection-end-line="153" selection-end-column="57" />
  53 |               <folding>
  54 |                 <element signature="e#136#147#0" expanded="true" />
  55 |                 <element signature="e#542#857#0" />
  56 |               </folding>
  57 |             </state>
  58 |           </provider>
  59 |         </entry>
  60 |       </file>
  61 |       <file pinned="false" current-in-tab="false">
  62 |         <entry file="file://$PROJECT_DIR$/data_load.py">
  63 |           <provider selected="true" editor-type-id="text-editor">
  64 |             <state relative-caret-position="374">
  65 |               <caret line="140" column="45" selection-start-line="140" selection-start-column="45" selection-end-line="140" selection-end-column="45" />
  66 |             </state>
  67 |           </provider>
  68 |         </entry>
  69 |       </file>
  70 |       <file pinned="false" current-in-tab="false">
  71 |         <entry file="file://$PROJECT_DIR$/model.py">
  72 |           <provider selected="true" editor-type-id="text-editor">
  73 |             <state relative-caret-position="675">
  74 |               <caret line="88" lean-forward="true" selection-start-line="88" selection-end-line="88" />
  75 |               <folding>
  76 |                 <element signature="e#136#150#0" expanded="true" />
  77 |                 <element signature="e#1040#2379#0" />
  78 |                 <element signature="e#5902#7743#0" />
  79 |                 <element signature="e#7801#8501#0" />
  80 |               </folding>
  81 |             </state>
  82 |           </provider>
  83 |         </entry>
  84 |       </file>
  85 |       <file pinned="false" current-in-tab="false">
  86 |         <entry file="file://$PROJECT_DIR$/modules.py">
  87 |           <provider selected="true" editor-type-id="text-editor">
  88 |             <state relative-caret-position="272">
  89 |               <caret line="267" column="46" selection-start-line="267" selection-start-column="46" selection-end-line="267" selection-end-column="46" />
  90 |             </state>
  91 |           </provider>
  92 |         </entry>
  93 |       </file>
  94 |       <file pinned="false" current-in-tab="false">
  95 |         <entry file="file://$PROJECT_DIR$/requirements.txt">
  96 |           <provider selected="true" editor-type-id="text-editor">
  97 |             <state relative-caret-position="51">
  98 |               <caret line="3" column="14" lean-forward="true" selection-start-line="3" selection-start-column="14" selection-end-line="3" selection-end-column="14" />
  99 |             </state>
 100 |           </provider>
 101 |         </entry>
 102 |       </file>
 103 |       <file pinned="false" current-in-tab="true">
 104 |         <entry file="file://$PROJECT_DIR$/README.md">
 105 |           <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
 106 |             <state split_layout="SPLIT">
 107 |               <first_editor relative-caret-position="507">
 108 |                 <caret line="75" column="86" selection-start-line="75" selection-start-column="86" selection-end-line="75" selection-end-column="86" />
 109 |               </first_editor>
 110 |               <second_editor />
 111 |             </state>
 112 |           </provider>
 113 |         </entry>
 114 |       </file>
 115 |       <file pinned="false" current-in-tab="false">
 116 |         <entry file="file://$PROJECT_DIR$/train.py">
 117 |           <provider selected="true" editor-type-id="text-editor">
 118 |             <state relative-caret-position="-23">
 119 |               <caret line="66" column="8" selection-start-line="66" selection-start-column="8" selection-end-line="66" selection-end-column="8" />
 120 |               <folding>
 121 |                 <element signature="e#137#151#0" expanded="true" />
 122 |               </folding>
 123 |             </state>
 124 |           </provider>
 125 |         </entry>
 126 |       </file>
 127 |       <file pinned="false" current-in-tab="false">
 128 |         <entry file="file://$PROJECT_DIR$/test.py">
 129 |           <provider selected="true" editor-type-id="text-editor">
 130 |             <state relative-caret-position="170">
 131 |               <caret line="10" column="19" selection-start-line="10" selection-start-column="19" selection-end-line="10" selection-end-column="19" />
 132 |             </state>
 133 |           </provider>
 134 |         </entry>
 135 |       </file>
 136 |       <file pinned="false" current-in-tab="false">
 137 |         <entry file="file://$PROJECT_DIR$/beam_search.py">
 138 |           <provider selected="true" editor-type-id="text-editor">
 139 |             <state relative-caret-position="28">
 140 |               <caret line="83" column="39" selection-start-line="83" selection-start-column="39" selection-end-line="83" selection-end-column="39" />
 141 |             </state>
 142 |           </provider>
 143 |         </entry>
 144 |       </file>
 145 |     </leaf>
 146 |   </component>
 147 |   <component name="FileTemplateManagerImpl">
 148 |     <option name="RECENT_TEMPLATES">
 149 |       <list>
 150 |         <option value="Python Script" />
 151 |       </list>
 152 |     </option>
 153 |   </component>
 154 |   <component name="FindInProjectRecents">
 155 |     <findStrings>
 156 |       <find>”“</find>
 157 |       <find>ga</find>
 158 |       <find>elems</find>
 159 |       <find>bleu score</find>
 160 |       <find>self.tf.</find>
 161 |       <find>evaldir</find>
 162 |       <find>(</find>
 163 |       <find>oovs</find>
 164 |       <find>memory</find>
 165 |       <find>苏泊尔被曝锰含量超标近4倍可致帕金森病</find>
 166 |       <find>LSCTS</find>
 167 |       <find>ckpt = self.tf.train.get_checkpoint_state(self.model_dir).all_model_checkpoint_paths[-1]</find>
 168 |       <find>gpu_list</find>
 169 |       <find>gpu_nums</find>
 170 |       <find>hp.eval3</find>
 171 |       <find />
 172 |       <find>num_eval_batches</find>
 173 |       <find>tf</find>
 174 |       <find>_gs</find>
 175 |       <find>i</find>
 176 |       <find>eval_rouge</find>
 177 |       <find>num_train_samples</find>
 178 |       <find>iter</find>
 179 |       <find>validation</find>
 180 |       <find>iterator</find>
 181 |       <find>隋炀帝墓135件文物首次展出杨广2颗牙齿亮相</find>
 182 |       <find>中医药为何逐渐被边缘化</find>
 183 |       <find>self.vocab_file</find>
 184 |       <find>output_shapes</find>
 185 |       <find>_encode</find>
 186 |     </findStrings>
 187 |     <replaceStrings>
 188 |       <replace>tf.</replace>
 189 |       <replace>LCSTS</replace>
 190 |       <replace>hp.test</replace>
 191 |       <replace>hp.eval_rouge</replace>
 192 |       <replace>iterator</replace>
 193 |       <replace>val</replace>
 194 |       <replace>iter</replace>
 195 |     </replaceStrings>
 196 |   </component>
 197 |   <component name="Git.Settings">
 198 |     <option name="ROOT_SYNC" value="DONT_SYNC" />
 199 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 200 |   </component>
 201 |   <component name="HighlightingSettingsPerFile">
 202 |     <setting file="file://$PROJECT_DIR$/hparams.py" root0="FORCE_HIGHLIGHTING" />
 203 |     <setting file="file://$PROJECT_DIR$/data_load.py" root0="FORCE_HIGHLIGHTING" />
 204 |   </component>
 205 |   <component name="IdeDocumentHistory">
 206 |     <option name="CHANGED_PATHS">
 207 |       <list>
 208 |         <option value="$PROJECT_DIR$/prepro.py" />
 209 |         <option value="$PROJECT_DIR$/data/vocab" />
 210 |         <option value="$PROJECT_DIR$/data/data3.csv" />
 211 |         <option value="$PROJECT_DIR$/data/data2.csv" />
 212 |         <option value="$PROJECT_DIR$/data/vocab1" />
 213 |         <option value="$PROJECT_DIR$/data/eval_summary.csv" />
 214 |         <option value="$PROJECT_DIR$/data/eval1.csv" />
 215 |         <option value="$PROJECT_DIR$/prediction.py" />
 216 |         <option value="$PROJECT_DIR$/tf1.2_legacy/train.py" />
 217 |         <option value="$PROJECT_DIR$/test3.py" />
 218 |         <option value="$PROJECT_DIR$/data_load_bak.py" />
 219 |         <option value="$PROJECT_DIR$/data/test.csv" />
 220 |         <option value="$PROJECT_DIR$/data_load1.py" />
 221 |         <option value="$PROJECT_DIR$/data/vocab.txt" />
 222 |         <option value="$PROJECT_DIR$/data/vocabs1.txt" />
 223 |         <option value="$PROJECT_DIR$/beam_search1.py" />
 224 |         <option value="$PROJECT_DIR$/test2.py" />
 225 |         <option value="$PROJECT_DIR$/pre_data.py" />
 226 |         <option value="$PROJECT_DIR$/model/pointer/checkpoint" />
 227 |         <option value="$PROJECT_DIR$/fafasf.py" />
 228 |         <option value="$PROJECT_DIR$/fafaf.py" />
 229 |         <option value="$PROJECT_DIR$/ffasfa.py" />
 230 |         <option value="$PROJECT_DIR$/data/test_summary.csv" />
 231 |         <option value="$PROJECT_DIR$/rouge.py" />
 232 |         <option value="$PROJECT_DIR$/data/test_summary1.csv" />
 233 |         <option value="$PROJECT_DIR$/sumjieba/dict/stop.txt" />
 234 |         <option value="$PROJECT_DIR$/utils1.py" />
 235 |         <option value="$PROJECT_DIR$/pred_beamsearch.py" />
 236 |         <option value="$PROJECT_DIR$/test1.py" />
 237 |         <option value="$PROJECT_DIR$/train1.py" />
 238 |         <option value="$PROJECT_DIR$/beam_search.py" />
 239 |         <option value="$PROJECT_DIR$/pred.py" />
 240 |         <option value="$PROJECT_DIR$/data/test1.csv" />
 241 |         <option value="$PROJECT_DIR$/test.py" />
 242 |         <option value="$PROJECT_DIR$/utils.py" />
 243 |         <option value="$PROJECT_DIR$/requirements.txt" />
 244 |         <option value="$PROJECT_DIR$/train.py" />
 245 |         <option value="$PROJECT_DIR$/modules.py" />
 246 |         <option value="$PROJECT_DIR$/data_load.py" />
 247 |         <option value="$PROJECT_DIR$/hparams.py" />
 248 |         <option value="$PROJECT_DIR$/model.py" />
 249 |         <option value="$PROJECT_DIR$/README.md" />
 250 |       </list>
 251 |     </option>
 252 |   </component>
 253 |   <component name="ProjectFrameBounds" extendedState="6">
 254 |     <option name="x" value="2550" />
 255 |     <option name="y" value="-10" />
 256 |     <option name="width" value="2580" />
 257 |     <option name="height" value="1410" />
 258 |   </component>
 259 |   <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
 260 |   <component name="ProjectView">
 261 |     <navigator currentView="Scope" currentSubView="Scope 'Project Files'; set:Project Files; class com.intellij.psi.search.scope.ProjectFilesScope" proportions="" version="1">
 262 |       <foldersAlwaysOnTop value="true" />
 263 |     </navigator>
 264 |     <panes>
 265 |       <pane id="Scope">
 266 |         <subPane subId="Scope 'Project Files'; set:Project Files; class com.intellij.psi.search.scope.ProjectFilesScope">
 267 |           <expand>
 268 |             <path>
 269 |               <item name="transformer-pointer-github - 副本" type="3d21c010:ScopeViewTreeModel$ProjectNode" />
 270 |               <item name="D:\Awake\Code\transformer-pointer-github - 副本" type="442cc68d:ScopeViewTreeModel$RootNode" />
 271 |             </path>
 272 |             <path>
 273 |               <item name="transformer-pointer-github - 副本" type="3d21c010:ScopeViewTreeModel$ProjectNode" />
 274 |               <item name="D:\Awake\Code\transformer-pointer-github - 副本" type="442cc68d:ScopeViewTreeModel$RootNode" />
 275 |               <item name="data" type="9f88c78c:ScopeViewTreeModel$FileNode" />
 276 |             </path>
 277 |           </expand>
 278 |           <select />
 279 |         </subPane>
 280 |       </pane>
 281 |       <pane id="ProjectPane">
 282 |         <subPane>
 283 |           <expand>
 284 |             <path>
 285 |               <item name="transformer-master" type="b2602c69:ProjectViewProjectNode" />
 286 |               <item name="transformer-master" type="462c0819:PsiDirectoryNode" />
 287 |             </path>
 288 |             <path>
 289 |               <item name="transformer-master" type="b2602c69:ProjectViewProjectNode" />
 290 |               <item name="transformer-master" type="462c0819:PsiDirectoryNode" />
 291 |               <item name="iwslt2016" type="462c0819:PsiDirectoryNode" />
 292 |             </path>
 293 |             <path>
 294 |               <item name="transformer-master" type="b2602c69:ProjectViewProjectNode" />
 295 |               <item name="transformer-master" type="462c0819:PsiDirectoryNode" />
 296 |               <item name="log" type="462c0819:PsiDirectoryNode" />
 297 |             </path>
 298 |           </expand>
 299 |           <select />
 300 |         </subPane>
 301 |       </pane>
 302 |     </panes>
 303 |   </component>
 304 |   <component name="PropertiesComponent">
 305 |     <property name="DatabaseDriversLRU" value="mysql" />
 306 |     <property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
 307 |     <property name="WebServerToolWindowFactoryState" value="false" />
 308 |     <property name="last_opened_file_path" value="$USER_HOME$/Desktop/Global-Encoding-master" />
 309 |     <property name="settings.editor.selected.configurable" value="fileTemplates" />
 310 |   </component>
 311 |   <component name="PyFindUsagesOptions">
 312 |     <option name="isSearchForTextOccurrences" value="true" />
 313 |   </component>
 314 |   <component name="RecentsManager">
 315 |     <key name="MoveFile.RECENT_KEYS">
 316 |       <recent name="D:\Awake\Code\transformer-cn\model\based" />
 317 |       <recent name="D:\Awake\Code\transformer-cn\data\lscts" />
 318 |     </key>
 319 |     <key name="CopyFile.RECENT_KEYS">
 320 |       <recent name="D:\Awake\Code\transformer-pointer-github" />
 321 |       <recent name="D:\Awake\Code\transformer-pointer-github\data" />
 322 |       <recent name="D:\Awake\Code\transformer-pointer-github\fig" />
 323 |       <recent name="D:\Awake\Code\transformer-cn\model" />
 324 |       <recent name="D:\Awake\Code\transformer-cn\model\pointer" />
 325 |     </key>
 326 |   </component>
 327 |   <component name="RunDashboard">
 328 |     <option name="ruleStates">
 329 |       <list>
 330 |         <RuleState>
 331 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
 332 |         </RuleState>
 333 |         <RuleState>
 334 |           <option name="name" value="StatusDashboardGroupingRule" />
 335 |         </RuleState>
 336 |       </list>
 337 |     </option>
 338 |   </component>
 339 |   <component name="RunManager" selected="Python.train">
 340 |     <configuration default="true" type="PythonConfigurationType" factoryName="Python">
 341 |       <module name="transformer-master" />
 342 |       <option name="INTERPRETER_OPTIONS" value="" />
 343 |       <option name="PARENT_ENVS" value="true" />
 344 |       <envs>
 345 |         <env name="PYTHONUNBUFFERED" value="1" />
 346 |       </envs>
 347 |       <option name="SDK_HOME" value="" />
 348 |       <option name="WORKING_DIRECTORY" value="" />
 349 |       <option name="IS_MODULE_SDK" value="false" />
 350 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 351 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 352 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 353 |       <option name="SCRIPT_NAME" value="" />
 354 |       <option name="PARAMETERS" value="" />
 355 |       <option name="SHOW_COMMAND_LINE" value="true" />
 356 |       <option name="EMULATE_TERMINAL" value="false" />
 357 |       <option name="MODULE_MODE" value="false" />
 358 |       <option name="REDIRECT_INPUT" value="false" />
 359 |       <option name="INPUT_FILE" value="" />
 360 |       <method v="2" />
 361 |     </configuration>
 362 |     <configuration name="model" type="PythonConfigurationType" factoryName="Python" temporary="true">
 363 |       <module name="transformer-master" />
 364 |       <option name="INTERPRETER_OPTIONS" value="" />
 365 |       <option name="PARENT_ENVS" value="true" />
 366 |       <envs>
 367 |         <env name="PYTHONUNBUFFERED" value="1" />
 368 |       </envs>
 369 |       <option name="SDK_HOME" value="" />
 370 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
 371 |       <option name="IS_MODULE_SDK" value="true" />
 372 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 373 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 374 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 375 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/model.py" />
 376 |       <option name="PARAMETERS" value="" />
 377 |       <option name="SHOW_COMMAND_LINE" value="true" />
 378 |       <option name="EMULATE_TERMINAL" value="false" />
 379 |       <option name="MODULE_MODE" value="false" />
 380 |       <option name="REDIRECT_INPUT" value="false" />
 381 |       <option name="INPUT_FILE" value="" />
 382 |       <method v="2" />
 383 |     </configuration>
 384 |     <configuration name="test (1)" type="PythonConfigurationType" factoryName="Python" temporary="true">
 385 |       <module name="transformer-master" />
 386 |       <option name="INTERPRETER_OPTIONS" value="" />
 387 |       <option name="PARENT_ENVS" value="true" />
 388 |       <envs>
 389 |         <env name="PYTHONUNBUFFERED" value="1" />
 390 |       </envs>
 391 |       <option name="SDK_HOME" value="" />
 392 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
 393 |       <option name="IS_MODULE_SDK" value="true" />
 394 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 395 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 396 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 397 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/test.py" />
 398 |       <option name="PARAMETERS" value="" />
 399 |       <option name="SHOW_COMMAND_LINE" value="false" />
 400 |       <option name="EMULATE_TERMINAL" value="false" />
 401 |       <option name="MODULE_MODE" value="false" />
 402 |       <option name="REDIRECT_INPUT" value="false" />
 403 |       <option name="INPUT_FILE" value="" />
 404 |       <method v="2" />
 405 |     </configuration>
 406 |     <configuration name="test1" type="PythonConfigurationType" factoryName="Python" temporary="true">
 407 |       <module name="transformer-master" />
 408 |       <option name="INTERPRETER_OPTIONS" value="" />
 409 |       <option name="PARENT_ENVS" value="true" />
 410 |       <envs>
 411 |         <env name="PYTHONUNBUFFERED" value="1" />
 412 |       </envs>
 413 |       <option name="SDK_HOME" value="" />
 414 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
 415 |       <option name="IS_MODULE_SDK" value="true" />
 416 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 417 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 418 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 419 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/test1.py" />
 420 |       <option name="PARAMETERS" value="" />
 421 |       <option name="SHOW_COMMAND_LINE" value="false" />
 422 |       <option name="EMULATE_TERMINAL" value="false" />
 423 |       <option name="MODULE_MODE" value="false" />
 424 |       <option name="REDIRECT_INPUT" value="false" />
 425 |       <option name="INPUT_FILE" value="" />
 426 |       <method v="2" />
 427 |     </configuration>
 428 |     <configuration name="train" type="PythonConfigurationType" factoryName="Python" temporary="true">
 429 |       <module name="transformer-master" />
 430 |       <option name="INTERPRETER_OPTIONS" value="" />
 431 |       <option name="PARENT_ENVS" value="true" />
 432 |       <envs>
 433 |         <env name="PYTHONUNBUFFERED" value="1" />
 434 |       </envs>
 435 |       <option name="SDK_HOME" value="" />
 436 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
 437 |       <option name="IS_MODULE_SDK" value="true" />
 438 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 439 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 440 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 441 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train.py" />
 442 |       <option name="PARAMETERS" value="" />
 443 |       <option name="SHOW_COMMAND_LINE" value="false" />
 444 |       <option name="EMULATE_TERMINAL" value="false" />
 445 |       <option name="MODULE_MODE" value="false" />
 446 |       <option name="REDIRECT_INPUT" value="false" />
 447 |       <option name="INPUT_FILE" value="" />
 448 |       <method v="2" />
 449 |     </configuration>
 450 |     <configuration name="train1" type="PythonConfigurationType" factoryName="Python" temporary="true">
 451 |       <module name="transformer-master" />
 452 |       <option name="INTERPRETER_OPTIONS" value="" />
 453 |       <option name="PARENT_ENVS" value="true" />
 454 |       <envs>
 455 |         <env name="PYTHONUNBUFFERED" value="1" />
 456 |       </envs>
 457 |       <option name="SDK_HOME" value="" />
 458 |       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
 459 |       <option name="IS_MODULE_SDK" value="true" />
 460 |       <option name="ADD_CONTENT_ROOTS" value="true" />
 461 |       <option name="ADD_SOURCE_ROOTS" value="true" />
 462 |       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
 463 |       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/train1.py" />
 464 |       <option name="PARAMETERS" value="" />
 465 |       <option name="SHOW_COMMAND_LINE" value="false" />
 466 |       <option name="EMULATE_TERMINAL" value="false" />
 467 |       <option name="MODULE_MODE" value="false" />
 468 |       <option name="REDIRECT_INPUT" value="false" />
 469 |       <option name="INPUT_FILE" value="" />
 470 |       <method v="2" />
 471 |     </configuration>
 472 |     <list>
 473 |       <item itemvalue="Python.train" />
 474 |       <item itemvalue="Python.test (1)" />
 475 |       <item itemvalue="Python.model" />
 476 |       <item itemvalue="Python.train1" />
 477 |       <item itemvalue="Python.test1" />
 478 |     </list>
 479 |     <recent_temporary>
 480 |       <list>
 481 |         <item itemvalue="Python.train" />
 482 |         <item itemvalue="Python.test (1)" />
 483 |         <item itemvalue="Python.test1" />
 484 |         <item itemvalue="Python.train1" />
 485 |         <item itemvalue="Python.model" />
 486 |       </list>
 487 |     </recent_temporary>
 488 |   </component>
 489 |   <component name="SvnConfiguration">
 490 |     <configuration />
 491 |   </component>
 492 |   <component name="TaskManager">
 493 |     <task active="true" id="Default" summary="Default task">
 494 |       <changelist id="ecab60aa-5d06-4fd2-a363-9a3baaaaa418" name="Default Changelist" comment="" />
 495 |       <created>1552362774445</created>
 496 |       <option name="number" value="Default" />
 497 |       <option name="presentableId" value="Default" />
 498 |       <updated>1552362774445</updated>
 499 |       <workItem from="1557709693114" duration="235000" />
 500 |       <workItem from="1557710387406" duration="5308000" />
 501 |       <workItem from="1557795790234" duration="31354000" />
 502 |       <workItem from="1557969169828" duration="47000" />
 503 |       <workItem from="1557972700683" duration="18741000" />
 504 |       <workItem from="1558054990650" duration="12756000" />
 505 |       <workItem from="1558080411886" duration="3169000" />
 506 |       <workItem from="1558084063888" duration="8826000" />
 507 |       <workItem from="1558322018856" duration="856000" />
 508 |       <workItem from="1558323201740" duration="1803000" />
 509 |       <workItem from="1558331438899" duration="8900000" />
 510 |       <workItem from="1558341287234" duration="1074000" />
 511 |       <workItem from="1558344720608" duration="4661000" />
 512 |       <workItem from="1558407507050" duration="3292000" />
 513 |       <workItem from="1558422714299" duration="146000" />
 514 |       <workItem from="1558427972264" duration="56000" />
 515 |       <workItem from="1558429584267" duration="2431000" />
 516 |       <workItem from="1558590794713" duration="68000" />
 517 |       <workItem from="1558598416450" duration="277000" />
 518 |       <workItem from="1558598784458" duration="4604000" />
 519 |       <workItem from="1558670049802" duration="1471000" />
 520 |       <workItem from="1558683555726" duration="19702000" />
 521 |       <workItem from="1558766722205" duration="11266000" />
 522 |       <workItem from="1558920437840" duration="34098000" />
 523 |       <workItem from="1559025105821" duration="7484000" />
 524 |       <workItem from="1559032828348" duration="4551000" />
 525 |       <workItem from="1559178671767" duration="4484000" />
 526 |       <workItem from="1559464373195" duration="7420000" />
 527 |       <workItem from="1559544988892" duration="660000" />
 528 |       <workItem from="1559546852958" duration="18740000" />
 529 |       <workItem from="1559620544156" duration="28230000" />
 530 |       <workItem from="1559719293034" duration="5809000" />
 531 |     </task>
 532 |     <task id="LOCAL-00001" summary="change vocab">
 533 |       <created>1557365218479</created>
 534 |       <option name="number" value="00001" />
 535 |       <option name="presentableId" value="LOCAL-00001" />
 536 |       <option name="project" value="LOCAL" />
 537 |       <updated>1557365218479</updated>
 538 |     </task>
 539 |     <task id="LOCAL-00002" summary="add beam search and prediction">
 540 |       <created>1557972785135</created>
 541 |       <option name="number" value="00002" />
 542 |       <option name="presentableId" value="LOCAL-00002" />
 543 |       <option name="project" value="LOCAL" />
 544 |       <updated>1557972785135</updated>
 545 |     </task>
 546 |     <task id="LOCAL-00003" summary="1. reload transformer parameters&#10;2. version transformer-pointer 1.0">
 547 |       <created>1558085745605</created>
 548 |       <option name="number" value="00003" />
 549 |       <option name="presentableId" value="LOCAL-00003" />
 550 |       <option name="project" value="LOCAL" />
 551 |       <updated>1558085745605</updated>
 552 |     </task>
 553 |     <task id="LOCAL-00004" summary="optimize code of pred.py and beam_search.py">
 554 |       <created>1558337002489</created>
 555 |       <option name="number" value="00004" />
 556 |       <option name="presentableId" value="LOCAL-00004" />
 557 |       <option name="project" value="LOCAL" />
 558 |       <updated>1558337002489</updated>
 559 |     </task>
 560 |     <task id="LOCAL-00005" summary="add README.md">
 561 |       <created>1558348841756</created>
 562 |       <option name="number" value="00005" />
 563 |       <option name="presentableId" value="LOCAL-00005" />
 564 |       <option name="project" value="LOCAL" />
 565 |       <updated>1558348841756</updated>
 566 |     </task>
 567 |     <task id="LOCAL-00006" summary="add vocabs">
 568 |       <created>1558401210881</created>
 569 |       <option name="number" value="00006" />
 570 |       <option name="presentableId" value="LOCAL-00006" />
 571 |       <option name="project" value="LOCAL" />
 572 |       <updated>1558401210881</updated>
 573 |     </task>
 574 |     <task id="LOCAL-00007" summary="add code signature">
 575 |       <created>1558408517378</created>
 576 |       <option name="number" value="00007" />
 577 |       <option name="presentableId" value="LOCAL-00007" />
 578 |       <option name="project" value="LOCAL" />
 579 |       <updated>1558408517378</updated>
 580 |     </task>
 581 |     <task id="LOCAL-00008" summary="add transformer loss and transformer-pointer generator loss">
 582 |       <created>1558409466718</created>
 583 |       <option name="number" value="00008" />
 584 |       <option name="presentableId" value="LOCAL-00008" />
 585 |       <option name="project" value="LOCAL" />
 586 |       <updated>1558409466718</updated>
 587 |     </task>
 588 |     <task id="LOCAL-00009" summary="change the picture of transformer loss">
 589 |       <created>1558409636201</created>
 590 |       <option name="number" value="00009" />
 591 |       <option name="presentableId" value="LOCAL-00009" />
 592 |       <option name="project" value="LOCAL" />
 593 |       <updated>1558409636201</updated>
 594 |     </task>
 595 |     <task id="LOCAL-00010" summary="add todo things">
 596 |       <created>1558409968858</created>
 597 |       <option name="number" value="00010" />
 598 |       <option name="presentableId" value="LOCAL-00010" />
 599 |       <option name="project" value="LOCAL" />
 600 |       <updated>1558409968858</updated>
 601 |     </task>
 602 |     <task id="LOCAL-00011" summary="modfiy README.md">
 603 |       <created>1558410453262</created>
 604 |       <option name="number" value="00011" />
 605 |       <option name="presentableId" value="LOCAL-00011" />
 606 |       <option name="project" value="LOCAL" />
 607 |       <updated>1558410453262</updated>
 608 |     </task>
 609 |     <task id="LOCAL-00012" summary="modify README.md">
 610 |       <created>1558410517294</created>
 611 |       <option name="number" value="00012" />
 612 |       <option name="presentableId" value="LOCAL-00012" />
 613 |       <option name="project" value="LOCAL" />
 614 |       <updated>1558410517294</updated>
 615 |     </task>
 616 |     <task id="LOCAL-00013" summary="modify README.md">
 617 |       <created>1558422819896</created>
 618 |       <option name="number" value="00013" />
 619 |       <option name="presentableId" value="LOCAL-00013" />
 620 |       <option name="project" value="LOCAL" />
 621 |       <updated>1558422819896</updated>
 622 |     </task>
 623 |     <task id="LOCAL-00014" summary="modify some bugs">
 624 |       <created>1558493250965</created>
 625 |       <option name="number" value="00014" />
 626 |       <option name="presentableId" value="LOCAL-00014" />
 627 |       <option name="project" value="LOCAL" />
 628 |       <updated>1558493250965</updated>
 629 |     </task>
 630 |     <task id="LOCAL-00015" summary="modify README.md add explain about multi gpu">
 631 |       <created>1558493509793</created>
 632 |       <option name="number" value="00015" />
 633 |       <option name="presentableId" value="LOCAL-00015" />
 634 |       <option name="project" value="LOCAL" />
 635 |       <updated>1558493509793</updated>
 636 |     </task>
 637 |     <task id="LOCAL-00016" summary="add rouge score, and set which gpus can be seem by tensorflow">
 638 |       <created>1558692358913</created>
 639 |       <option name="number" value="00016" />
 640 |       <option name="presentableId" value="LOCAL-00016" />
 641 |       <option name="project" value="LOCAL" />
 642 |       <updated>1558692358913</updated>
 643 |     </task>
 644 |     <task id="LOCAL-00017" summary="modify requirement">
 645 |       <created>1558692586145</created>
 646 |       <option name="number" value="00017" />
 647 |       <option name="presentableId" value="LOCAL-00017" />
 648 |       <option name="project" value="LOCAL" />
 649 |       <updated>1558692586145</updated>
 650 |     </task>
 651 |     <task id="LOCAL-00018" summary="fix rouge score bug">
 652 |       <created>1558768389765</created>
 653 |       <option name="number" value="00018" />
 654 |       <option name="presentableId" value="LOCAL-00018" />
 655 |       <option name="project" value="LOCAL" />
 656 |       <updated>1558768389765</updated>
 657 |     </task>
 658 |     <task id="LOCAL-00019" summary="modify README.md">
 659 |       <created>1558768991425</created>
 660 |       <option name="number" value="00019" />
 661 |       <option name="presentableId" value="LOCAL-00019" />
 662 |       <option name="project" value="LOCAL" />
 663 |       <updated>1558768991425</updated>
 664 |     </task>
 665 |     <task id="LOCAL-00020" summary="fixed multi gpu bug">
 666 |       <created>1558777069599</created>
 667 |       <option name="number" value="00020" />
 668 |       <option name="presentableId" value="LOCAL-00020" />
 669 |       <option name="project" value="LOCAL" />
 670 |       <updated>1558777069599</updated>
 671 |     </task>
 672 |     <task id="LOCAL-00021" summary="fixed range float bug">
 673 |       <created>1558778049864</created>
 674 |       <option name="number" value="00021" />
 675 |       <option name="presentableId" value="LOCAL-00021" />
 676 |       <option name="project" value="LOCAL" />
 677 |       <updated>1558778049864</updated>
 678 |     </task>
 679 |     <task id="LOCAL-00022" summary="fixed some bug">
 680 |       <created>1558780427260</created>
 681 |       <option name="number" value="00022" />
 682 |       <option name="presentableId" value="LOCAL-00022" />
 683 |       <option name="project" value="LOCAL" />
 684 |       <updated>1558780427260</updated>
 685 |     </task>
 686 |     <option name="localTasksCounter" value="23" />
 687 |     <servers />
 688 |   </component>
 689 |   <component name="TimeTrackingManager">
 690 |     <option name="totallyTimeSpent" value="252519000" />
 691 |   </component>
 692 |   <component name="TodoView">
 693 |     <todo-panel id="selected-file">
 694 |       <is-autoscroll-to-source value="true" />
 695 |     </todo-panel>
 696 |     <todo-panel id="all">
 697 |       <are-packages-shown value="true" />
 698 |       <is-autoscroll-to-source value="true" />
 699 |     </todo-panel>
 700 |   </component>
 701 |   <component name="ToolWindowManager">
 702 |     <frame x="2553" y="-7" width="2062" height="1126" extended-state="6" />
 703 |     <editor active="true" />
 704 |     <layout>
 705 |       <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.09031936" />
 706 |       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
 707 |       <window_info id="Favorites" order="2" side_tool="true" />
 708 |       <window_info anchor="bottom" id="Message" order="0" />
 709 |       <window_info anchor="bottom" id="Find" order="1" weight="0.32897383" />
 710 |       <window_info anchor="bottom" id="Run" order="2" sideWeight="0.499002" visible="true" weight="0.30482897" />
 711 |       <window_info anchor="bottom" id="Debug" order="3" sideWeight="0.499002" weight="0.35110664" />
 712 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
 713 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
 714 |       <window_info anchor="bottom" id="TODO" order="6" show_stripe_button="false" weight="0.32897383" />
 715 |       <window_info anchor="bottom" id="Version Control" order="7" weight="0.32997987" />
 716 |       <window_info anchor="bottom" id="Terminal" order="8" weight="0.3993964" />
 717 |       <window_info anchor="bottom" id="Event Log" order="9" sideWeight="0.500998" side_tool="true" weight="0.30482897" />
 718 |       <window_info anchor="bottom" id="Python Console" order="10" weight="0.32897383" />
 719 |       <window_info anchor="bottom" id="Docker" order="11" show_stripe_button="false" />
 720 |       <window_info anchor="bottom" id="Database Changes" order="12" weight="0.32997987" />
 721 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
 722 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
 723 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
 724 |       <window_info anchor="right" id="SciView" order="3" weight="0.32984033" />
 725 |       <window_info anchor="right" x="0" y="0" width="465" height="926" id="Documentation" order="4" side_tool="true" weight="0.32984033" />
 726 |       <window_info anchor="right" id="Database" order="5" weight="0.16816367" />
 727 |     </layout>
 728 |   </component>
 729 |   <component name="TypeScriptGeneratedFilesManager">
 730 |     <option name="version" value="1" />
 731 |   </component>
 732 |   <component name="VcsManagerConfiguration">
 733 |     <option name="LOCAL_CHANGES_DETAILS_PREVIEW_SHOWN" value="true" />
 734 |     <MESSAGE value="change vocab" />
 735 |     <MESSAGE value="add beam search and prediction" />
 736 |     <MESSAGE value="1. reload transformer parameters&#10;2. version transformer-pointer 1.0" />
 737 |     <MESSAGE value="optimize code of pred.py and beam_search.py" />
 738 |     <MESSAGE value="add README.md" />
 739 |     <MESSAGE value="add vocabs" />
 740 |     <MESSAGE value="add code signature" />
 741 |     <MESSAGE value="add transformer loss and transformer-pointer generator loss" />
 742 |     <MESSAGE value="change the picture of transformer loss" />
 743 |     <MESSAGE value="add todo things" />
 744 |     <MESSAGE value="modfiy README.md" />
 745 |     <MESSAGE value="modify some bugs" />
 746 |     <MESSAGE value="modify README.md add explain about multi gpu" />
 747 |     <MESSAGE value="add rouge score, and set which gpus can be seem by tensorflow" />
 748 |     <MESSAGE value="modify requirement" />
 749 |     <MESSAGE value="fix rouge score bug" />
 750 |     <MESSAGE value="modify README.md" />
 751 |     <MESSAGE value="fixed multi gpu bug" />
 752 |     <MESSAGE value="fixed range float bug" />
 753 |     <MESSAGE value="fixed some bug" />
 754 |     <option name="LAST_COMMIT_MESSAGE" value="fixed some bug" />
 755 |   </component>
 756 |   <component name="XDebuggerManager">
 757 |     <breakpoint-manager>
 758 |       <breakpoints>
 759 |         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
 760 |           <url>file://$PROJECT_DIR$/utils.py</url>
 761 |           <line>143</line>
 762 |           <option name="timeStamp" value="25" />
 763 |         </line-breakpoint>
 764 |       </breakpoints>
 765 |     </breakpoint-manager>
 766 |   </component>
 767 |   <component name="com.intellij.coverage.CoverageDataManagerImpl">
 768 |     <SUITE FILE_PATH="coverage/transformer_cn$model.coverage" NAME="model Coverage Results" MODIFIED="1557972857563" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 769 |     <SUITE FILE_PATH="coverage/transformer_pointer_github$train.coverage" NAME="train Coverage Results" MODIFIED="1559026947234" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 770 |     <SUITE FILE_PATH="coverage/transformer_pointer_github$test1.coverage" NAME="test1 Coverage Results" MODIFIED="1559617883823" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 771 |     <SUITE FILE_PATH="coverage/transformer_pointer_github___$train1.coverage" NAME="train1 Coverage Results" MODIFIED="1559632479693" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 772 |     <SUITE FILE_PATH="coverage/transformer_cn$test2.coverage" NAME="test2 Coverage Results" MODIFIED="1557990673973" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 773 |     <SUITE FILE_PATH="coverage/transformer_pointer_github___$test__1_.coverage" NAME="test (1) Coverage Results" MODIFIED="1559700584060" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 774 |     <SUITE FILE_PATH="coverage/transformer_cn$train.coverage" NAME="train Coverage Results" MODIFIED="1558322647909" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 775 |     <SUITE FILE_PATH="coverage/transformer_pointer_github$rouge.coverage" NAME="rouge Coverage Results" MODIFIED="1558684225390" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 776 |     <SUITE FILE_PATH="coverage/transformer_cn$pre_data.coverage" NAME="pre_data Coverage Results" MODIFIED="1558320941085" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 777 |     <SUITE FILE_PATH="coverage/transformer_pointer_github$test__1_.coverage" NAME="test (1) Coverage Results" MODIFIED="1558939823344" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 778 |     <SUITE FILE_PATH="coverage/transformer_pointer_github___$test1.coverage" NAME="test1 Coverage Results" MODIFIED="1559633007327" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 779 |     <SUITE FILE_PATH="coverage/transformer_cn$test1.coverage" NAME="test1 Coverage Results" MODIFIED="1558065564502" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 780 |     <SUITE FILE_PATH="coverage/transformer_cn$pred.coverage" NAME="pred Coverage Results" MODIFIED="1558341156355" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 781 |     <SUITE FILE_PATH="coverage/transformer_pointer_github___$train.coverage" NAME="train Coverage Results" MODIFIED="1559724446246" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 782 |     <SUITE FILE_PATH="coverage/transformer_pointer_github$train1.coverage" NAME="train1 Coverage Results" MODIFIED="1559619779730" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
 783 |   </component>
 784 |   <component name="editorHistoryManager">
 785 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/data/util/nest.py">
 786 |       <provider selected="true" editor-type-id="text-editor">
 787 |         <state relative-caret-position="195">
 788 |           <caret line="312" selection-start-line="312" selection-end-line="312" />
 789 |         </state>
 790 |       </provider>
 791 |     </entry>
 792 |     <entry file="file://$PROJECT_DIR$/eval/1/trans_pointer00L7.21" />
 793 |     <entry file="file://$PROJECT_DIR$/eval/1/trans_pointer00L7.18" />
 794 |     <entry file="file://$PROJECT_DIR$/rouge.txt" />
 795 |     <entry file="file://$PROJECT_DIR$/multi-bleu.perl" />
 796 |     <entry file="file://$PROJECT_DIR$/data/test_summary.csv" />
 797 |     <entry file="file://$PROJECT_DIR$/data/test_summary1.csv" />
 798 |     <entry file="file://$PROJECT_DIR$/rouge.py" />
 799 |     <entry file="file://$PROJECT_DIR$/eval/1/trans_pointer00L6.56" />
 800 |     <entry file="file://$PROJECT_DIR$/sumjieba/dict/stop_old.txt" />
 801 |     <entry file="file://$PROJECT_DIR$/sumjieba/dict/stop.txt" />
 802 |     <entry file="file://$PROJECT_DIR$/stop_vocab" />
 803 |     <entry file="file://$PROJECT_DIR$/data/eval.csv">
 804 |       <provider selected="true" editor-type-id="csv-text-editor">
 805 |         <state relative-caret-position="255">
 806 |           <caret line="15" column="22" selection-start-line="15" selection-start-column="22" selection-end-line="15" selection-end-column="22" />
 807 |         </state>
 808 |       </provider>
 809 |     </entry>
 810 |     <entry file="file://$PROJECT_DIR$/fig/transformer-pointer gererator-loss.png">
 811 |       <provider selected="true" editor-type-id="images" />
 812 |     </entry>
 813 |     <entry file="file://$PROJECT_DIR$/fig/transformer-loss.png">
 814 |       <provider selected="true" editor-type-id="images" />
 815 |     </entry>
 816 |     <entry file="file://$PROJECT_DIR$/fig/structure.jpg">
 817 |       <provider selected="true" editor-type-id="images" />
 818 |     </entry>
 819 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/ops/array_ops.py">
 820 |       <provider selected="true" editor-type-id="text-editor">
 821 |         <state relative-caret-position="512">
 822 |           <caret line="238" selection-start-line="238" selection-end-line="238" />
 823 |         </state>
 824 |       </provider>
 825 |     </entry>
 826 |     <entry file="file://$PROJECT_DIR$/log/2/specs" />
 827 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/framework/op_def_library.py">
 828 |       <provider selected="true" editor-type-id="text-editor">
 829 |         <state relative-caret-position="495">
 830 |           <caret line="786" selection-start-line="786" selection-end-line="786" />
 831 |         </state>
 832 |       </provider>
 833 |     </entry>
 834 |     <entry file="file://$PROJECT_DIR$/log/2/hparams">
 835 |       <provider selected="true" editor-type-id="text-editor">
 836 |         <state>
 837 |           <caret column="146" selection-start-column="146" selection-end-column="146" />
 838 |         </state>
 839 |       </provider>
 840 |     </entry>
 841 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/framework/ops.py">
 842 |       <provider selected="true" editor-type-id="text-editor">
 843 |         <state relative-caret-position="410">
 844 |           <caret line="1642" column="42" lean-forward="true" selection-start-line="1642" selection-start-column="42" selection-end-line="1642" selection-end-column="42" />
 845 |         </state>
 846 |       </provider>
 847 |     </entry>
 848 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/keras/layers/core.py">
 849 |       <provider selected="true" editor-type-id="text-editor">
 850 |         <state relative-caret-position="374">
 851 |           <caret line="932" selection-start-line="932" selection-end-line="932" />
 852 |         </state>
 853 |       </provider>
 854 |     </entry>
 855 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/layers/core.py">
 856 |       <provider selected="true" editor-type-id="text-editor">
 857 |         <state relative-caret-position="-1003">
 858 |           <caret line="112" column="4" selection-start-line="112" selection-start-column="4" selection-end-line="112" selection-end-column="4" />
 859 |         </state>
 860 |       </provider>
 861 |     </entry>
 862 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/framework/tensor_shape.py">
 863 |       <provider selected="true" editor-type-id="text-editor">
 864 |         <state relative-caret-position="493">
 865 |           <caret line="36" selection-start-line="36" selection-end-line="36" />
 866 |         </state>
 867 |       </provider>
 868 |     </entry>
 869 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/ops/math_ops.py">
 870 |       <provider selected="true" editor-type-id="text-editor">
 871 |         <state relative-caret-position="206">
 872 |           <caret line="1288" column="4" selection-start-line="1288" selection-start-column="4" selection-end-line="1288" selection-end-column="4" />
 873 |         </state>
 874 |       </provider>
 875 |     </entry>
 876 |     <entry file="file://$PROJECT_DIR$/eval/1/trans_pointer400L4.85" />
 877 |     <entry file="file://$PROJECT_DIR$/eval/1/rouge">
 878 |       <provider selected="true" editor-type-id="text-editor" />
 879 |     </entry>
 880 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/summary/writer/writer.py">
 881 |       <provider selected="true" editor-type-id="text-editor">
 882 |         <state relative-caret-position="257">
 883 |           <caret line="99" column="6" selection-start-line="99" selection-start-column="6" selection-end-line="99" selection-end-column="6" />
 884 |         </state>
 885 |       </provider>
 886 |     </entry>
 887 |     <entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
 888 |       <provider selected="true" editor-type-id="text-editor">
 889 |         <state relative-caret-position="398">
 890 |           <caret line="1751" selection-start-line="1751" selection-end-line="1751" />
 891 |         </state>
 892 |       </provider>
 893 |     </entry>
 894 |     <entry file="file://$PROJECT_DIR$/pred_beamsearch.py" />
 895 |     <entry file="file://$PROJECT_DIR$/test1.py" />
 896 |     <entry file="file://$PROJECT_DIR$/train1.py" />
 897 |     <entry file="file://$PROJECT_DIR$/utils1.py" />
 898 |     <entry file="file://$PROJECT_DIR$/pred.py">
 899 |       <provider selected="true" editor-type-id="text-editor">
 900 |         <state relative-caret-position="267">
 901 |           <caret line="80" lean-forward="true" selection-start-line="80" selection-end-line="80" />
 902 |         </state>
 903 |       </provider>
 904 |     </entry>
 905 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/client/session.py">
 906 |       <provider selected="true" editor-type-id="text-editor">
 907 |         <state relative-caret-position="206">
 908 |           <caret line="1347" selection-start-line="1347" selection-end-line="1347" />
 909 |         </state>
 910 |       </provider>
 911 |     </entry>
 912 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/sumeval/metrics/rouge.py">
 913 |       <provider selected="true" editor-type-id="text-editor">
 914 |         <state relative-caret-position="213">
 915 |           <caret line="113" column="41" selection-start-line="113" selection-start-column="31" selection-end-line="113" selection-end-column="41" />
 916 |         </state>
 917 |       </provider>
 918 |     </entry>
 919 |     <entry file="file://$PROJECT_DIR$/data/test.csv">
 920 |       <provider selected="true" editor-type-id="csv-text-editor">
 921 |         <state relative-caret-position="187">
 922 |           <caret line="11" lean-forward="true" selection-start-line="6" selection-end-line="11" />
 923 |         </state>
 924 |       </provider>
 925 |       <provider editor-type-id="csv-table-editor">
 926 |         <state showInfoPanel="true" fixedHeaders="false" rowLines="3" />
 927 |       </provider>
 928 |     </entry>
 929 |     <entry file="file://$PROJECT_DIR$/data/test1.csv">
 930 |       <provider selected="true" editor-type-id="csv-text-editor">
 931 |         <state relative-caret-position="68">
 932 |           <caret line="4" column="22" selection-start-line="4" selection-start-column="22" selection-end-line="4" selection-end-column="22" />
 933 |         </state>
 934 |       </provider>
 935 |       <provider editor-type-id="csv-table-editor">
 936 |         <state showInfoPanel="true" fixedHeaders="false" rowLines="3" />
 937 |       </provider>
 938 |     </entry>
 939 |     <entry file="file://$PROJECT_DIR$/rouge">
 940 |       <provider selected="true" editor-type-id="text-editor">
 941 |         <state relative-caret-position="17">
 942 |           <caret line="1" lean-forward="true" selection-start-line="1" selection-end-line="1" />
 943 |         </state>
 944 |       </provider>
 945 |     </entry>
 946 |     <entry file="file://$PROJECT_DIR$/eval/1/trans_pointer00L0.00">
 947 |       <provider selected="true" editor-type-id="text-editor">
 948 |         <state relative-caret-position="34">
 949 |           <caret line="2" lean-forward="true" selection-start-line="2" selection-end-line="2" />
 950 |         </state>
 951 |       </provider>
 952 |     </entry>
 953 |     <entry file="file://$PROJECT_DIR$/test.py">
 954 |       <provider selected="true" editor-type-id="text-editor">
 955 |         <state relative-caret-position="170">
 956 |           <caret line="10" column="19" selection-start-line="10" selection-start-column="19" selection-end-line="10" selection-end-column="19" />
 957 |         </state>
 958 |       </provider>
 959 |     </entry>
 960 |     <entry file="file://$PROJECT_DIR$/beam_search.py">
 961 |       <provider selected="true" editor-type-id="text-editor">
 962 |         <state relative-caret-position="28">
 963 |           <caret line="83" column="39" selection-start-line="83" selection-start-column="39" selection-end-line="83" selection-end-column="39" />
 964 |         </state>
 965 |       </provider>
 966 |     </entry>
 967 |     <entry file="file://C:/ProgramData/Anaconda3/Lib/site-packages/tensorflow/python/data/ops/dataset_ops.py">
 968 |       <provider selected="true" editor-type-id="text-editor">
 969 |         <state relative-caret-position="-187">
 970 |           <caret line="340" column="59" selection-start-line="340" selection-start-column="46" selection-end-line="340" selection-end-column="59" />
 971 |         </state>
 972 |       </provider>
 973 |     </entry>
 974 |     <entry file="file://$PROJECT_DIR$/data_load.py">
 975 |       <provider selected="true" editor-type-id="text-editor">
 976 |         <state relative-caret-position="374">
 977 |           <caret line="140" column="45" selection-start-line="140" selection-start-column="45" selection-end-line="140" selection-end-column="45" />
 978 |         </state>
 979 |       </provider>
 980 |     </entry>
 981 |     <entry file="file://$PROJECT_DIR$/utils.py">
 982 |       <provider selected="true" editor-type-id="text-editor">
 983 |         <state relative-caret-position="289">
 984 |           <caret line="153" column="57" selection-start-line="153" selection-start-column="57" selection-end-line="153" selection-end-column="57" />
 985 |           <folding>
 986 |             <element signature="e#136#147#0" expanded="true" />
 987 |             <element signature="e#542#857#0" />
 988 |           </folding>
 989 |         </state>
 990 |       </provider>
 991 |     </entry>
 992 |     <entry file="file://$PROJECT_DIR$/hparams.py">
 993 |       <provider selected="true" editor-type-id="text-editor">
 994 |         <state relative-caret-position="527">
 995 |           <caret line="44" column="56" lean-forward="true" selection-start-line="44" selection-start-column="56" selection-end-line="44" selection-end-column="56" />
 996 |         </state>
 997 |       </provider>
 998 |     </entry>
 999 |     <entry file="file://$PROJECT_DIR$/train.py">
1000 |       <provider selected="true" editor-type-id="text-editor">
1001 |         <state relative-caret-position="-23">
1002 |           <caret line="66" column="8" selection-start-line="66" selection-start-column="8" selection-end-line="66" selection-end-column="8" />
1003 |           <folding>
1004 |             <element signature="e#137#151#0" expanded="true" />
1005 |           </folding>
1006 |         </state>
1007 |       </provider>
1008 |     </entry>
1009 |     <entry file="file://$PROJECT_DIR$/modules.py">
1010 |       <provider selected="true" editor-type-id="text-editor">
1011 |         <state relative-caret-position="272">
1012 |           <caret line="267" column="46" selection-start-line="267" selection-start-column="46" selection-end-line="267" selection-end-column="46" />
1013 |         </state>
1014 |       </provider>
1015 |     </entry>
1016 |     <entry file="file://$PROJECT_DIR$/model.py">
1017 |       <provider selected="true" editor-type-id="text-editor">
1018 |         <state relative-caret-position="675">
1019 |           <caret line="88" lean-forward="true" selection-start-line="88" selection-end-line="88" />
1020 |           <folding>
1021 |             <element signature="e#136#150#0" expanded="true" />
1022 |             <element signature="e#1040#2379#0" />
1023 |             <element signature="e#5902#7743#0" />
1024 |             <element signature="e#7801#8501#0" />
1025 |           </folding>
1026 |         </state>
1027 |       </provider>
1028 |     </entry>
1029 |     <entry file="file://$PROJECT_DIR$/requirements.txt">
1030 |       <provider selected="true" editor-type-id="text-editor">
1031 |         <state relative-caret-position="51">
1032 |           <caret line="3" column="14" lean-forward="true" selection-start-line="3" selection-start-column="14" selection-end-line="3" selection-end-column="14" />
1033 |         </state>
1034 |       </provider>
1035 |     </entry>
1036 |     <entry file="file://$PROJECT_DIR$/README.md">
1037 |       <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
1038 |         <state split_layout="SPLIT">
1039 |           <first_editor relative-caret-position="507">
1040 |             <caret line="75" column="86" selection-start-line="75" selection-start-column="86" selection-end-line="75" selection-end-column="86" />
1041 |           </first_editor>
1042 |           <second_editor />
1043 |         </state>
1044 |       </provider>
1045 |     </entry>
1046 |   </component>
1047 | </project>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Cally
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # A Abstractive Summarization Implementation with Transformer and Pointer-generator
 2 | when I wanted to get summary by neural network, I tried many ways to generate abstract summary, but the result was not good.
 3 | when I heared 2018 byte cup, I found some information about it, and the champion's solution attracted me, but I found some websites,
 4 | like github gitlab, I didn't find the official code, so I decided to implement it.
 5 | 
 6 | ## Requirements
 7 | * python==3.x (Let's move on to python 3 if you still use python 2)
 8 | * tensorflow==1.12.0
 9 | * tqdm>=4.28.1
10 | * jieba>=0.3x
11 | * sumeval>=0.2.0
12 | 
13 | ## Model Structure
14 | ### Based
15 | My model is based on [Attention Is All You Need](https://arxiv.org/abs/1706.03762) and [Get To The Point: Summarization with Pointer-Generator Networks](https://arxiv.org/abs/1704.04368)
16 | ### Change
17 | * The pointer-generator model has two mechanisms, which are **copy mechanism** and **coverage mechanism**, I found some materials, 
18 | they show the Coverage mechanism doesn't suit short summary, so I didn't use this mechanism, just use the first one.
19 | * Pointer generator model has a inadequacy, which can let the loss got nan, I tried some times and wanted to fix it,
20 | but the result was I can't, I think the reason was when calculate final logists, it will 
21 |  extend vocab length to oov and vocab length, it will get more zeroes. so I delete the mechanism of extend final logists, just use their mechanism of 
22 | deocode from article and vocab. there is more [detail](https://github.com/abisee/pointer-generator/issues/4) about it, 
23 | in this model, I just use word than vocab, this idea is from bert.
24 | ### Structure
25 | <img src="fig/structure.jpg">
26 | 
27 | ## Training
28 | * STEP 1. [download](https://pan.baidu.com/s/1szq0Wa60AS5ISpM_SNPcbA) the dataset, pwd is ayn6, the dataset is LCSTS by pre processed, so you will see very different dataset structure with LCSTS in the file
29 | each line is abstract and article, they split by **","**, if you worry the amount of the dataset is different between my and LCSTS, don't 
30 | worry, the amout of the dataset is same as LCSTS. 
31 | * STEP 2. Run the following command.
32 | ```
33 | python train.py
34 | ```
35 | Check `hparams.py` to see which parameters are possible. For example,
36 | ```
37 | python train.py --logdir myLog --batch_size 32 --train myTrain --eval myEval
38 | ```
39 | My code also improve multi gpu to train this model, if you have more than one gpu, just run like this
40 | ```
41 | python train.py --logdir myLog --batch_size 32 --train myTrain --eval myEval --gpu_nums=myGPUNums
42 | ```
43 | 
44 | | name | type | detail |
45 | |--------------------|------|-------------|
46 | vocab_size | int | vocab size
47 | train | str | train dataset dir
48 | eval | str| eval dataset dir
49 | test | str| data for calculate rouge score
50 | vocab | str| vocabulary file path
51 | batch_size | int| train batch size
52 | eval_batch_size | int| eval batch size
53 | lr | float| learning rate
54 | warmup_steps | int| warmup steps by learing rate
55 | logdir | str| log directory
56 | num_epochs | int| the number of train epoch
57 | evaldir | str| evaluation dir
58 | d_model | int| hidden dimension of encoder/decoder
59 | d_ff | int| hidden dimension of feedforward layer
60 | num_blocks | int| number of encoder/decoder blocks
61 | num_heads | int| number of attention heads
62 | maxlen1 | int| maximum length of a source sequence
63 | maxlen2 | int| maximum length of a target sequence
64 | dropout_rate | float| dropout rate
65 | beam_size | int| beam size for decode
66 | gpu_nums | int| gpu amount, which can allow how many gpu to train this model， default 1
67 | ### Note
68 | Don't change the hyper-parameters of transformer util you have good solution, it will let the loss can't go down! if you have good solution, I hope you can tell me.
69 | 
70 | ## Evaluation
71 | ### Loss
72 | * Transformer-Pointer generator
73 | <img src="fig/transformer-pointer gererator-loss.png" />
74 | * Transformer 
75 | <img src="fig/transformer-loss.png" />
76 | As you see, transformer-pointer generator model can let the loss go down very quickly!
77 | 
78 | ## If you like it, and think it useful for you, hope you can star.


--------------------------------------------------------------------------------
/__pycache__/beam_search.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/beam_search.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/data_load.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/data_load.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/hparams.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/hparams.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/model.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/model.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/modules.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/modules.cpython-36.pyc


--------------------------------------------------------------------------------
/__pycache__/utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/__pycache__/utils.cpython-36.pyc


--------------------------------------------------------------------------------
/beam_search.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #!/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | 
  9 | import tensorflow as tf
 10 | # add self to decode memory
 11 | class Hypothesis:
 12 |     """
 13 |         Defines a hypothesis during beam search.
 14 |     """
 15 |     def __init__(self, tokens, log_prob, sents, normalize_by_length=True):
 16 |         """
 17 |         :param tokens: a list, which are ids in vocab
 18 |         :param log_prob: log probability, add by beam search
 19 |         :param sents: already decode words,
 20 |         :param normalize_by_length: sort hypothesis by prob / len, if not, just by prob
 21 |         """
 22 |         self.tokens = tokens
 23 |         self.log_prob = log_prob
 24 |         self.normalize_by_length = normalize_by_length
 25 |         self.sents = sents
 26 | 
 27 |     def extend(self, token, log_prob, word):
 28 |         """
 29 |         Extend the hypothesis with result from latest step.
 30 |         :param token: latest token from decoding
 31 |         :param log_prob: log prob of the latest decoded tokens.
 32 |         :param word: word piece by transformer decode
 33 |         :return: new Hypothesis with the results from latest step.
 34 |         """
 35 | 
 36 |         return Hypothesis(self.tokens + [token], self.log_prob + log_prob, self.sents + word)
 37 | 
 38 |     @property
 39 |     def latest_token(self):
 40 |         return self.tokens[-1]
 41 | 
 42 |     def __str__(self):
 43 |         return ''.join(list(self.sents))
 44 | 
 45 | class BeamSearch:
 46 |     def __init__(self, model, beam_size, start_token, end_token, id2token, max_steps, input_x, input_y, logits,
 47 |                  normalize_by_length=False):
 48 |         """
 49 |         :param model: transformer model
 50 |         :param beam_size: beam size
 51 |         :param start_token: start token
 52 |         :param end_token: end token
 53 |         :param id2token: id to token dict
 54 |         :param max_steps: max steps in decode
 55 |         :param input_x: input x
 56 |         :param input_y: input y
 57 |         :param logits: logits by decode
 58 |         :param normalize_by_length: sort hypothesis by prob / len, if not, just by prob
 59 |         """
 60 |         # basic params
 61 |         self.model = model
 62 |         self.beam_size = beam_size
 63 |         self.start_token = start_token
 64 |         self.end_token = end_token
 65 |         self.max_steps = max_steps
 66 |         self.id2token = id2token
 67 | 
 68 |         # placeholder
 69 |         self.input_x = input_x
 70 |         self.input_y = input_y
 71 | 
 72 |         self.top_k_ = tf.nn.top_k(tf.nn.softmax(logits), k=self.beam_size * 2)
 73 | 
 74 |         # This length normalization is only effective for the final results.
 75 |         self.normalize_by_length = normalize_by_length
 76 | 
 77 |     def search(self, sess, input_x, memory):
 78 |         """
 79 |         use beam search for decoding
 80 |         :param sess: tensorflow session
 81 |         :param input_x: article by list, and convert to id by vocab
 82 |         :param memory: transformer encode result
 83 |         :return: hyps: list of Hypothesis, the best hypotheses found by beam search,
 84 |                        ordered by score
 85 |         """
 86 |         # create a list, which each element is Hypothesis
 87 |         hyps = [Hypothesis([self.start_token], 0.0, '')] * self.beam_size
 88 | 
 89 |         results = []
 90 |         steps = 0
 91 |         while steps < self.max_steps and len(results) < self.beam_size:
 92 |             top_k = sess.run([self.top_k_], feed_dict={self.model.memory: [memory] * self.beam_size,
 93 |                                                        self.input_x: [input_x] * self.beam_size,
 94 |                                                        self.input_y: [h.tokens for h in hyps]})
 95 |             # print(time.time() - start)
 96 |             indices = [list(indice[-1]) for indice in top_k[0][1]]
 97 |             probs = [list(prob[-1]) for prob in top_k[0][0]]
 98 | 
 99 |             all_hyps = []
100 | 
101 |             num_orig_hyps = 1 if steps == 0 else len(hyps)
102 |             for i in range(num_orig_hyps):
103 |                 h = hyps[i]
104 |                 for j in range(self.beam_size*2):
105 |                     new_h = h.extend(indices[i][j], probs[i][j], self.id2token[indices[i][j]])
106 |                     all_hyps.append(new_h)
107 | 
108 |             # Filter and collect any hypotheses that have the end token
109 |             hyps = []
110 |             for h in self.best_hyps(all_hyps):
111 |                 if h.latest_token == self.end_token:
112 |                     # Pull the hypothesis off the beam if the end token is reached.
113 |                     results.append(h)
114 |                 else:
115 |                     # Otherwise continue to the extend the hypothesis.
116 |                     hyps.append(h)
117 |                 if len(hyps) == self.beam_size or len(results) == self.beam_size:
118 |                     break
119 | 
120 |             steps += 1
121 | 
122 |             if steps == self.max_steps:
123 |                 results.extend(hyps)
124 | 
125 |         return self.best_hyps(results)
126 | 
127 |     def best_hyps(self, hyps):
128 |         """
129 |         Sort the hyps based on log probs and length.
130 |         :param hyps: A list of hypothesis
131 |         :return: A list of sorted hypothesis in reverse log_prob order.
132 |         """
133 |         # This length normalization is only effective for the final results.
134 |         if self.normalize_by_length:
135 |             return sorted(hyps, key=lambda h: h.log_prob / len(h.tokens), reverse=True)
136 |         else:
137 |             return sorted(hyps, key=lambda h: h.log_prob, reverse=True)


--------------------------------------------------------------------------------
/data_load.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | 
  9 | import tensorflow as tf
 10 | from utils import calc_num_batches
 11 | 
 12 | def _load_vocab(vocab_fpath):
 13 |     '''Loads vocabulary file and returns idx<->token maps
 14 |     vocab_fpath: string. vocabulary file path.
 15 |     Note that these are reserved
 16 |     0: <pad>, 1: <unk>, 2: <s>, 3: </s>
 17 | 
 18 |     Returns
 19 |     two dictionaries.
 20 |     '''
 21 |     vocab = []
 22 |     with open(vocab_fpath, 'r', encoding='utf-8') as f:
 23 |         for line in f:
 24 |             vocab.append(line.replace('\n', ''))
 25 |     token2idx = {token: idx for idx, token in enumerate(vocab)}
 26 |     idx2token = {idx: token for idx, token in enumerate(vocab)}
 27 | 
 28 |     return token2idx, idx2token
 29 | 
 30 | def load_stop(vocab_path):
 31 |     """
 32 |     load stop word
 33 |     :param vocab_path: stop word path
 34 |     :return: stop word list
 35 |     """
 36 |     stop_words = []
 37 |     with open(vocab_path, 'r', encoding='utf-8') as f:
 38 |         for line in f:
 39 |             stop_words.append(line.replace('\n', ''))
 40 | 
 41 |     return sorted(stop_words, key=lambda i: len(i), reverse=True)
 42 | 
 43 | def _load_data(fpaths, maxlen1, maxlen2):
 44 |     '''Loads source and target data and filters out too lengthy samples.
 45 |     fpath1: source file path. string.
 46 |     fpath2: target file path. string.
 47 |     maxlen1: source sent maximum length. scalar.
 48 |     maxlen2: target sent maximum length. scalar.
 49 | 
 50 |     Returns
 51 |     sents1: list of source sents
 52 |     sents2: list of target sents
 53 |     '''
 54 |     sents1, sents2 = [], []
 55 |     for fpath in fpaths.split('|'):
 56 |         with open(fpath, 'r', encoding='utf-8') as f:
 57 |             for line in f:
 58 |                 splits = line.split(',')
 59 |                 if len(splits) != 2: continue
 60 |                 sen1 = splits[1].replace('\n', '').strip()
 61 |                 sen2 = splits[0].strip()
 62 |                 if len(list(sen1)) + 1 > maxlen1-2: continue
 63 |                 if len(list(sen2)) + 1 > maxlen2-1: continue
 64 | 
 65 |                 sents1.append(sen1.encode('utf-8'))
 66 |                 sents2.append(sen2.encode('utf-8'))
 67 | 
 68 |     return sents1[:400000], sents2[:400000]
 69 | 
 70 | def _encode(inp, token2idx, maxlen, type):
 71 |     '''Converts string to number. Used for `generator_fn`.
 72 |     inp: 1d byte array.
 73 |     type: "x" (source side) or "y" (target side)
 74 |     dict: token2idx dictionary
 75 | 
 76 |     Returns
 77 |     list of numbers
 78 |     '''
 79 |     inp = inp.decode('utf-8')
 80 |     if type == 'x':
 81 |         tokens = ['<s>'] + list(inp) + ['</s>']
 82 |         while len(tokens) < maxlen:
 83 |             tokens.append('<pad>')
 84 |         return [token2idx.get(token, token2idx['<unk>']) for token in tokens]
 85 | 
 86 |     else:
 87 |         inputs = ['<s>'] + list(inp)
 88 |         target = list(inp) + ['</s>']
 89 |         while len(target) < maxlen:
 90 |             inputs.append('<pad>')
 91 |             target.append('<pad>')
 92 |         return [token2idx.get(token, token2idx['<unk>']) for token in inputs], [token2idx.get(token, token2idx['<unk>']) for token in target]
 93 | 
 94 | def _generator_fn(sents1, sents2, vocab_fpath, maxlen1, maxlen2):
 95 |     '''Generates training / evaluation data
 96 |     sents1: list of source sents
 97 |     sents2: list of target sents
 98 |     vocab_fpath: string. vocabulary file path.
 99 | 
100 |     yields
101 |     xs: tuple of
102 |         x: list of source token ids in a sent
103 |         x_seqlen: int. sequence length of x
104 |         sent1: str. raw source (=input) sentence
105 |     labels: tuple of
106 |         decoder_input: decoder_input: list of encoded decoder inputs
107 |         y: list of target token ids in a sent
108 |         y_seqlen: int. sequence length of y
109 |         sent2: str. target sentence
110 |     '''
111 |     token2idx, _ = _load_vocab(vocab_fpath)
112 |     for sent1, sent2 in zip(sents1, sents2):
113 |         x = _encode(sent1, token2idx, maxlen1, "x")
114 | 
115 |         inputs, targets = _encode(sent2, token2idx, maxlen2, "y")
116 | 
117 |         yield (x, sent1.decode('utf-8')), (inputs, targets, sent2.decode('utf-8'))
118 | 
119 | def _input_fn(sents1, sents2, vocab_fpath, batch_size, gpu_nums, maxlen1, maxlen2, shuffle=False):
120 |     '''Batchify data
121 |     sents1: list of source sents
122 |     sents2: list of target sents
123 |     vocab_fpath: string. vocabulary file path.
124 |     batch_size: scalar
125 |     shuffle: boolean
126 | 
127 |     Returns
128 |     xs: tuple of
129 |         x: int32 tensor. (N, T1)
130 |         x_seqlens: int32 tensor. (N,)
131 |         sents1: str tensor. (N,)
132 |     ys: tuple of
133 |         decoder_input: int32 tensor. (N, T2)
134 |         y: int32 tensor. (N, T2)
135 |         y_seqlen: int32 tensor. (N, )
136 |         sents2: str tensor. (N,)
137 |     '''
138 |     shapes = (([maxlen1], ()),
139 |               ([maxlen2], [maxlen2], ()))
140 |     types = ((tf.int32, tf.string),
141 |              (tf.int32, tf.int32, tf.string))
142 | 
143 |     dataset = tf.data.Dataset.from_generator(
144 |         _generator_fn,
145 |         output_shapes=shapes,
146 |         output_types=types,
147 |         args=(sents1, sents2, vocab_fpath, maxlen1, maxlen2))  # <- arguments for generator_fn. converted to np string arrays
148 | 
149 |     if shuffle: # for training
150 |         dataset = dataset.shuffle(128*batch_size*gpu_nums)
151 | 
152 |     dataset = dataset.repeat()  # iterate forever
153 |     dataset = dataset.batch(batch_size*gpu_nums)
154 | 
155 |     return dataset
156 | 
157 | def get_batch(fpath, maxlen1, maxlen2, vocab_fpath, batch_size, gpu_nums, shuffle=False):
158 |     '''Gets training / evaluation mini-batches
159 |     fpath: source file path. string.
160 |     maxlen1: source sent maximum length. scalar.
161 |     maxlen2: target sent maximum length. scalar.
162 |     vocab_fpath: string. vocabulary file path.
163 |     batch_size: scalar
164 |     shuffle: boolean
165 | 
166 |     Returns
167 |     batches
168 |     num_batches: number of mini-batches
169 |     num_samples
170 |     '''
171 |     sents1, sents2 = _load_data(fpath, maxlen1, maxlen2)
172 |     batches = _input_fn(sents1, sents2, vocab_fpath, batch_size, gpu_nums, maxlen1, maxlen2, shuffle=shuffle)
173 |     num_batches = calc_num_batches(len(sents1), batch_size*gpu_nums)
174 |     return batches, num_batches, len(sents1)
175 | 


--------------------------------------------------------------------------------
/fig/structure.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/fig/structure.jpg


--------------------------------------------------------------------------------
/fig/transformer-loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/fig/transformer-loss.png


--------------------------------------------------------------------------------
/fig/transformer-pointer gererator-loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xiongma/transformer-pointer-generator/6547cd40aa7aaa3802081b14b4d0490493cd794d/fig/transformer-pointer gererator-loss.png


--------------------------------------------------------------------------------
/hparams.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | #/usr/bin/python3
 3 | '''
 4 | date: 2019/5/21
 5 | mail: cally.maxiong@gmail.com
 6 | page: http://www.cnblogs.com/callyblog/
 7 | '''
 8 | import argparse
 9 | 
10 | class Hparams:
11 |     parser = argparse.ArgumentParser()
12 | 
13 |     # prepro
14 |     parser.add_argument('--vocab_size', default=10598, type=int)
15 | 
16 |     # train
17 |     ## files
18 |     parser.add_argument('--train', default='data/test.csv',
19 |                              help="data for train")
20 | 
21 |     parser.add_argument('--eval', default='data/test.csv',
22 |                              help="data for evaluation")
23 |     parser.add_argument('--eval_rouge', default='data/test_summary.csv',
24 |                              help="data for calculate rouge score")
25 | 
26 |     ## vocabulary
27 |     parser.add_argument('--vocab', default='vocab',
28 |                         help="vocabulary file path")
29 | 
30 |     parser.add_argument('--stop_vocab', default='stop_vocab',
31 |                         help="stop vocabulary file path")
32 | 
33 |     # training scheme
34 |     parser.add_argument('--batch_size', default=32, type=int)
35 |     parser.add_argument('--eval_batch_size', default=32, type=int)
36 | 
37 |     parser.add_argument('--lr', default=0.0005, type=float, help="learning rate")
38 |     parser.add_argument('--warmup_steps', default=4000, type=int)
39 |     parser.add_argument('--logdir', default="log/2", help="log directory")
40 |     parser.add_argument('--num_epochs', default=5, type=int)
41 |     parser.add_argument('--evaldir', default="eval/1", help="evaluation dir")
42 | 
43 |     # model
44 |     parser.add_argument('--d_model', default=512, type=int,
45 |                         help="hidden dimension of encoder/decoder")
46 |     parser.add_argument('--d_ff', default=2048, type=int,
47 |                         help="hidden dimension of feedforward layer")
48 |     parser.add_argument('--num_blocks', default=6, type=int,
49 |                         help="number of encoder/decoder blocks")
50 |     parser.add_argument('--num_heads', default=8, type=int,
51 |                         help="number of attention heads")
52 |     parser.add_argument('--maxlen1', default=150, type=int,
53 |                         help="maximum length of a source sequence")
54 |     parser.add_argument('--maxlen2', default=25, type=int,
55 |                         help="maximum length of a target sequence")
56 |     parser.add_argument('--dropout_rate', default=0.1, type=float)
57 |     parser.add_argument('--beam_size', default=4, type=int,
58 |                         help="beam size")
59 |     parser.add_argument('--gpu_nums', default=1, type=int,
60 |                         help="gpu amount, which can allow how many gpu to train this model")


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | import logging
  9 | 
 10 | import tensorflow as tf
 11 | from tqdm import tqdm
 12 | 
 13 | from data_load import _load_vocab
 14 | from modules import get_token_embeddings, ff, positional_encoding, multihead_attention, noam_scheme
 15 | from utils import convert_idx_to_token_tensor, split_input
 16 | 
 17 | logging.basicConfig(level=logging.INFO)
 18 | 
 19 | class Transformer:
 20 |     def __init__(self, hp):
 21 |         self.hp = hp
 22 |         self.token2idx, self.idx2token = _load_vocab(hp.vocab)
 23 |         self.embeddings = get_token_embeddings(self.hp.vocab_size, self.hp.d_model, zero_pad=True)
 24 | 
 25 |     def encode(self, xs, training=True):
 26 |         '''
 27 |         Returns
 28 |         memory: encoder outputs. (N, T1, d_model)
 29 |         '''
 30 |         with tf.variable_scope("encoder", reuse=tf.AUTO_REUSE):
 31 |             self.x, sents1 = xs
 32 | 
 33 |             # embedding
 34 |             enc = tf.nn.embedding_lookup(self.embeddings, self.x) # (N, T1, d_model)
 35 |             enc *= self.hp.d_model**0.5 # scale
 36 | 
 37 |             enc += positional_encoding(enc, self.hp.maxlen1)
 38 |             enc = tf.layers.dropout(enc, self.hp.dropout_rate, training=training)
 39 |             ## Blocks
 40 |             for i in range(self.hp.num_blocks):
 41 |                 with tf.variable_scope("num_blocks_{}".format(i), reuse=tf.AUTO_REUSE):
 42 |                     # self-attention
 43 |                     enc, _ = multihead_attention(queries=enc,
 44 |                                                   keys=enc,
 45 |                                                   values=enc,
 46 |                                                   num_heads=self.hp.num_heads,
 47 |                                                   dropout_rate=self.hp.dropout_rate,
 48 |                                                   training=training,
 49 |                                                   causality=False)
 50 |                     # feed forward
 51 |                     enc = ff(enc, num_units=[self.hp.d_ff, self.hp.d_model])
 52 |         self.enc_output = enc
 53 |         return self.enc_output, sents1
 54 | 
 55 |     def decode(self, xs, ys, memory, training=True):
 56 |         '''
 57 |         memory: encoder outputs. (N, T1, d_model)
 58 | 
 59 |         Returns
 60 |         logits: (N, T2, V). float32.
 61 |         y: (N, T2). int32
 62 |         sents2: (N,). string.
 63 |         '''
 64 |         self.memory = memory
 65 |         with tf.variable_scope("decoder", reuse=tf.AUTO_REUSE):
 66 |             self.decoder_inputs, y, sents2 = ys
 67 |             x, _, = xs
 68 | 
 69 |             # embedding
 70 |             dec = tf.nn.embedding_lookup(self.embeddings, self.decoder_inputs)  # (N, T2, d_model)
 71 |             dec *= self.hp.d_model ** 0.5  # scale
 72 | 
 73 |             dec += positional_encoding(dec, self.hp.maxlen2)
 74 | 
 75 |             before_dec = dec
 76 | 
 77 |             dec = tf.layers.dropout(dec, self.hp.dropout_rate, training=training)
 78 | 
 79 |             attn_dists = []
 80 |             # Blocks
 81 |             for i in range(self.hp.num_blocks):
 82 |                 with tf.variable_scope("num_blocks_{}".format(i), reuse=tf.AUTO_REUSE):
 83 |                     # Masked self-attention (Note that causality is True at this time)
 84 |                     dec, _ = multihead_attention(queries=dec,
 85 |                                                  keys=dec,
 86 |                                                  values=dec,
 87 |                                                  num_heads=self.hp.num_heads,
 88 |                                                  dropout_rate=self.hp.dropout_rate,
 89 |                                                  training=training,
 90 |                                                  causality=True,
 91 |                                                  scope="self_attention")
 92 |                     # Vanilla attention
 93 |                     dec, attn_dist = multihead_attention(queries=dec,
 94 |                                                           keys=self.memory,
 95 |                                                           values=self.memory,
 96 |                                                           num_heads=self.hp.num_heads,
 97 |                                                           dropout_rate=self.hp.dropout_rate,
 98 |                                                           training=training,
 99 |                                                           causality=False,
100 |                                                           scope="vanilla_attention")
101 |                     attn_dists.append(attn_dist)
102 |                     ### Feed Forward
103 |                     dec = ff(dec, num_units=[self.hp.d_ff, self.hp.d_model])
104 | 
105 |         # Final linear projection (embedding weights are shared)
106 |         weights = tf.transpose(self.embeddings) # (d_model, vocab_size)
107 |         logits = tf.einsum('ntd,dk->ntk', dec, weights) # (N, T2, vocab_size)
108 | 
109 |         with tf.variable_scope("gen", reuse=tf.AUTO_REUSE):
110 |             gens = tf.layers.dense(tf.concat([before_dec, dec, attn_dists[-1]], axis=-1), units=1, activation=tf.sigmoid,
111 |                                    trainable=training, use_bias=False)
112 | 
113 |         logits = tf.nn.softmax(logits)
114 | 
115 |         # final distribution
116 |         self.logits = self._calc_final_dist(x, gens, logits, attn_dists[-1])
117 | 
118 |         return self.logits, y, sents2
119 | 
120 |     def _calc_final_dist(self, x, gens, vocab_dists, attn_dists):
121 |         """Calculate the final distribution, for the pointer-generator model
122 | 
123 |         Args:
124 |           x: encoder input which contain oov number
125 |           gens: the generation, choose vocab from article or vocab
126 |           vocab_dists: The vocabulary distributions. List length max_dec_steps of (batch_size, vsize) arrays.
127 |                        The words are in the order they appear in the vocabulary file.
128 |           attn_dists: The attention distributions. List length max_dec_steps of (batch_size, attn_len) arrays
129 | 
130 |         Returns:
131 |           final_dists: The final distributions. List length max_dec_steps of (batch_size, extended_vsize) arrays.
132 |         """
133 |         with tf.variable_scope('final_distribution', reuse=tf.AUTO_REUSE):
134 |             # Multiply vocab dists by p_gen and attention dists by (1-p_gen)
135 |             vocab_dists = gens * vocab_dists
136 |             attn_dists = (1-gens) * attn_dists
137 | 
138 |             batch_size = tf.shape(attn_dists)[0]
139 |             dec_t = tf.shape(attn_dists)[1]
140 |             attn_len = tf.shape(attn_dists)[2]
141 | 
142 |             dec = tf.range(0, limit=dec_t) # [dec]
143 |             dec = tf.expand_dims(dec, axis=-1) # [dec, 1]
144 |             dec = tf.tile(dec, [1, attn_len]) # [dec, atten_len]
145 |             dec = tf.expand_dims(dec, axis=0) # [1, dec, atten_len]
146 |             dec = tf.tile(dec, [batch_size, 1, 1]) # [batch_size, dec, atten_len]
147 | 
148 |             x = tf.expand_dims(x, axis=1) # [batch_size, 1, atten_len]
149 |             x = tf.tile(x, [1, dec_t, 1]) # [batch_size, dec, atten_len]
150 |             x = tf.stack([dec, x], axis=3)
151 | 
152 |             attn_dists_projected = tf.map_fn(fn=lambda y: tf.scatter_nd(y[0], y[1], [dec_t, self.hp.vocab_size]),
153 |                                              elems=(x, attn_dists), dtype=tf.float32)
154 | 
155 |             final_dists = attn_dists_projected + vocab_dists
156 | 
157 |         return final_dists
158 | 
159 |     def _calc_loss(self, targets, final_dists):
160 |         """
161 |         calculate loss
162 |         :param targets: reference
163 |         :param final_dists:  transformer decoder output add by pointer generator
164 |         :return: loss
165 |         """
166 |         with tf.name_scope('loss'):
167 |             dec = tf.shape(targets)[1]
168 |             batch_nums = tf.shape(targets)[0]
169 |             dec = tf.range(0, limit=dec)
170 |             dec = tf.expand_dims(dec, axis=0)
171 |             dec = tf.tile(dec, [batch_nums, 1])
172 |             indices = tf.stack([dec, targets], axis=2) # [batch_size, dec, 2]
173 | 
174 |             loss = tf.map_fn(fn=lambda x: tf.gather_nd(x[1], x[0]), elems=(indices, final_dists), dtype=tf.float32)
175 |             loss = tf.log(0.9) - tf.log(loss)
176 | 
177 |             nonpadding = tf.to_float(tf.not_equal(targets, self.token2idx["<pad>"]))  # 0: <pad>
178 |             loss = tf.reduce_sum(loss * nonpadding) / (tf.reduce_sum(nonpadding) + 1e-7)
179 | 
180 |             return loss
181 | 
182 |     def train(self, xs, ys):
183 |         """
184 |         train model
185 |         :param xs: dataset xs
186 |         :param ys: dataset ys
187 |         :return: loss
188 |                  train op
189 |                  global step
190 |                  tensorflow summary
191 |         """
192 |         tower_grads = []
193 |         global_step = tf.train.get_or_create_global_step()
194 |         global_step_ = global_step * self.hp.gpu_nums
195 |         lr = noam_scheme(self.hp.d_model, global_step_, self.hp.warmup_steps)
196 |         optimizer = tf.train.AdamOptimizer(lr)
197 |         losses = []
198 |         xs, ys = split_input(xs, ys, self.hp.gpu_nums)
199 |         with tf.variable_scope(tf.get_variable_scope()):
200 |             for no in range(self.hp.gpu_nums):
201 |                 with tf.device("/gpu:%d" % no):
202 |                     with tf.name_scope("tower_%d" % no):
203 |                         memory, sents1 = self.encode(xs[no])
204 |                         logits, y, sents2 = self.decode(xs[no], ys[no], memory)
205 |                         tf.get_variable_scope().reuse_variables()
206 | 
207 |                         loss = self._calc_loss(y, logits)
208 |                         losses.append(loss)
209 |                         grads = optimizer.compute_gradients(loss)
210 |                         tower_grads.append(grads)
211 | 
212 |         with tf.device("/cpu:0"):
213 |             grads = self.average_gradients(tower_grads)
214 |             train_op = optimizer.apply_gradients(grads, global_step=global_step)
215 |             loss = sum(losses) / len(losses)
216 |             tf.summary.scalar('lr', lr)
217 |             tf.summary.scalar("train_loss", loss)
218 |             summaries = tf.summary.merge_all()
219 | 
220 |         return loss, train_op, global_step_, summaries
221 | 
222 |     def average_gradients(self, tower_grads):
223 |         """
224 |         average gradients of all gpu gradients
225 |         :param tower_grads: list, each element is a gradient of gpu
226 |         :return: be averaged gradient
227 |         """
228 |         average_grads = []
229 |         for grad_and_vars in zip(*tower_grads):
230 |             grads = []
231 |             for g, _ in grad_and_vars:
232 |                 expend_g = tf.expand_dims(g, 0)
233 |                 grads.append(expend_g)
234 |             grad = tf.concat(grads, 0)
235 |             grad = tf.reduce_mean(grad, 0)
236 |             v = grad_and_vars[0][1]
237 |             grad_and_var = (grad, v)
238 |             average_grads.append(grad_and_var)
239 | 
240 |         return average_grads
241 | 
242 |     def eval(self, xs, ys):
243 |         '''Predicts autoregressively
244 |         At inference, input ys is ignored.
245 |         Returns
246 |         y_hat: (N, T2)
247 |         tensorflow summary
248 |         '''
249 |         # decoder_inputs <s> sentences
250 |         decoder_inputs, y, sents2 = ys
251 | 
252 |         # decoder_inputs shape: [batch_size, 1] [[<s>], [<s>], [<s>], [<s>]]
253 |         decoder_inputs = tf.ones((tf.shape(xs[0])[0], 1), tf.int32) * self.token2idx["<s>"]
254 |         ys = (decoder_inputs, y, sents2)
255 | 
256 |         memory, sents1 = self.encode(xs, False)
257 | 
258 |         y_hat = None
259 |         logging.info("Inference graph is being built. Please be patient.")
260 |         for _ in tqdm(range(self.hp.maxlen2)):
261 |             logits, y, sents2 = self.decode(xs, ys, memory, False)
262 |             y_hat = tf.to_int32(tf.argmax(logits, axis=-1))
263 | 
264 |             if tf.reduce_sum(y_hat, 1) == self.token2idx["<pad>"]: break
265 | 
266 |             _decoder_inputs = tf.concat((decoder_inputs, y_hat), 1)
267 |             ys = (_decoder_inputs, y, sents2)
268 | 
269 |         # monitor a random sample
270 |         n = tf.random_uniform((), 0, tf.shape(y_hat)[0]-1, tf.int32)
271 |         sent1 = sents1[n]
272 |         pred = convert_idx_to_token_tensor(y_hat[n], self.idx2token)
273 |         sent2 = sents2[n]
274 | 
275 |         tf.summary.text("sent1", sent1)
276 |         tf.summary.text("pred", pred)
277 |         tf.summary.text("sent2", sent2)
278 |         summaries = tf.summary.merge_all()
279 | 
280 |         return y_hat, summaries


--------------------------------------------------------------------------------
/modules.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | 
  9 | import numpy as np
 10 | import tensorflow as tf
 11 | 
 12 | def ln(inputs, epsilon = 1e-8, scope="ln"):
 13 |     '''Applies layer normalization. See https://arxiv.org/abs/1607.06450.
 14 |     inputs: A tensor with 2 or more dimensions, where the first dimension has `batch_size`.
 15 |     epsilon: A floating number. A very small number for preventing ZeroDivision Error.
 16 |     scope: Optional scope for `variable_scope`.
 17 |       
 18 |     Returns:
 19 |       A tensor with the same shape and data dtype as `inputs`.
 20 |     '''
 21 |     with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
 22 |         inputs_shape = inputs.get_shape()
 23 |         params_shape = inputs_shape[-1:]
 24 |     
 25 |         mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
 26 |         beta= tf.get_variable("beta", params_shape, initializer=tf.zeros_initializer())
 27 |         gamma = tf.get_variable("gamma", params_shape, initializer=tf.ones_initializer())
 28 |         normalized = (inputs - mean) / ( (variance + epsilon) ** (.5) )
 29 |         outputs = gamma * normalized + beta
 30 |         
 31 |     return outputs
 32 | 
 33 | def get_token_embeddings(vocab_size, num_units, zero_pad=True):
 34 |     '''Constructs token embedding matrix.
 35 |     Note that the column of index 0's are set to zeros.
 36 |     vocab_size: scalar. V.
 37 |     num_units: embedding dimensionalty. E.
 38 |     zero_pad: Boolean. If True, all the values of the first row (id = 0) should be constant zero
 39 |     To apply query/key masks easily, zero pad is turned on.
 40 | 
 41 |     Returns
 42 |     weight variable: (V, E)
 43 |     '''
 44 |     with tf.variable_scope("shared_weight_matrix", reuse=tf.AUTO_REUSE):
 45 |         embeddings = tf.get_variable('weight_mat',
 46 |                                    dtype=tf.float32,
 47 |                                    shape=(vocab_size, num_units),
 48 |                                    initializer=tf.contrib.layers.xavier_initializer())
 49 |         if zero_pad:
 50 |             embeddings = tf.concat((tf.zeros(shape=[1, num_units]),
 51 |                                     embeddings[1:, :]), 0)
 52 |     return embeddings
 53 | 
 54 | def scaled_dot_product_attention(Q, K, V,
 55 |                                  num_heads,
 56 |                                  causality=False, dropout_rate=0.,
 57 |                                  training=True,
 58 |                                  scope="scaled_dot_product_attention"):
 59 |     '''See 3.2.1.
 60 |     Q: Packed queries. 3d tensor. [N, T_q, d_k].
 61 |     K: Packed keys. 3d tensor. [N, T_k, d_k].
 62 |     V: Packed values. 3d tensor. [N, T_k, d_v].
 63 |     causality: If True, applies masking for future blinding
 64 |     dropout_rate: A floating point number of [0, 1].
 65 |     training: boolean for controlling droput
 66 |     scope: Optional scope for `variable_scope`.
 67 |     '''
 68 |     with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
 69 |         d_k = Q.get_shape().as_list()[-1]
 70 | 
 71 |         # dot product
 72 |         outputs = tf.matmul(Q, tf.transpose(K, [0, 2, 1]))  # (N, T_q, T_k)
 73 | 
 74 |         # scale
 75 |         outputs /= d_k ** 0.5
 76 | 
 77 |         # key masking, delete key 0
 78 |         outputs = mask(outputs, Q, K, type="key")
 79 | 
 80 |         # causality or future blinding masking
 81 |         if causality:
 82 |             outputs = mask(outputs, type="future")
 83 | 
 84 |         # softmax
 85 |         attn_dists = tf.nn.softmax(tf.reduce_sum(tf.split(outputs, num_heads, axis=0), axis=0))
 86 |         outputs = tf.nn.softmax(outputs)
 87 |         attention = tf.transpose(outputs, [0, 2, 1])
 88 |         tf.summary.image("attention", tf.expand_dims(attention[:1], -1))
 89 | 
 90 |         # query masking, delete query <pad>
 91 |         outputs = mask(outputs, Q, K, type="query")
 92 | 
 93 |         # dropout
 94 |         outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training)
 95 | 
 96 |         # weighted sum (context vectors)
 97 |         outputs = tf.matmul(outputs, V)  # (N, T_q, d_v)
 98 | 
 99 |     return outputs, attn_dists
100 | 
101 | def mask(inputs, queries=None, keys=None, type=None):
102 |     """Masks paddings on keys or queries to inputs
103 |     inputs: 3d tensor. (N, T_q, T_k)
104 |     queries: 3d tensor. (N, T_q, d)
105 |     keys: 3d tensor. (N, T_k, d)
106 | 
107 |     e.g.,
108 |     >> queries = tf.constant([[[1.],
109 |                         [2.],
110 |                         [0.]]], tf.float32) # (1, 3, 1)
111 |     >> keys = tf.constant([[[4.],
112 |                      [0.]]], tf.float32)  # (1, 2, 1)
113 |     >> inputs = tf.constant([[[4., 0.],
114 |                                [8., 0.],
115 |                                [0., 0.]]], tf.float32)
116 |     >> mask(inputs, queries, keys, "key")
117 |     array([[[ 4.0000000e+00, -4.2949673e+09],
118 |         [ 8.0000000e+00, -4.2949673e+09],
119 |         [ 0.0000000e+00, -4.2949673e+09]]], dtype=float32)
120 |     >> inputs = tf.constant([[[1., 0.],
121 |                              [1., 0.],
122 |                               [1., 0.]]], tf.float32)
123 |     >> mask(inputs, queries, keys, "query")
124 |     array([[[1., 0.],
125 |         [1., 0.],
126 |         [0., 0.]]], dtype=float32)
127 |     """
128 |     padding_num = -2 ** 32 + 1
129 |     if type in ("k", "key", "keys"):
130 |         # Generate masks
131 |         masks = tf.sign(tf.reduce_sum(tf.abs(keys), axis=-1))  # (N, T_k)
132 |         masks = tf.expand_dims(masks, 1) # (N, 1, T_k)
133 |         masks = tf.tile(masks, [1, tf.shape(queries)[1], 1])  # (N, T_q, T_k)
134 | 
135 |         # Apply masks to inputs
136 |         paddings = tf.ones_like(inputs) * padding_num
137 | 
138 |         outputs = tf.where(tf.equal(masks, 0), paddings, inputs)  # (N, T_q, T_k)
139 |     elif type in ("q", "query", "queries"):
140 |         # Generate masks
141 |         masks = tf.sign(tf.reduce_sum(tf.abs(queries), axis=-1))  # (N, T_q)
142 |         masks = tf.expand_dims(masks, -1)  # (N, T_q, 1)
143 |         masks = tf.tile(masks, [1, 1, tf.shape(keys)[1]])  # (N, T_q, T_k)
144 | 
145 |         # Apply masks to inputs
146 |         outputs = inputs*masks
147 |     elif type in ("f", "future", "right"):
148 |         diag_vals = tf.ones_like(inputs[0, :, :])  # (T_q, T_k)
149 |         tril = tf.linalg.LinearOperatorLowerTriangular(diag_vals).to_dense()  # (T_q, T_k)
150 |         masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(inputs)[0], 1, 1])  # (N, T_q, T_k)
151 | 
152 |         paddings = tf.ones_like(masks) * padding_num
153 |         outputs = tf.where(tf.equal(masks, 0), paddings, inputs)
154 |     else:
155 |         print("Check if you entered type correctly!")
156 | 
157 | 
158 |     return outputs
159 | 
160 | def multihead_attention(queries, keys, values,
161 |                         num_heads=8, 
162 |                         dropout_rate=0,
163 |                         training=True,
164 |                         causality=False,
165 |                         scope="multihead_attention"):
166 |     '''Applies multihead attention. See 3.2.2
167 |     queries: A 3d tensor with shape of [N, T_q, d_model].
168 |     keys: A 3d tensor with shape of [N, T_k, d_model].
169 |     values: A 3d tensor with shape of [N, T_k, d_model].
170 |     num_heads: An int. Number of heads.
171 |     dropout_rate: A floating point number.
172 |     training: Boolean. Controller of mechanism for dropout.
173 |     causality: Boolean. If true, units that reference the future are masked.
174 |     scope: Optional scope for `variable_scope`.
175 |         
176 |     Returns
177 |       A 3d tensor with shape of (N, T_q, C)  
178 |     '''
179 |     d_model = queries.get_shape().as_list()[-1]
180 |     with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
181 |         # Linear projections
182 |         Q = tf.layers.dense(queries, d_model, use_bias=False) # (N, T_q, d_model)
183 |         K = tf.layers.dense(keys, d_model, use_bias=False) # (N, T_k, d_model)
184 |         V = tf.layers.dense(values, d_model, use_bias=False) # (N, T_k, d_model)
185 |         
186 |         # Split and concat
187 |         Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) # (h*N, T_q, d_model/h)
188 |         K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) # (h*N, T_k, d_model/h)
189 |         V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) # (h*N, T_k, d_model/h)
190 | 
191 |         # Attention
192 |         outputs, attn_dists = scaled_dot_product_attention(Q_, K_, V_, num_heads, causality, dropout_rate, training)
193 | 
194 |         # Restore shape
195 |         outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2) # (N, T_q, d_model)
196 |         # Residual connection
197 |         outputs = queries + outputs
198 |               
199 |         # Normalize
200 |         outputs = ln(outputs)
201 |  
202 |     return outputs, attn_dists
203 | 
204 | def ff(inputs, num_units, scope="positionwise_feedforward"):
205 |     '''position-wise feed forward net. See 3.3
206 |     
207 |     inputs: A 3d tensor with shape of [N, T, C].
208 |     num_units: A list of two integers.
209 |     scope: Optional scope for `variable_scope`.
210 | 
211 |     Returns:
212 |       A 3d tensor with the same shape and dtype as inputs
213 |     '''
214 |     with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
215 |         # Inner layer
216 |         outputs = tf.layers.dense(inputs, num_units[0], activation=tf.nn.relu)
217 | 
218 |         # Outer layer
219 |         outputs = tf.layers.dense(outputs, num_units[1])
220 | 
221 |         # Residual connection
222 |         outputs += inputs
223 |         
224 |         # Normalize
225 |         outputs = ln(outputs)
226 |     
227 |     return outputs
228 | 
229 | def label_smoothing(inputs, epsilon=0.1):
230 |     '''Applies label smoothing. See 5.4 and https://arxiv.org/abs/1512.00567.
231 |     inputs: 3d tensor. [N, T, V], where V is the number of vocabulary.
232 |     epsilon: Smoothing rate.
233 |     
234 |     For example,
235 |     
236 |     ```
237 |     import tensorflow as tf
238 |     inputs = tf.convert_to_tensor([[[0, 0, 1], 
239 |        [0, 1, 0],
240 |        [1, 0, 0]],
241 | 
242 |       [[1, 0, 0],
243 |        [1, 0, 0],
244 |        [0, 1, 0]]], tf.float32)
245 |        
246 |     outputs = label_smoothing(inputs)
247 |     
248 |     with tf.Session() as sess:
249 |         print(sess.run([outputs]))
250 |     
251 |     >>
252 |     [array([[[ 0.03333334,  0.03333334,  0.93333334],
253 |         [ 0.03333334,  0.93333334,  0.03333334],
254 |         [ 0.93333334,  0.03333334,  0.03333334]],
255 | 
256 |        [[ 0.93333334,  0.03333334,  0.03333334],
257 |         [ 0.93333334,  0.03333334,  0.03333334],
258 |         [ 0.03333334,  0.93333334,  0.03333334]]], dtype=float32)]   
259 |     ```    
260 |     '''
261 |     V = tf.cast(tf.shape(inputs)[-1], tf.float32) # number of channels
262 |     return ((1-epsilon) * inputs) + (epsilon / V)
263 |     
264 | def positional_encoding(inputs,
265 |                         maxlen,
266 |                         masking=True,
267 |                         scope="positional_encoding"):
268 |     '''Sinusoidal Positional_Encoding. See 3.5
269 |     inputs: 3d tensor. (N, T, E)
270 |     maxlen: scalar. Must be >= T
271 |     masking: Boolean. If True, padding positions are set to zeros.
272 |     scope: Optional scope for `variable_scope`.
273 | 
274 |     returns
275 |     3d tensor that has the same shape as inputs.
276 |     '''
277 | 
278 |     E = inputs.get_shape().as_list()[-1] # static
279 |     N, T = tf.shape(inputs)[0], tf.shape(inputs)[1] # dynamic
280 |     with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
281 |         # position indices
282 |         position_ind = tf.tile(tf.expand_dims(tf.range(T), 0), [N, 1]) # (N, T)
283 | 
284 |         # First part of the PE function: sin and cos argument
285 |         position_enc = np.array([
286 |             [pos / np.power(10000, (i-i%2)/E) for i in range(E)]
287 |             for pos in range(maxlen)])
288 | 
289 |         # Second part, apply the cosine to even columns and sin to odds.
290 |         position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])  # dim 2i
291 |         position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])  # dim 2i+1
292 |         position_enc = tf.convert_to_tensor(position_enc, tf.float32) # (maxlen, E)
293 | 
294 |         # lookup
295 |         outputs = tf.nn.embedding_lookup(position_enc, position_ind)
296 | 
297 |         # masks
298 |         if masking:
299 |             outputs = tf.where(tf.equal(inputs, 0), inputs, outputs)
300 | 
301 |         return tf.to_float(outputs)
302 | 
303 | def noam_scheme(d_model, global_step, warmup_steps=4000.):
304 |     '''Noam scheme learning rate decay
305 |     d_model: encoder and decoder embedding
306 |     global_step: scalar.
307 |     warmup_steps: scalar. During warmup_steps, learning rate increases
308 |         until it reaches init_lr.
309 |     '''
310 |     step = tf.cast(global_step + 1, dtype=tf.float32)
311 |     return d_model ** -0.5 * tf.minimum(step * warmup_steps ** -1.5, step ** -0.5)


--------------------------------------------------------------------------------
/pred.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | import os
  9 | 
 10 | from beam_search import BeamSearch
 11 | from data_load import _load_vocab
 12 | from hparams import Hparams
 13 | from model import Transformer
 14 | 
 15 | def import_tf(device_id=-1, verbose=False):
 16 |     """
 17 |     import tensorflow, set tensorflow graph load device, set tensorflow log level, return tensorflow instance
 18 |     :param device_id: GPU id
 19 |     :param verbose: tensorflow logging level
 20 |     :return: tensorflow instance
 21 |     """
 22 |     # set visible gpu, -1 is cpu
 23 |     os.environ['CUDA_VISIBLE_DEVICES'] = '-1' if device_id < 0 else str(device_id)
 24 |     os.environ['TF_CPP_MIN_LOG_LEVEL'] = '0' if verbose else '3'
 25 |     import tensorflow as tf
 26 |     tf.logging.set_verbosity(tf.logging.DEBUG if verbose else tf.logging.ERROR)
 27 |     return tf
 28 | 
 29 | class Prediction:
 30 |     def __init__(self, args):
 31 |         """
 32 |         :param model_dir: model dir path
 33 |         :param vocab_file: vocab file path
 34 |         """
 35 |         self.tf = import_tf(0)
 36 | 
 37 |         self.args = args
 38 |         self.model_dir = args.logdir
 39 |         self.vocab_file = args.vocab
 40 |         self.token2idx, self.idx2token = _load_vocab(args.vocab)
 41 | 
 42 |         hparams = Hparams()
 43 |         parser = hparams.parser
 44 |         self.hp = parser.parse_args()
 45 | 
 46 |         self.model = Transformer(self.hp)
 47 | 
 48 |         self._add_placeholder()
 49 |         self._init_graph()
 50 | 
 51 |     def _init_graph(self):
 52 |         """
 53 |         init graph
 54 |         """
 55 |         self.ys = (self.input_y, None, None)
 56 |         self.xs = (self.input_x, None)
 57 |         self.memory = self.model.encode(self.xs, False)[0]
 58 |         self.logits = self.model.decode(self.xs, self.ys, self.memory, False)[0]
 59 | 
 60 |         ckpt = self.tf.train.get_checkpoint_state(self.model_dir).all_model_checkpoint_paths[-1]
 61 | 
 62 |         graph = self.logits.graph
 63 |         sess_config = self.tf.ConfigProto(allow_soft_placement=True)
 64 |         sess_config.gpu_options.allow_growth = True
 65 | 
 66 |         saver = self.tf.train.Saver()
 67 |         self.sess = self.tf.Session(config=sess_config, graph=graph)
 68 | 
 69 |         self.sess.run(self.tf.global_variables_initializer())
 70 |         self.tf.reset_default_graph()
 71 |         saver.restore(self.sess, ckpt)
 72 | 
 73 |         self.bs = BeamSearch(self.model,
 74 |                              self.hp.beam_size,
 75 |                              list(self.idx2token.keys())[2],
 76 |                              list(self.idx2token.keys())[3],
 77 |                              self.idx2token,
 78 |                              self.hp.maxlen2,
 79 |                              self.input_x,
 80 |                              self.input_y,
 81 |                              self.logits)
 82 | 
 83 |     def predict(self, content):
 84 |         """
 85 |         abstract prediction by beam search
 86 |         :param content: article content
 87 |         :return: prediction result
 88 |         """
 89 |         input_x = list(content)
 90 |         while len(input_x) < self.args.maxlen1: input_x.append('<pad>')
 91 |         input_x = input_x[:self.args.maxlen1]
 92 | 
 93 |         input_x = [self.token2idx.get(s, self.token2idx['<unk>']) for s in input_x]
 94 | 
 95 |         memory = self.sess.run(self.memory, feed_dict={self.input_x: [input_x]})
 96 | 
 97 |         return self.bs.search(self.sess, input_x, memory[0])
 98 | 
 99 |     def _add_placeholder(self):
100 |         """
101 |         add tensorflow placeholder
102 |         """
103 |         self.input_x = self.tf.placeholder(dtype=self.tf.int32, shape=[None, self.args.maxlen1], name='input_x')
104 |         self.input_y = self.tf.placeholder(dtype=self.tf.int32, shape=[None, None], name='input_y')
105 | 
106 | if __name__ == '__main__':
107 |     hparams = Hparams()
108 |     parser = hparams.parser
109 |     hp = parser.parse_args()
110 |     preds = Prediction(hp)
111 |     content = '2014年，51信用卡管家跟宜信等P2P公司合作，推出线上信贷产品“瞬时贷”，其是一种纯在线操作的信贷模式。51信用卡管家创始人孙海涛说，51目前每天放贷1000万，预计2015年，自营产品加上瞬>时贷，放贷额度将远超'
112 |     result = preds.predict(content)
113 |     for res in result:
114 |         print(res)


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | tensorflow==1.15.4
2 | tqdm>=4.28.1
3 | jieba>=0.3x
4 | sumeval>=0.2.0


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | 
  9 | import logging
 10 | import os
 11 | 
 12 | from sumeval.metrics.rouge import RougeCalculator
 13 | from tqdm import tqdm
 14 | 
 15 | from beam_search import BeamSearch
 16 | from data_load import get_batch, _load_vocab
 17 | from hparams import Hparams
 18 | from model import Transformer
 19 | from utils import save_hparams, save_variable_specs, get_hypotheses, calc_rouge, import_tf
 20 | 
 21 | logging.basicConfig(level=logging.INFO)
 22 | 
 23 | rouge = RougeCalculator(stopwords=True, lang="zh")
 24 | 
 25 | logging.info("# hparams")
 26 | hparams = Hparams()
 27 | parser = hparams.parser
 28 | hp = parser.parse_args()
 29 | 
 30 | # import tensorflow
 31 | gpu_list = [str(i) for i in list(range(hp.gpu_nums))]
 32 | tf = import_tf(gpu_list)
 33 | 
 34 | save_hparams(hp, hp.logdir)
 35 | 
 36 | logging.info("# Prepare train/eval batches")
 37 | train_batches, num_train_batches, num_train_samples = get_batch(hp.train,
 38 |                                                                 hp.maxlen1,
 39 |                                                                 hp.maxlen2,
 40 |                                                                 hp.vocab,
 41 |                                                                 hp.batch_size,
 42 |                                                                 hp.gpu_nums,
 43 |                                                                 shuffle=True)
 44 | 
 45 | eval_batches, num_eval_batches, num_eval_samples = get_batch(hp.eval,
 46 |                                                              hp.maxlen1,
 47 |                                                              hp.maxlen2,
 48 |                                                              hp.vocab,
 49 |                                                              hp.eval_batch_size,
 50 |                                                              hp.gpu_nums,
 51 |                                                              shuffle=False)
 52 | 
 53 | handle = tf.placeholder(tf.string, shape=[])
 54 | iter = tf.data.Iterator.from_string_handle(
 55 |     handle, train_batches.output_types, train_batches.output_shapes)
 56 | 
 57 | # create a iter of the correct shape and type
 58 | xs, ys = iter.get_next()
 59 | 
 60 | logging.info('# init data')
 61 | training_iter = train_batches.make_one_shot_iterator()
 62 | val_iter = eval_batches.make_initializable_iterator()
 63 | 
 64 | logging.info("# Load model")
 65 | m = Transformer(hp)
 66 | 
 67 | # get op
 68 | loss, train_op, global_step, train_summaries = m.train(xs, ys)
 69 | y_hat, eval_summaries = m.eval(xs, ys)
 70 | 
 71 | token2idx, idx2token = _load_vocab(hp.vocab)
 72 | 
 73 | bs = BeamSearch(m, hp.beam_size, list(idx2token.keys())[2], list(idx2token.keys())[3], idx2token, hp.maxlen2, m.x,
 74 |                 m.decoder_inputs, m.logits)
 75 | 
 76 | logging.info("# Session")
 77 | saver = tf.train.Saver(max_to_keep=hp.num_epochs)
 78 | with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
 79 |     ckpt = tf.train.latest_checkpoint(hp.logdir)
 80 |     if ckpt is None:
 81 |         logging.info("Initializing from scratch")
 82 |         sess.run(tf.global_variables_initializer())
 83 |         save_variable_specs(os.path.join(hp.logdir, "specs"))
 84 |     else:
 85 |         saver.restore(sess, ckpt)
 86 | 
 87 |     summary_writer = tf.summary.FileWriter(hp.logdir, sess.graph)
 88 | 
 89 |     # Iterator.string_handle() get a tensor that can be got value to feed handle placeholder
 90 |     training_handle = sess.run(training_iter.string_handle())
 91 |     val_handle = sess.run(val_iter.string_handle())
 92 | 
 93 |     total_steps = hp.num_epochs * num_train_batches
 94 |     _gs = sess.run(global_step)
 95 |     for i in tqdm(range(_gs, total_steps+1)):
 96 |         _, _gs, _summary = sess.run([train_op, global_step, train_summaries], feed_dict={handle: training_handle})
 97 |         summary_writer.add_summary(_summary, _gs)
 98 |         if _gs % (hp.gpu_nums * 5000) == 0 and _gs != 0:
 99 |             logging.info("steps {} is done".format(_gs))
100 | 
101 |             logging.info("# test evaluation")
102 |             sess.run(val_iter.initializer) # initial val dataset
103 |             _eval_summaries = sess.run(eval_summaries, feed_dict={handle: val_handle})
104 |             summary_writer.add_summary(_eval_summaries, _gs)
105 | 
106 |             logging.info("# beam search")
107 |             hypotheses, all_targets = get_hypotheses(num_eval_batches, num_eval_samples, sess, m, bs, [xs[0], ys[2]],
108 |                                                      handle, val_handle)
109 | 
110 |             logging.info("# calc rouge score ")
111 |             if not os.path.exists(hp.evaldir): os.makedirs(hp.evaldir)
112 |             rouge_l = calc_rouge(rouge, all_targets, hypotheses, _gs, hp.evaldir)
113 | 
114 |             model_output = "trans_pointer%02dL%.2f" % (_gs, rouge_l)
115 | 
116 |             logging.info('# write hypotheses')
117 |             with open(os.path.join(hp.evaldir, model_output), 'w', encoding='utf-8') as f:
118 |                 for target, hypothes in zip(all_targets, hypotheses):
119 |                     f.write('{}-{} \n'.format(target, ' '.join(hypothes)))
120 | 
121 |             logging.info("# save models")
122 | 
123 |             ckpt_name = os.path.join(hp.logdir, model_output)
124 |             saver.save(sess, ckpt_name, global_step=_gs)
125 |             logging.info("after training of {} steps, {} has been saved.".format(_gs, ckpt_name))
126 | 
127 |             logging.info("# fall back to train mode")
128 |     summary_writer.close()
129 | 
130 | logging.info("Done")


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #/usr/bin/python3
  3 | '''
  4 | date: 2019/5/21
  5 | mail: cally.maxiong@gmail.com
  6 | page: http://www.cnblogs.com/callyblog/
  7 | '''
  8 | import json
  9 | import logging
 10 | import os
 11 | from tqdm import tqdm
 12 | 
 13 | logging.basicConfig(level=logging.INFO)
 14 | 
 15 | def calc_num_batches(total_num, batch_size):
 16 |     '''Calculates the number of batches.
 17 |     total_num: total sample number
 18 |     batch_size
 19 | 
 20 |     Returns
 21 |     number of batches, allowing for remainders.'''
 22 |     return total_num // batch_size + int(total_num % batch_size != 0)
 23 | 
 24 | def convert_idx_to_token_tensor(inputs, idx2token):
 25 |     '''Converts int32 tensor to string tensor.
 26 |     inputs: 1d int32 tensor. indices.
 27 |     idx2token: dictionary
 28 | 
 29 |     Returns
 30 |     1d string tensor.
 31 |     '''
 32 |     import tensorflow as tf
 33 |     def my_func(inputs):
 34 |         return " ".join(idx2token[elem] for elem in inputs)
 35 | 
 36 |     return tf.py_func(my_func, [inputs], tf.string)
 37 | 
 38 | def postprocess(hypotheses):
 39 |     '''Processes translation outputs.
 40 |     hypotheses: list of encoded predictions
 41 |     idx2token: dictionary
 42 | 
 43 |     Returns
 44 |     processed hypotheses
 45 |     '''
 46 |     _hypotheses = []
 47 |     for h in hypotheses:
 48 |         h = str(h)
 49 |         h = h.replace('<s>', '')
 50 |         h = h.replace('</s>', '')
 51 |         h = h.replace('<pad>', '')
 52 |         _hypotheses.append(h)
 53 | 
 54 |     return _hypotheses
 55 | 
 56 | def save_hparams(hparams, path):
 57 |     '''Saves hparams to path
 58 |     hparams: argsparse object.
 59 |     path: output directory.
 60 | 
 61 |     Writes
 62 |     hparams as literal dictionary to path.
 63 |     '''
 64 |     if not os.path.exists(path): os.makedirs(path)
 65 |     hp = json.dumps(vars(hparams))
 66 |     with open(os.path.join(path, "hparams"), 'w') as fout:
 67 |         fout.write(hp)
 68 | 
 69 | def load_hparams(parser, path):
 70 |     '''Loads hparams and overrides parser
 71 |     parser: argsparse parser
 72 |     path: directory or file where hparams are saved
 73 |     '''
 74 |     if not os.path.isdir(path):
 75 |         path = os.path.dirname(path)
 76 |     d = open(os.path.join(path, "hparams"), 'r').read()
 77 |     flag2val = json.loads(d)
 78 |     for f, v in flag2val.items():
 79 |         parser.f = v
 80 | 
 81 | def save_variable_specs(fpath):
 82 |     '''Saves information about variables such as
 83 |     their name, shape, and total parameter number
 84 |     fpath: string. output file path
 85 | 
 86 |     Writes
 87 |     a text file named fpath.
 88 |     '''
 89 |     import tensorflow as tf
 90 |     def _get_size(shp):
 91 |         '''Gets size of tensor shape
 92 |         shp: TensorShape
 93 | 
 94 |         Returns
 95 |         size
 96 |         '''
 97 |         size = 1
 98 |         for d in range(len(shp)):
 99 |             size *=shp[d]
100 |         return size
101 | 
102 |     params, num_params = [], 0
103 |     for v in tf.global_variables():
104 |         params.append("{}==={}".format(v.name, v.shape))
105 |         num_params += _get_size(v.shape)
106 |     print("num_params: ", num_params)
107 |     with open(fpath, 'w') as fout:
108 |         fout.write("num_params: {}\n".format(num_params))
109 |         fout.write("\n".join(params))
110 |     logging.info("Variables info has been saved.")
111 | 
112 | def get_hypotheses(num_batches, num_samples, sess, model, beam_search, tensor, handle_placehoder, handle):
113 |     '''Gets hypotheses.
114 |     num_batches: scalar.
115 |     num_samples: scalar.
116 |     sess: tensorflow sess object
117 |     tensor: target tensor to fetch
118 |     dict: idx2token dictionary
119 | 
120 |     Returns
121 |     hypotheses: list of sents
122 |     '''
123 |     hypotheses, all_targets = [], []
124 |     for _ in tqdm(range(num_batches)):
125 |         articles, targets = sess.run(tensor, feed_dict={handle_placehoder: handle})
126 |         memories = sess.run(model.enc_output, feed_dict={model.x: articles})
127 |         for article, memory in zip(articles, memories):
128 |             summary = beam_search.search(sess, article, memory)
129 |             summary = postprocess(summary)
130 |             hypotheses.append(summary)
131 |         all_targets.extend([target.decode('utf-8') for target in targets])
132 | 
133 |     return hypotheses[:num_samples], all_targets[:num_samples]
134 | 
135 | def calc_rouge(rouge, references, models, global_step, logdir):
136 |     """
137 |     calculate rouge score
138 |     :param references: reference sentences
139 |     :param models: model sentences
140 |     :param global_step: global step
141 |     :param logdir: log dir
142 |     :return: rouge score
143 |     """
144 |     # delete symbol
145 |     references = [reference.replace('</s>', '') for reference in references]
146 | 
147 |     # calculate rouge score
148 |     rouge1_scores = [_rouge(rouge, model, reference, type='rouge1') for model, reference in zip(models, references)]
149 |     rouge2_scores = [_rouge(rouge, model, reference, type='rouge2') for model, reference in zip(models, references)]
150 |     rougel_scores = [_rouge(rouge, model, reference, type='rougel') for model, reference in zip(models, references)]
151 | 
152 |     # get rouge score
153 |     rouge1_score = sum(rouge1_scores) / len(rouge1_scores)
154 |     rouge2_score = sum(rouge2_scores) / len(rouge2_scores)
155 |     rougel_score = sum(rougel_scores) / len(rouge2_scores)
156 | 
157 |     # write result
158 |     with open(os.path.join(logdir, 'rouge'), 'a', encoding='utf-8') as f:
159 |         f.write('global step: {}, ROUGE 1: {}, ROUGE 2: {}, ROUGE L: {}\n'.format(str(global_step), str(rouge1_score),
160 |                                                                                   str(rouge2_score), str(rougel_score)))
161 |     return rouge1_score
162 | 
163 | def _rouge(rouge, model, reference, type='rouge1'):
164 |     """
165 |     calculate rouge socore
166 |     :param rouge: sumeval instance
167 |     :param model: model prediction, list
168 |     :param reference: reference
169 |     :param type: rouge1, rouge2, rougel
170 |     :return: rouge 1 score
171 |     """
172 |     scores = None
173 |     if type == 'rouge1':
174 |         scores = [rouge.rouge_n(summary=m, references=reference, n=1) for m in model]
175 | 
176 |     if type == 'rouge2':
177 |         scores = [rouge.rouge_n(summary=m, references=reference, n=2) for m in model]
178 | 
179 |     if type == 'rougel':
180 |         scores = [rouge.rouge_l(summary=m, references=reference) for m in model]
181 | 
182 |     return max(scores)
183 | 
184 | def import_tf(gpu_list):
185 |     """
186 |     import tensorflow, set tensorflow graph load device
187 |     :param gpu_list: GPU list
188 |     :return: tensorflow instance
189 |     """
190 |     import tensorflow as tf
191 |     os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(gpu_list)
192 | 
193 |     return tf
194 | 
195 | def split_input(xs, ys, gpu_nums):
196 |     """
197 |     split input
198 |     :param xs: articles
199 |     :param ys: summaries
200 |     :param gpu_nums: gpu numbers
201 |     :return: split input by gpu numbers
202 |     """
203 |     import tensorflow as tf
204 |     xs = [tf.split(x, num_or_size_splits=gpu_nums, axis=0) for x in xs]
205 |     ys = [tf.split(y, num_or_size_splits=gpu_nums, axis=0) for y in ys]
206 | 
207 |     return [(xs[0][i], xs[1][i]) for i in range(gpu_nums)], [(ys[0][i], ys[1][i], ys[2][i]) for i in range(gpu_nums)]


--------------------------------------------------------------------------------