├── .gitignore ├── README.md ├── data └── orm_data │ ├── debug.tok.eng.phrase │ ├── eng.model │ ├── eng.vocab │ ├── full-vocab.orm │ ├── orm.model │ ├── orm.vocab │ ├── set0-dev.interleave.eng │ ├── set0-dev.interleave.null.eng │ ├── set0-dev.nltk.eng │ ├── set0-dev.nltk.orm │ ├── set0-dev.nltk.orm-eng │ ├── set0-dev.orig.eng │ ├── set0-dev.orig.orm │ ├── set0-dev.orig.orm-eng │ ├── set0-dev.parse.bpe.eng │ ├── set0-dev.tok.eng │ ├── set0-dev.tok.eng.bina │ ├── set0-dev.tok.eng.bina+w │ ├── set0-dev.tok.eng.dep │ ├── set0-dev.tok.eng.phrase │ ├── set0-dev.tok.eng.right_bina │ ├── set0-dev.tok.eng.right_branch │ ├── set0-dev.tok.eng.tri │ ├── set0-dev.tok.eng.w_bina │ ├── set0-dev.tok.orm │ ├── set0-dev.tok.parse.eng │ ├── set0-dev.tok.piece.eng │ ├── set0-dev.tok.piece.eng.bina+w │ ├── set0-dev.tok.piece.eng.post │ ├── set0-dev.tok.piece.orm │ ├── set0-dev.tok.piece.orm.bina+w │ ├── set0-dev.tok.piece.orm.post │ ├── set0-test.nltk.eng │ ├── set0-test.nltk.orm │ ├── set0-test.nltk.orm-eng │ ├── set0-test.nopos.parse.eng │ ├── set0-test.orig.eng │ ├── set0-test.orig.orm │ ├── set0-test.orig.orm-eng │ ├── set0-test.parse.bpe.eng │ ├── set0-test.tok.eng │ ├── set0-test.tok.eng.bina │ ├── set0-test.tok.eng.bina+w │ ├── set0-test.tok.eng.dep │ ├── set0-test.tok.eng.phrase │ ├── set0-test.tok.eng.right_bina │ ├── set0-test.tok.eng.right_branch │ ├── set0-test.tok.eng.tri │ ├── set0-test.tok.eng.w_bina │ ├── set0-test.tok.orm │ ├── set0-test.tok.parse.eng │ ├── set0-test.tok.piece.eng │ ├── set0-test.tok.piece.orm │ ├── set0-train.nltk.eng │ ├── set0-train.nltk.orm │ ├── set0-train.nltk.orm-eng │ ├── set0-trainunfilt-head10.tok.parse.eng │ ├── set0-trainunfilt-head10.tok.piece.eng │ ├── set0-trainunfilt-head10.tok.piece.orm │ ├── set0-trainunfilt-head100.tok.parse.eng │ ├── set0-trainunfilt-head100.tok.piece.eng │ ├── set0-trainunfilt-head100.tok.piece.orm │ ├── set0-trainunfilt.interleave.eng │ ├── set0-trainunfilt.interleave.null.eng │ ├── set0-trainunfilt.nltk.eng │ ├── set0-trainunfilt.nltk.orm │ ├── set0-trainunfilt.nltk.orm-eng │ ├── set0-trainunfilt.orig.eng │ ├── set0-trainunfilt.orig.orm │ ├── set0-trainunfilt.orig.orm-eng │ ├── set0-trainunfilt.parse.bpe.eng │ ├── set0-trainunfilt.tok.eng │ ├── set0-trainunfilt.tok.eng.bina │ ├── set0-trainunfilt.tok.eng.bina+w │ ├── set0-trainunfilt.tok.eng.dep │ ├── set0-trainunfilt.tok.eng.phrase │ ├── set0-trainunfilt.tok.eng.right_bina │ ├── set0-trainunfilt.tok.eng.right_branch │ ├── set0-trainunfilt.tok.eng.tri │ ├── set0-trainunfilt.tok.eng.w_bina │ ├── set0-trainunfilt.tok.orm │ ├── set0-trainunfilt.tok.parse.eng │ ├── set0-trainunfilt.tok.piece.eng │ ├── set0-trainunfilt.tok.piece.eng.bina+w │ ├── set0-trainunfilt.tok.piece.orm │ ├── set0-trainunfilt.tok.piece.orm.bina+w │ ├── set0.bpe │ ├── set0.null.tag │ ├── set0.tag │ ├── setE-500.clean.piece.eng │ ├── setE-500.clean.piece.orm │ ├── setE-500.clean.ptok.eng │ ├── setE-500.clean.ptok.eng.right_branch │ ├── setE-500.clean.ptok.orm │ ├── setE-500.clean.tok.eng │ ├── setE-500.clean.tok.orm │ ├── setE-500.tok.eng │ ├── setE-500.tok.orm │ ├── train_level3.parse.eng │ ├── train_level3_nopos.parse.eng │ ├── train_nopos.parse.eng │ ├── unseq-setE.tok.eng │ ├── unseq-setE.tok.orm │ ├── vocab.bina_rule.eng │ ├── vocab.bina_word.eng │ ├── vocab.ccg.eng │ ├── vocab.ccg.null.eng │ ├── vocab.d4-rule.eng │ ├── vocab.d4-word.eng │ ├── vocab.dep_rule.eng │ ├── vocab.dep_word.eng │ ├── vocab.eng │ ├── vocab.lin_parse.eng │ ├── vocab.no-lhs-rule.eng │ ├── vocab.no-lhs-word.eng │ ├── vocab.orm │ ├── vocab.phrase_rule.eng │ ├── vocab.phrase_word.eng │ ├── vocab.pos-no-lhs-rule.eng │ ├── vocab.pos-no-lhs-word.eng │ ├── vocab.random_bina_rule.eng │ ├── vocab.random_bina_word.eng │ ├── vocab.right_branch.eng │ ├── vocab.right_branch_rule.eng │ ├── vocab.right_branch_word.eng │ ├── vocab.rule.eng │ ├── vocab.tri_rule.eng │ ├── vocab.tri_word.eng │ └── vocab.word.eng ├── multi-bleu.perl ├── scripts ├── orm_trdec.sh └── orm_trdec_translate.sh └── src ├── __init__.py ├── clean_data.py ├── correct_dep.py ├── data_utils.py ├── get_rule_vocab.py ├── get_vocab.py ├── hparams.py ├── main.py ├── make_tree.py ├── models.py ├── null_ccg.py ├── translate.py ├── trdec.py ├── trdec_attn.py ├── trdec_attn_v1.py ├── trdec_single.py ├── tree_utils.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Auto-generated files 2 | *.swp 3 | *.pyc 4 | outputs* 5 | outputs/* 6 | !archived_exp/* 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # dotenv 90 | .env 91 | 92 | # virtualenv 93 | .venv 94 | venv/ 95 | ENV/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | 110 | data/ 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TrDec: a tree-based decoder for neural machine translation 2 | 3 | source code for paper 4 | > [A Tree-based Decoder for Neural Machine Translation] (https://arxiv.org/abs/1808.09374) 5 | 6 | > Xinyi Wang, Hieu Pham, Pengcheng Yin, Graham Neubig 7 | 8 | ## Requirements 9 | 10 | Python 3.6, PyTorch 0.4.1 11 | 12 | ## Training 13 | We provide the data and scripts to run the experiments for the small orormo dataset 14 | To train the oromo TrDec model 15 | 16 | `./scripts/orm_trdec.py` 17 | 18 | ## Decode 19 | To decode from the trained TrDec model 20 | 21 | `./scripts/orm_trdec_translate.py` 22 | 23 | ## 24 | We will update with other scripts used for the paper soon 25 | -------------------------------------------------------------------------------- /data/orm_data/debug.tok.eng.phrase: -------------------------------------------------------------------------------- 1 | (ROOT (ROOT (ROOT (ROOT (ROOT (ROOT (ROOT And)(ROOT the))(ROOT winepress))(ROOT (ROOT was)(ROOT trodden)))(ROOT (ROOT (ROOT outside)(ROOT the))(ROOT (ROOT city)(ROOT ,))))(ROOT (ROOT (ROOT (ROOT and)(ROOT blood))(ROOT (ROOT flowed)(ROOT from)))(ROOT (ROOT (ROOT the)(ROOT winepress))(ROOT ,))))(ROOT (ROOT (ROOT (ROOT (ROOT as)(ROOT high))(ROOT (ROOT as)(ROOT a)))(ROOT (ROOT (ROOT horse)(ROOT 's))(ROOT (ROOT bridle)(ROOT ,))))(ROOT (ROOT (ROOT for)(ROOT 1,600))(ROOT (ROOT stadia)(ROOT .))))) 2 | -------------------------------------------------------------------------------- /data/orm_data/eng.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cindyxinyiwang/TrDec_pytorch/877bb2a7949828b3da13ac3345a8a26650177697/data/orm_data/eng.model -------------------------------------------------------------------------------- /data/orm_data/orm.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cindyxinyiwang/TrDec_pytorch/877bb2a7949828b3da13ac3345a8a26650177697/data/orm_data/orm.model -------------------------------------------------------------------------------- /data/orm_data/set0-trainunfilt-head10.tok.parse.eng: -------------------------------------------------------------------------------- 1 | (ROOT (S (NP (FW i)) (VP (VBP like) (NP (PRP$ my) (NN steak) (NN medium))) (. .)) ) 2 | (ROOT (FRAG (`` ``) (INTJ (UH Woe) (PP (TO to) (NP (PRP you))) (, ,) (INTJ (UH Chorazin) (. !) (INTJ (UH Woe) (PP (TO to) (NP (PRP you))) (, ,) (NNP Bethsaida)))) (. !) (S (IN For) (SBAR (IN if) (S (NP (NP (DT the) (JJ mighty) (NNS works)) (VP (VBN done) (PP (IN in) (NP (PRP you))))) (VP (VBD had) (VP (VBN been) (VP (VBN done) (PP (IN in) (NP (NNP Tyre) (CC and) (NNP Sidon)))))))) (, ,) (NP (PRP they)) (VP (MD would) (VP (VB have) (VP (VBN repented) (ADVP (RB long) (RB ago)) (, ,) (S (VP (VBG sitting) (PP (IN in) (NP (NN sackcloth) (CC and) (NNS ashes))))))))) (. .)) ) 3 | (ROOT (FRAG (`` ``) (INTJ (UH Woe) (PP (TO to) (NP (PRP you))) (, ,) (INTJ (UH Chorazin) (. !) (INTJ (UH Woe) (PP (TO to) (NP (PRP you))) (, ,) (NNP Bethsaida)))) (. !) (S (IN For) (SBAR (IN if) (S (NP (NP (DT the) (JJ mighty) (NNS works)) (VP (VBN done) (PP (IN in) (NP (PRP you))))) (VP (VBD had) (VP (VBN been) (VP (VBN done) (PP (IN in) (NP (NNP Tyre) (CC and) (NNP Sidon)))))))) (, ,) (NP (PRP they)) (VP (MD would) (VP (VB have) (VP (VBD repented) (ADVP (RB long)) (ADVP (RB ago) (PP (IN in) (NP (NN sackcloth) (CC and) (NNS ashes)))))))) (. .)) ) 4 | (ROOT (S (ADVP (RB Immediately)) (NP (PRP he)) (VP (VBD made) (S (NP (PRP$ his) (NNS disciples)) (VP (VP (VB get) (PP (IN into) (NP (DT the) (NN boat)))) (CC and) (VP (VB go) (PP (IN before) (NP (PRP him))) (PP (TO to) (NP (DT the) (JJ other) (NN side)))))) (, ,) (PP (TO to) (NP (NNP Bethsaida))) (, ,) (SBAR (IN while) (S (NP (PRP he)) (VP (VBD dismissed) (NP (DT the) (NN crowd)))))) (. .)) ) 5 | (ROOT (S (VP (VB Let) (S (NP (NP (NP (DT all) (NN bitterness) (CC and) (NN wrath)) (CC and) (NN anger)) (CC and) (NP (NN clamor) (CC and) (NN slander))) (VP (VB be) (VP (VBN put) (PRT (RP away)) (PP (IN from) (NP (PRP you))) (, ,) (PP (IN along) (PP (IN with) (NP (DT all) (NN malice)))))))) (. .)) ) 6 | (ROOT (S (NP (PRP I)) (VP (VBP have) (VP (VBN been) (VP (VBN crucified) (PP (IN with) (NP (NNP Christ)))))) (. .) (NP (PRP It)) (VP (VBZ is) (ADVP (RB no) (RB longer)) (NP (NP (NP (PRP I)) (SBAR (WHNP (WP who)) (S (VBP live)))) (, ,) (CC but) (NP (NP (NNP Christ)) (SBAR (WHNP (WP who)) (S (VP (VBZ lives) (PP (IN in) (NP (PRP me))))))))) (. .) (S (CC And) (NP (NP (DT the) (NN life)) (SBAR (S (NP (PRP I)) (ADVP (RB now)) (VP (VBP live) (PP (IN in) (NP (DT the) (NN flesh))))))) (S (NP (PRP I)) (VP (VBP live) (PP (IN by) (NP (NP (NN faith)) (PP (IN in) (NP (NP (NP (DT the) (NNP Son)) (PP (IN of) (NP (NNP God)))) (, ,) (SBAR (WHNP (WP who)) (S (VP (VP (VBD loved) (NP (PRP me))) (CC and) (VP (VBD gave) (NP (PRP himself)) (PP (IN for) (NP (PRP me)))))))))))))) (. .)) ) 7 | (ROOT (SBARQ (WHNP (WHADJP (WRB how) (JJ much)) (JJR more)) (SQ (MD will) (NP (NP (DT the) (NN blood)) (PP (IN of) (NP (NP (NNP Christ)) (, ,) (SBAR (WHNP (WP who)) (S (PP (IN through) (NP (DT the) (JJ eternal) (NNP Spirit))) (VP (VBD offered) (NP (PRP himself)) (PP (IN without) (NP (NP (NN blemish)) (PP (TO to) (NP (NNP God)))))))) (, ,)))) (VP (VB purify) (NP (PRP$ our) (NN conscience)) (PP (IN from) (NP (JJ dead) (NNS works))) (S (VP (TO to) (VP (VB serve) (NP (DT the) (NN living) (NNP God))))))) (. .)) ) 8 | (ROOT (S (SBAR (RB So) (S (NP (DT these)) (VP (VP (VBD came) (PP (TO to) (NP (NP (NNP Philip)) (, ,) (SBAR (WHNP (WP who)) (S (VP (VBD was) (PP (IN from) (NP (NNP Bethsaida))) (PP (IN in) (NP (NNP Galilee)))))) (, ,)))) (CC and) (VP (VBD asked) (NP (PRP him)))))) (PRN (, ,) (`` ``) (NNP Sir) (, ,)) (NP (PRP we)) (VP (VBP wish) (S (VP (TO to) (VP (VB see) (NP (NNP Jesus)))))) (. .) ('' '')) ) 9 | (ROOT (S (S (SBAR (WHADVP (WRB When)) (S (NP (PRP he)) (VP (VBD was) (VP (VBN reviled))))) (, ,) (NP (PRP he)) (VP (VBD did) (RB not) (VP (VB revile) (PP (IN in) (NP (NN return)))))) (: ;) (S (SBAR (WHADVP (WRB when)) (S (NP (PRP he)) (VP (VBD suffered)))) (, ,) (NP (PRP he)) (VP (VP (VBD did) (RB not) (VP (VB threaten))) (, ,) (CC but) (VP (VBD continued) (S (VP (VBG entrusting) (NP (PRP himself)) (PP (TO to) (NP (PRP him))) (SBAR (WHNP (WP who)) (S (NP (NNS judges)) (VP (VBP justly))))))))) (. .)) ) 10 | (ROOT (S (CC And) (ADVP (RB suddenly)) (NP (EX there)) (VP (VBD was) (PP (IN with) (NP (DT the) (NN angel))) (NP (NP (DT a) (NN multitude)) (PP (IN of) (NP (NP (DT the) (JJ heavenly) (NN host)) (VP (VP (VBG praising) (NP (NNP God))) (CC and) (VP (VBG saying))))))) (, ,)) ) 11 | -------------------------------------------------------------------------------- /data/orm_data/set0-trainunfilt-head10.tok.piece.eng: -------------------------------------------------------------------------------- 1 | ▁i ▁like ▁my ▁st eak ▁me dium ▁. 2 | ▁`` ▁Wo e ▁to ▁you ▁, ▁Ch or azin ▁! ▁Wo e ▁to ▁you ▁, ▁Beth said a ▁! ▁For ▁ if ▁the ▁might y ▁work s ▁ done ▁in ▁you ▁ had ▁be en ▁ done ▁in ▁Tyr e ▁ and ▁Si don ▁, ▁the y ▁w ould ▁ have ▁repent ed ▁long ▁ago ▁, ▁sit ting ▁in ▁ sack cloth ▁ and ▁ash es ▁. 3 | ▁`` ▁Wo e ▁to ▁you ▁, ▁Ch or azin ▁! ▁Wo e ▁to ▁you ▁, ▁Beth said a ▁! ▁For ▁ if ▁the ▁might y ▁work s ▁ done ▁in ▁you ▁ had ▁be en ▁ done ▁in ▁Tyr e ▁ and ▁Si don ▁, ▁the y ▁w ould ▁ have ▁repent ed ▁long ▁ago ▁in ▁ sack cloth ▁ and ▁ash es ▁. 4 | ▁I m mediate ly ▁he ▁mad e ▁ his ▁disciples ▁get ▁in to ▁the ▁boat ▁ and ▁go ▁before ▁him ▁to ▁the ▁other ▁side ▁, ▁to ▁Beth said a ▁, ▁w hile ▁he ▁dis miss ed ▁the ▁crowd ▁. 5 | ▁Le t ▁all ▁bitter ness ▁ and ▁w rath ▁ and ▁ anger ▁ and ▁cla mor ▁ and ▁slander ▁be ▁put ▁a way ▁fro m ▁you ▁, ▁along ▁with ▁all ▁malic e ▁. 6 | ▁I ▁ have ▁be en ▁crucif i ed ▁with ▁Christ ▁. ▁It ▁is ▁no ▁long er ▁I ▁who ▁live ▁, ▁but ▁Christ ▁who ▁lives ▁in ▁me ▁. ▁And ▁the ▁life ▁I ▁now ▁live ▁in ▁the ▁flesh ▁I ▁live ▁by ▁faith ▁in ▁the ▁So n ▁of ▁God ▁, ▁who ▁love d ▁me ▁ and ▁g ave ▁him self ▁for ▁me ▁. 7 | ▁how ▁m uch ▁mor e ▁will ▁the ▁blood ▁of ▁Christ ▁, ▁who ▁through ▁the ▁ eter nal ▁S pirit ▁offer ed ▁him self ▁with out ▁blemish ▁to ▁God ▁, ▁purif y ▁our ▁conscience ▁fro m ▁dead ▁work s ▁to ▁serve ▁the ▁liv ing ▁God ▁. 8 | ▁So ▁the se ▁came ▁to ▁Philip ▁, ▁who ▁was ▁fro m ▁Beth said a ▁in ▁Galile e ▁, ▁ and ▁ask ed ▁him ▁, ▁`` ▁Sir ▁, ▁we ▁wish ▁to ▁see ▁Jes us ▁. ▁'' 9 | ▁When ▁he ▁was ▁revile d ▁, ▁he ▁di d ▁not ▁revile ▁in ▁return ▁; ▁when ▁he ▁suffer ed ▁, ▁he ▁di d ▁not ▁threaten ▁, ▁but ▁continue d ▁entrust ing ▁him self ▁to ▁him ▁who ▁judge s ▁just ly ▁. 10 | ▁And ▁su dden ly ▁there ▁was ▁with ▁the ▁angel ▁a ▁multitude ▁of ▁the ▁heaven ly ▁host ▁prais ing ▁God ▁ and ▁saying ▁, 11 | -------------------------------------------------------------------------------- /data/orm_data/set0-trainunfilt-head10.tok.piece.orm: -------------------------------------------------------------------------------- 1 | ▁i ▁like ▁my ▁st ▁eak ▁me ▁dium . 2 | ▁Yesuus ▁itti ▁fur ee ▁, ▁`` ▁S iif ▁magan ▁yaa ▁ K o raa z iin ▁, ▁siif ▁magan ▁yaa ▁Beetesayida a ▁, ▁hojiin ▁humna ▁Rabbii ▁, ▁tan ▁isin ▁keeysatti ▁godhamt e ▁, ▁odoo ▁ X i iroosii ▁fi ▁Sidoon aa ▁keeysatti ▁waan ▁godhamt e ▁taatee ▁, ▁umma nni ▁isaan ▁keeysa ▁jiraatu ▁si laa ▁wayta ▁tana ▁, ▁uffata ▁gadd aa ▁uffatee ▁, ▁daa raa ▁keeysaa s ▁taa ' ee ▁, ▁toowbachuu ▁isaa ▁ni ▁mullisa ▁ture ▁. 3 | ▁`` ▁S iif ▁magan ▁yaa ▁ K o raa z iin ▁, ▁siif ▁magan ▁yaa ▁Beetesayida a ▁! ▁Hojii n ▁quduraa ▁Rabbii ▁tan ▁isin ▁keeysatti ▁godhamt e ▁, ▁odoo ▁ X i iroosii ▁fi ▁Sidoon ▁keeysatti ▁waan ▁godhamt e ▁taatee ▁, ▁si laa ▁qa a ta ▁umma nni ▁isaan ▁keeysa ▁jiraatu ▁, ▁uffata ▁gadd aatii ▁fi ▁daa raa ▁dhaan ▁toowbachuu ▁isaanii ▁mullisa nii ▁turan ▁. 4 | ▁Hogguusuma ▁Yesuus ▁hanga ▁ifiif ▁ummata ▁san ▁gadi ▁dhiisuu ▁, ▁baratt oonni ▁isaa ▁doonii ▁seenanii ▁, ▁daf anii ▁isa ▁dura ▁Beetesayida tti ▁gama ▁akka ▁ceeh an ▁godhe ▁. 5 | ▁ H a dhaa ▁hundumaa ▁, ▁kanneen ▁akka ▁lolaa ▁, ▁aara a ▁, ▁iyyan s a ▁, ▁arrab s oo ▁fi ▁ammaa s ▁hamm een y i ▁hundin uu ▁isin ▁irraa ▁haa ▁fagaat anii ▁! 6 | ▁As ii ▁achi ▁anaa ▁miti ▁kan ▁jiraatu ▁, ▁Kiristoosii ▁tu ▁na ▁keeysa ▁jiraata ▁malee ▁; ▁jiruu n ▁amma ▁jasadaa n ▁jiraadh u ▁tun ▁, ▁tan iin ▁Ilma ▁Rabbitti ▁amanuu ▁dhaan ▁jiraadh uu ▁dha ▁; ▁inni ▁ niis ▁kan ▁na ▁jaalatee ▁if ▁dabarsee ▁anaaf ▁jecha ▁kannee ▁dha ▁. 7 | ▁dhiiy ni ▁Kiristoos ▁, ▁Ruuhaa ▁bara ▁baraa ▁tiin ▁aarsaa ▁illaa ▁hin ▁qabne ▁godhee ▁ifuma ▁isaa ▁kan ▁Rabbii f ▁dhiheeyse ▁sun ▁, ▁akka ▁Rabbii ▁Jiraat aa ▁kaadam uu ▁dandeeny uuf ▁, ▁gocha ▁du ' atti ▁nama ▁geeysu ▁irraa ▁, ▁xalalii ▁teenya ▁daran ▁cal chi see ▁xaliil eeysu u ▁ni ▁dandaya ▁. 8 | ▁Isaan ▁kun ▁Filiphoos ▁, ▁nama ▁Beetesayida a ▁kan ▁kutaa ▁Galiilaa ▁sana ▁bira ▁dhufanii ▁ti ▁, ▁`` ▁Yesuus ▁arkuu ▁dha ▁ f een a ▁, ▁Gooftaa ▁'' ▁jedhaniin ▁. 9 | ▁Inni ▁wayta ▁arrab f ame ▁, ▁deebisee ▁hin ▁arrab s in e ▁; ▁wayta ▁dararam ee s ▁, ▁Rabbi ▁kan ▁firdii ▁qaceelaa ▁kannu tti ▁kanna te ▁malee ▁, ▁warra ▁isa ▁darar aniis ▁hin ▁ s os s o daa chi f ne ▁. 10 | ▁Hogguusuma ▁, ▁dingat uma ▁kun oo ▁! ▁ J u n d iin ▁samii ▁dhaa ▁hedduun ▁maleeykich aan ▁wajjiin ▁mullata nii ▁ti ▁; 11 | -------------------------------------------------------------------------------- /data/orm_data/set0-trainunfilt-head100.tok.piece.eng: -------------------------------------------------------------------------------- 1 | ▁i ▁like ▁my ▁st eak ▁me dium ▁. 2 | ▁`` ▁Wo e ▁to ▁you ▁, ▁Ch or azin ▁! ▁Wo e ▁to ▁you ▁, ▁Beth said a ▁! ▁For ▁ if ▁the ▁might y ▁work s ▁ done ▁in ▁you ▁ had ▁be en ▁ done ▁in ▁Tyr e ▁ and ▁Si don ▁, ▁the y ▁w ould ▁ have ▁repent ed ▁long ▁ago ▁, ▁sit ting ▁in ▁ sack cloth ▁ and ▁ash es ▁. 3 | ▁`` ▁Wo e ▁to ▁you ▁, ▁Ch or azin ▁! ▁Wo e ▁to ▁you ▁, ▁Beth said a ▁! ▁For ▁ if ▁the ▁might y ▁work s ▁ done ▁in ▁you ▁ had ▁be en ▁ done ▁in ▁Tyr e ▁ and ▁Si don ▁, ▁the y ▁w ould ▁ have ▁repent ed ▁long ▁ago ▁in ▁ sack cloth ▁ and ▁ash es ▁. 4 | ▁I m mediate ly ▁he ▁mad e ▁ his ▁disciples ▁get ▁in to ▁the ▁boat ▁ and ▁go ▁before ▁him ▁to ▁the ▁other ▁side ▁, ▁to ▁Beth said a ▁, ▁w hile ▁he ▁dis miss ed ▁the ▁crowd ▁. 5 | ▁Le t ▁all ▁bitter ness ▁ and ▁w rath ▁ and ▁ anger ▁ and ▁cla mor ▁ and ▁slander ▁be ▁put ▁a way ▁fro m ▁you ▁, ▁along ▁with ▁all ▁malic e ▁. 6 | ▁I ▁ have ▁be en ▁crucif i ed ▁with ▁Christ ▁. ▁It ▁is ▁no ▁long er ▁I ▁who ▁live ▁, ▁but ▁Christ ▁who ▁lives ▁in ▁me ▁. ▁And ▁the ▁life ▁I ▁now ▁live ▁in ▁the ▁flesh ▁I ▁live ▁by ▁faith ▁in ▁the ▁So n ▁of ▁God ▁, ▁who ▁love d ▁me ▁ and ▁g ave ▁him self ▁for ▁me ▁. 7 | ▁how ▁m uch ▁mor e ▁will ▁the ▁blood ▁of ▁Christ ▁, ▁who ▁through ▁the ▁ eter nal ▁S pirit ▁offer ed ▁him self ▁with out ▁blemish ▁to ▁God ▁, ▁purif y ▁our ▁conscience ▁fro m ▁dead ▁work s ▁to ▁serve ▁the ▁liv ing ▁God ▁. 8 | ▁So ▁the se ▁came ▁to ▁Philip ▁, ▁who ▁was ▁fro m ▁Beth said a ▁in ▁Galile e ▁, ▁ and ▁ask ed ▁him ▁, ▁`` ▁Sir ▁, ▁we ▁wish ▁to ▁see ▁Jes us ▁. ▁'' 9 | ▁When ▁he ▁was ▁revile d ▁, ▁he ▁di d ▁not ▁revile ▁in ▁return ▁; ▁when ▁he ▁suffer ed ▁, ▁he ▁di d ▁not ▁threaten ▁, ▁but ▁continue d ▁entrust ing ▁him self ▁to ▁him ▁who ▁judge s ▁just ly ▁. 10 | ▁And ▁su dden ly ▁there ▁was ▁with ▁the ▁angel ▁a ▁multitude ▁of ▁the ▁heaven ly ▁host ▁prais ing ▁God ▁ and ▁saying ▁, 11 | ▁And ▁the ▁Wor d ▁be came ▁flesh ▁ and ▁dwel t ▁am ong ▁us ▁, ▁ and ▁we ▁ have ▁see n ▁ his ▁glor y ▁, ▁glor y ▁as ▁of ▁the ▁on ly ▁So n ▁fro m ▁the ▁Father ▁, ▁full ▁of ▁ grace ▁ and ▁truth ▁. 12 | ▁And ▁ if ▁I ▁go ▁ and ▁prepare ▁a ▁place ▁for ▁you ▁, ▁I ▁will ▁come ▁again ▁ and ▁will ▁take ▁you ▁to ▁my self ▁, ▁tha t ▁where ▁I ▁am ▁you ▁ma y ▁be ▁a lso ▁. 13 | ▁He ▁ said ▁to ▁him ▁the ▁thir d ▁time ▁, ▁`` ▁S imon ▁, ▁son ▁of ▁John ▁, ▁do ▁you ▁love ▁me ▁? ▁'' ▁P eter ▁was ▁grieve d ▁be cause ▁he ▁ said ▁to ▁him ▁the ▁thir d ▁time ▁, ▁`` ▁Do ▁you ▁love ▁me ▁? ▁'' ▁ and ▁he ▁ said ▁to ▁him ▁, ▁`` ▁Lord ▁, ▁you ▁know ▁every thing ▁; ▁you ▁know ▁tha t ▁I ▁love ▁you ▁. ▁'' ▁Jes us ▁ said ▁to ▁him ▁, ▁`` ▁Fe ed ▁my ▁sheep ▁. 14 | ▁In ▁the ▁begin ning ▁was ▁the ▁Wor d ▁, ▁ and ▁the ▁Wor d ▁was ▁with ▁God ▁, ▁ and ▁the ▁Wor d ▁was ▁God ▁. 15 | ▁For ▁un to ▁you ▁is ▁born ▁this ▁day ▁in ▁the ▁cit y ▁of ▁Da vid ▁a ▁S avi or ▁, ▁who ▁is ▁Christ ▁the ▁Lord ▁. 16 | ▁The ▁n ext ▁day ▁Jes us ▁decide d ▁to ▁go ▁to ▁Galile e ▁. ▁He ▁found ▁Philip ▁ and ▁ said ▁to ▁him ▁, ▁`` ▁F ollow ▁me ▁. ▁'' 17 | ▁On ▁their ▁return ▁the ▁apostles ▁to ld ▁him ▁all ▁tha t ▁the y ▁ had ▁ done ▁. ▁And ▁he ▁to ok ▁them ▁ and ▁with d rew ▁a part ▁to ▁a ▁town ▁call ed ▁Beth said a ▁. 18 | ▁And ▁ if ▁any one ▁will ▁not ▁receive ▁you ▁or ▁listen ▁to ▁your ▁word s ▁, ▁shake ▁off ▁the ▁du st ▁fro m ▁your ▁feet ▁when ▁you ▁leave ▁tha t ▁house ▁or ▁town ▁. 19 | ▁but ▁in ▁every ▁nation ▁any one ▁who ▁fear s ▁him ▁ and ▁doe s ▁what ▁is ▁right ▁is ▁accept able ▁to ▁him ▁. 20 | ▁No w ▁Philip ▁was ▁fro m ▁Beth said a ▁, ▁the ▁cit y ▁of ▁And rew ▁ and ▁P eter ▁. 21 | ▁let ▁us ▁draw ▁near ▁with ▁a ▁tru e ▁heart ▁in ▁full ▁assur ance ▁of ▁faith ▁, ▁with ▁our ▁heart s ▁sprinkl ed ▁clean ▁fro m ▁an ▁evil ▁conscience ▁ and ▁our ▁bod ies ▁wash ed ▁with ▁pure ▁water ▁. 22 | ▁No w ▁am ong ▁th ose ▁who ▁ went ▁up ▁to ▁worship ▁at ▁the ▁feast ▁ were ▁some ▁Greek s ▁. 23 | ▁I f ▁we ▁confess ▁our ▁sins ▁, ▁he ▁is ▁faithful ▁ and ▁just ▁to ▁forgive ▁us ▁our ▁sins ▁ and ▁to ▁cleanse ▁us ▁fro m ▁all ▁unrighteous ness ▁. 24 | ▁As ▁for ▁the ▁rich ▁in ▁this ▁present ▁age ▁, ▁charge ▁them ▁not ▁to ▁be ▁ha ught y ▁, ▁nor ▁to ▁set ▁their ▁hope s ▁on ▁the ▁uncertain t y ▁of ▁rich es ▁, ▁but ▁on ▁God ▁, ▁who ▁rich ly ▁provide s ▁us ▁with ▁every thing ▁to ▁enjoy ▁. 25 | ▁who ▁g ave ▁him self ▁for ▁us ▁to ▁redeem ▁us ▁fro m ▁all ▁lawless ness ▁ and ▁to ▁purif y ▁for ▁him self ▁a ▁people ▁for ▁ his ▁own ▁possession ▁who ▁are ▁zeal ous ▁for ▁good ▁work s ▁. 26 | ▁In ▁him ▁we ▁ have ▁rede m ption ▁through ▁ his ▁blood ▁, ▁the ▁forgive ness ▁of ▁our ▁trespass es ▁, ▁a ccording ▁to ▁the ▁rich es ▁of ▁ his ▁ grace ▁, 27 | ▁For ▁every one ▁who ▁exalt s ▁him self ▁will ▁be ▁humble d ▁, ▁ and ▁he ▁who ▁humble s ▁him self ▁will ▁be ▁exalt ed ▁. ▁'' 28 | ▁And ▁the y ▁came ▁to ▁Beth said a ▁. ▁And ▁some ▁people ▁br ought ▁to ▁him ▁a ▁blind ▁man ▁ and ▁begg ed ▁him ▁to ▁touch ▁him ▁. 29 | ▁He ▁him self ▁ bore ▁our ▁sins ▁in ▁ his ▁bod y ▁on ▁the ▁tree ▁, ▁tha t ▁we ▁might ▁die ▁to ▁sin ▁ and ▁live ▁to ▁righteousness ▁. ▁B y ▁ his ▁wound s ▁you ▁ have ▁be en ▁heal ed ▁. 30 | ▁When ▁the ▁crowd s ▁learn ed ▁it ▁, ▁the y ▁follow ed ▁him ▁, ▁ and ▁he ▁welcome d ▁them ▁ and ▁spoke ▁to ▁them ▁of ▁the ▁kingdom ▁of ▁God ▁ and ▁cur ed ▁th ose ▁who ▁ had ▁need ▁of ▁heal ing ▁. 31 | ▁Bu t ▁when ▁Herod ▁di ed ▁, ▁behold ▁, ▁an ▁angel ▁of ▁the ▁Lord ▁appear ed ▁in ▁a ▁dream ▁to ▁Jose ph ▁in ▁Egypt ▁, 32 | ▁for ▁he ▁who ▁is ▁might y ▁has ▁ done ▁great ▁thing s ▁for ▁me ▁, ▁ and ▁hol y ▁is ▁ his ▁name ▁. 33 | ▁And ▁he ▁ said ▁to ▁all ▁, ▁`` ▁I f ▁any one ▁w ould ▁come ▁after ▁me ▁, ▁let ▁him ▁deny ▁him self ▁ and ▁take ▁up ▁ his ▁cross ▁ dai ly ▁ and ▁follow ▁me ▁. 34 | ▁For ▁what ▁ credi t ▁is ▁it ▁ if ▁, ▁when ▁you ▁sin ▁ and ▁are ▁beat en ▁for ▁it ▁, ▁you ▁endure ▁? ▁Bu t ▁ if ▁when ▁you ▁do ▁good ▁ and ▁suffer ▁for ▁it ▁you ▁endure ▁, ▁this ▁is ▁a ▁gracious ▁thing ▁in ▁the ▁sight ▁of ▁God ▁. 35 | ▁`` ▁G lor y ▁to ▁God ▁in ▁the ▁high est ▁, ▁ and ▁on ▁earth ▁peace ▁am ong ▁th ose ▁with ▁whom ▁he ▁is ▁please d ▁! ▁'' 36 | ▁Bu t ▁ if ▁you ▁ bit e ▁ and ▁devour ▁one ▁an other ▁, ▁watch ▁out ▁tha t ▁you ▁are ▁not ▁consume d ▁by ▁one ▁an other ▁. 37 | ▁And ▁I ▁ have ▁other ▁sheep ▁tha t ▁are ▁not ▁of ▁this ▁ fold ▁. ▁I ▁must ▁bring ▁them ▁a lso ▁, ▁ and ▁the y ▁will ▁listen ▁to ▁my ▁voice ▁. ▁So ▁there ▁will ▁be ▁one ▁f lock ▁, ▁one ▁shepherd ▁. 38 | ▁And ▁call ing ▁the ▁crowd ▁to ▁him ▁with ▁ his ▁disciples ▁, ▁he ▁ said ▁to ▁them ▁, ▁`` ▁I f ▁any one ▁w ould ▁come ▁after ▁me ▁, ▁let ▁him ▁deny ▁him self ▁ and ▁take ▁up ▁ his ▁cross ▁ and ▁follow ▁me ▁. 39 | ▁He ▁ said ▁to ▁him ▁a ▁second ▁time ▁, ▁`` ▁S imon ▁, ▁son ▁of ▁John ▁, ▁do ▁you ▁love ▁me ▁? ▁'' ▁He ▁ said ▁to ▁him ▁, ▁`` ▁Yes ▁, ▁Lord ▁; ▁you ▁know ▁tha t ▁I ▁love ▁you ▁. ▁'' ▁He ▁ said ▁to ▁him ▁, ▁`` ▁Te nd ▁my ▁sheep ▁. ▁'' 40 | ▁And ▁he ▁to ok ▁a ▁ cup ▁, ▁ and ▁when ▁he ▁ had ▁ given ▁thanks ▁he ▁g ave ▁it ▁to ▁them ▁, ▁ and ▁the y ▁all ▁ d rank ▁of ▁it ▁. 41 | ▁I m mediate ly ▁he ▁mad e ▁the ▁disciples ▁get ▁in to ▁the ▁boat ▁ and ▁go ▁before ▁him ▁to ▁the ▁other ▁side ▁, ▁w hile ▁he ▁dis miss ed ▁the ▁crowd s ▁. 42 | ▁There fore ▁, ▁my ▁brother s ▁, ▁whom ▁I ▁love ▁ and ▁long ▁for ▁, ▁my ▁joy ▁ and ▁crown ▁, ▁stand ▁firm ▁th us ▁in ▁the ▁Lord ▁, ▁my ▁be love d ▁. 43 | ▁In ▁this ▁is ▁love ▁, ▁not ▁tha t ▁we ▁ have ▁love d ▁God ▁but ▁tha t ▁he ▁love d ▁us ▁ and ▁sent ▁ his ▁So n ▁to ▁be ▁the ▁prop iti ation ▁for ▁our ▁sins ▁. 44 | ▁And ▁the ▁shepherd s ▁return ed ▁, ▁glorify ing ▁ and ▁prais ing ▁God ▁for ▁all ▁the y ▁ had ▁hear d ▁ and ▁see n ▁, ▁as ▁it ▁ had ▁be en ▁to ld ▁them ▁. 45 | ▁And ▁you ▁, ▁Cap erna um ▁, ▁will ▁you ▁be ▁exalt ed ▁to ▁heaven ▁? ▁You ▁will ▁be ▁br ought ▁down ▁to ▁Ha d es ▁. ▁For ▁ if ▁the ▁might y ▁work s ▁ done ▁in ▁you ▁ had ▁be en ▁ done ▁in ▁So dom ▁, ▁it ▁w ould ▁ have ▁remain ed ▁unti l ▁this ▁day ▁. 46 | ▁The n ▁I ▁saw ▁a ▁new ▁heaven ▁ and ▁a ▁new ▁earth ▁, ▁for ▁the ▁first ▁heaven ▁ and ▁the ▁first ▁earth ▁ had ▁pass ed ▁a way ▁, ▁ and ▁the ▁sea ▁was ▁no ▁mor e ▁. 47 | ▁And ▁he ▁to ok ▁a ▁ cup ▁, ▁ and ▁when ▁he ▁ had ▁ given ▁thanks ▁he ▁ said ▁, ▁`` ▁T ake ▁this ▁, ▁ and ▁divide ▁it ▁am ong ▁yoursel ves ▁. 48 | ▁tha t ▁you ▁ma y ▁be ▁blameless ▁ and ▁in noc ent ▁, ▁child ren ▁of ▁God ▁with out ▁blemish ▁in ▁the ▁midst ▁of ▁a ▁cr ook ed ▁ and ▁twist ed ▁generation ▁, ▁am ong ▁whom ▁you ▁shine ▁as ▁light s ▁in ▁the ▁world ▁, 49 | ▁There ▁is ▁no ▁fear ▁in ▁love ▁, ▁but ▁perfect ▁love ▁cast s ▁out ▁fear ▁. ▁For ▁fear ▁has ▁to ▁do ▁with ▁punish ment ▁, ▁ and ▁who ever ▁fear s ▁has ▁not ▁be en ▁perfect ed ▁in ▁love ▁. 50 | ▁I ▁am ▁the ▁good ▁shepherd ▁. ▁The ▁good ▁shepherd ▁lay s ▁down ▁ his ▁life ▁for ▁the ▁sheep ▁. 51 | ▁Jes us ▁ said ▁to ▁them ▁, ▁`` ▁Re nder ▁to ▁Caesar ▁the ▁thing s ▁tha t ▁are ▁Caesar ▁' s ▁, ▁ and ▁to ▁God ▁the ▁thing s ▁tha t ▁are ▁God ▁' s ▁. ▁'' ▁And ▁the y ▁marvel ed ▁at ▁him ▁. 52 | ▁wait ing ▁for ▁our ▁bless ed ▁hope ▁, ▁the ▁appear ing ▁of ▁the ▁glor y ▁of ▁our ▁great ▁God ▁ and ▁S avi or ▁Jes us ▁Christ ▁, 53 | ▁He ▁ said ▁to ▁them ▁, ▁`` ▁The n ▁render ▁to ▁Caesar ▁the ▁thing s ▁tha t ▁are ▁Caesar ▁' s ▁, ▁ and ▁to ▁God ▁the ▁thing s ▁tha t ▁are ▁God ▁' s ▁. ▁'' 54 | ▁I ▁tell ▁you ▁, ▁this ▁man ▁ went ▁down ▁to ▁ his ▁house ▁justifie d ▁, ▁ra ther ▁than ▁the ▁other ▁. ▁For ▁every one ▁who ▁exalt s ▁him self ▁will ▁be ▁humble d ▁, ▁but ▁the ▁one ▁who ▁humble s ▁him self ▁will ▁be ▁exalt ed ▁. ▁'' 55 | ▁You ▁di d ▁not ▁choose ▁me ▁, ▁but ▁I ▁chose ▁you ▁ and ▁appoint ed ▁you ▁tha t ▁you ▁should ▁go ▁ and ▁bear ▁fruit ▁ and ▁tha t ▁your ▁fruit ▁should ▁abide ▁, ▁so ▁tha t ▁what ever ▁you ▁ask ▁the ▁Father ▁in ▁my ▁name ▁, ▁he ▁ma y ▁give ▁it ▁to ▁you ▁. 56 | ▁A gain ▁Jes us ▁spoke ▁to ▁them ▁, ▁saying ▁, ▁`` ▁I ▁am ▁the ▁light ▁of ▁the ▁world ▁. ▁Who ever ▁follow s ▁me ▁will ▁not ▁walk ▁in ▁dark ness ▁, ▁but ▁will ▁ have ▁the ▁light ▁of ▁life ▁. ▁'' 57 | ▁And ▁we ▁are ▁witnesse s ▁of ▁all ▁tha t ▁he ▁di d ▁both ▁in ▁the ▁country ▁of ▁the ▁Jew s ▁ and ▁in ▁Jer usa lem ▁. ▁The y ▁put ▁him ▁to ▁death ▁by ▁hang ing ▁him ▁on ▁a ▁tree ▁, 58 | ▁The n ▁Jes us ▁to ld ▁ his ▁disciples ▁, ▁`` ▁I f ▁any one ▁w ould ▁come ▁after ▁me ▁, ▁let ▁him ▁deny ▁him self ▁ and ▁take ▁up ▁ his ▁cross ▁ and ▁follow ▁me ▁. 59 | ▁For ▁the ▁ grace ▁of ▁God ▁has ▁appear ed ▁, ▁bring ing ▁salvation ▁for ▁all ▁people ▁, 60 | ▁The y ▁ said ▁, ▁`` ▁Caesar ▁' s ▁. ▁'' ▁The n ▁he ▁ said ▁to ▁them ▁, ▁`` ▁There fore ▁render ▁to ▁Caesar ▁the ▁thing s ▁tha t ▁are ▁Caesar ▁' s ▁, ▁ and ▁to ▁God ▁the ▁thing s ▁tha t ▁are ▁God ▁' s ▁. ▁'' 61 | ▁For ▁there ▁is ▁one ▁God ▁, ▁ and ▁there ▁is ▁one ▁mediat or ▁be twe en ▁God ▁ and ▁men ▁, ▁the ▁man ▁Christ ▁Jes us ▁, 62 | ▁in ▁whom ▁are ▁hi dden ▁all ▁the ▁treasure s ▁of ▁wis dom ▁ and ▁ knowledge ▁. 63 | ▁For ▁ if ▁we ▁live ▁, ▁we ▁live ▁to ▁the ▁Lord ▁, ▁ and ▁ if ▁we ▁die ▁, ▁we ▁die ▁to ▁the ▁Lord ▁. ▁So ▁the n ▁, ▁whe ther ▁we ▁live ▁or ▁whe ther ▁we ▁die ▁, ▁we ▁are ▁the ▁Lord ▁' s ▁. 64 | ▁For ▁where ▁your ▁treasure ▁is ▁, ▁there ▁your ▁heart ▁will ▁be ▁a lso ▁. 65 | ▁ and ▁endur ance ▁produce s ▁char ac ter ▁, ▁ and ▁char ac ter ▁produce s ▁hope ▁, 66 | ▁Go ▁ and ▁learn ▁what ▁this ▁mean s ▁, ▁` ▁I ▁desire ▁merc y ▁, ▁ and ▁not ▁sacrifice ▁. ▁' For ▁I ▁came ▁not ▁to ▁call ▁the ▁righteous ▁, ▁but ▁sinner s ▁. ▁'' 67 | ▁And ▁in ▁the ▁ s ame ▁region ▁there ▁ were ▁shepherd s ▁out ▁in ▁the ▁field ▁, ▁keep ing ▁watch ▁over ▁their ▁f lock ▁by ▁night ▁. 68 | ▁Be ▁ang ry ▁ and ▁do ▁not ▁sin ▁; ▁do ▁not ▁let ▁the ▁sun ▁go ▁down ▁on ▁your ▁ anger ▁, 69 | ▁In ▁the ▁day s ▁of ▁Herod ▁, ▁king ▁of ▁Judea ▁, ▁there ▁was ▁a ▁priest ▁name d ▁ Z e chari ah ▁, ▁of ▁the ▁division ▁of ▁Ab ijah ▁. ▁And ▁he ▁ had ▁a ▁w ife ▁fro m ▁the ▁daughter s ▁of ▁A ar on ▁, ▁ and ▁her ▁name ▁was ▁Eli za be th ▁. 70 | ▁No w ▁ his ▁ parent s ▁ went ▁to ▁Jer usa lem ▁every ▁year ▁at ▁the ▁Fea st ▁of ▁the ▁Pass over ▁. 71 | ▁And ▁he ▁ said ▁to ▁them ▁, ▁`` ▁F ollow ▁me ▁, ▁ and ▁I ▁will ▁make ▁you ▁fisher s ▁of ▁men ▁. ▁'' 72 | ▁For ▁al though ▁the y ▁kne w ▁God ▁, ▁the y ▁di d ▁not ▁honor ▁him ▁as ▁God ▁or ▁give ▁thanks ▁to ▁him ▁, ▁but ▁the y ▁be came ▁futil e ▁in ▁their ▁think ing ▁, ▁ and ▁their ▁foolish ▁heart s ▁ were ▁dark en ed ▁. 73 | ▁There fore ▁we ▁must ▁pay ▁m uch ▁close r ▁at tention ▁to ▁what ▁we ▁ have ▁hear d ▁, ▁les t ▁we ▁dri ft ▁a way ▁fro m ▁it ▁. 74 | ▁And ▁he ▁came ▁to ▁Nazare th ▁, ▁where ▁he ▁ had ▁be en ▁br ought ▁up ▁. ▁And ▁as ▁was ▁ his ▁custom ▁, ▁he ▁ went ▁to ▁the ▁synagogue ▁on ▁the ▁S abbath ▁day ▁, ▁ and ▁he ▁stoo d ▁up ▁to ▁read ▁. 75 | ▁And ▁he ▁began ▁to ▁speak ▁to ▁them ▁in ▁parable s ▁. ▁`` ▁A ▁man ▁plant ed ▁a ▁vine yard ▁ and ▁put ▁a ▁f ence ▁a round ▁it ▁ and ▁ d ug ▁a ▁pit ▁for ▁the ▁wine press ▁ and ▁buil t ▁a ▁to wer ▁, ▁ and ▁leas ed ▁it ▁to ▁t enant s ▁ and ▁ went ▁in to ▁an other ▁country ▁. 76 | ▁Brother s ▁, ▁I ▁do ▁not ▁consider ▁tha t ▁I ▁ have ▁mad e ▁it ▁my ▁own ▁. ▁Bu t ▁one ▁thing ▁I ▁do ▁: ▁forget ting ▁what ▁lie s ▁be hind ▁ and ▁stra in ing ▁forward ▁to ▁what ▁lie s ▁a head ▁, 77 | ▁When ▁the y ▁ had ▁finish ed ▁break fast ▁, ▁Jes us ▁ said ▁to ▁S imon ▁P eter ▁, ▁`` ▁S imon ▁, ▁son ▁of ▁John ▁, ▁do ▁you ▁love ▁me ▁mor e ▁than ▁the se ▁? ▁'' ▁He ▁ said ▁to ▁him ▁, ▁`` ▁Yes ▁, ▁Lord ▁; ▁you ▁know ▁tha t ▁I ▁love ▁you ▁. ▁'' ▁He ▁ said ▁to ▁him ▁, ▁`` ▁Fe ed ▁my ▁lamb s ▁. ▁'' 78 | ▁`` ▁Bless ed ▁are ▁the ▁peace make rs ▁, ▁for ▁the y ▁ s hall ▁be ▁call ed ▁son s ▁of ▁God ▁. 79 | ▁And ▁he ▁ said ▁to ▁them ▁, ▁`` ▁T ake ▁care ▁, ▁ and ▁be ▁on ▁your ▁guard ▁again st ▁all ▁covetous ness ▁, ▁for ▁one ▁' s ▁life ▁doe s ▁not ▁consi st ▁in ▁the ▁abundan ce ▁of ▁ his ▁possession s ▁. ▁'' 80 | ▁c inn amon ▁, ▁spice ▁, ▁in cense ▁, ▁m yrrh ▁, ▁fr ankin cense ▁, ▁wine ▁, ▁ oil ▁, ▁ fine ▁fl our ▁, ▁whe at ▁, ▁ca ttle ▁ and ▁sheep ▁, ▁horse s ▁ and ▁ ch ariot s ▁, ▁ and ▁slave s ▁, ▁tha t ▁is ▁, ▁human ▁soul s ▁. 81 | ▁So ▁P eter ▁open ed ▁ his ▁mouth ▁ and ▁ said ▁: ▁`` ▁Tru ly ▁I ▁understand ▁tha t ▁God ▁show s ▁no ▁partial ity ▁, 82 | ▁Bu t ▁Mar y ▁treasure d ▁up ▁all ▁the se ▁thing s ▁, ▁pond er ing ▁them ▁in ▁her ▁heart ▁. 83 | ▁And ▁go ing ▁in to ▁the ▁house ▁the y ▁saw ▁the ▁child ▁with ▁Mar y ▁ his ▁mother ▁, ▁ and ▁the y ▁fell ▁down ▁ and ▁worship ed ▁him ▁. ▁The n ▁, ▁open ing ▁their ▁treasure s ▁, ▁the y ▁offer ed ▁him ▁gift s ▁, ▁gold ▁ and ▁fr ankin cense ▁ and ▁m yrrh ▁. 84 | ▁On ▁an ▁appoint ed ▁day ▁Herod ▁put ▁on ▁ his ▁ roy al ▁robe s ▁, ▁to ok ▁ his ▁seat ▁up on ▁the ▁throne ▁, ▁ and ▁deliver ed ▁an ▁or ation ▁to ▁them ▁. 85 | ▁My ▁li ttle ▁child ren ▁, ▁I ▁am ▁writing ▁the se ▁thing s ▁to ▁you ▁so ▁tha t ▁you ▁ma y ▁not ▁sin ▁. ▁Bu t ▁ if ▁any one ▁doe s ▁sin ▁, ▁we ▁ have ▁an ▁adv o cate ▁with ▁the ▁Father ▁, ▁Jes us ▁Christ ▁the ▁righteous ▁. 86 | ▁Thi s ▁call s ▁for ▁wis dom ▁: ▁let ▁the ▁one ▁who ▁has ▁understand ing ▁cal culat e ▁the ▁number ▁of ▁the ▁beast ▁, ▁for ▁it ▁is ▁the ▁number ▁of ▁a ▁man ▁, ▁ and ▁ his ▁number ▁is ▁ 666 ▁. 87 | ▁`` ▁Al as ▁, ▁al as ▁, ▁for ▁the ▁great ▁cit y ▁tha t ▁was ▁cloth ed ▁in ▁ fine ▁line n ▁, ▁in ▁purp le ▁ and ▁scar let ▁, ▁adorn ed ▁with ▁gold ▁, ▁with ▁jewel s ▁, ▁ and ▁with ▁pearl s ▁! 88 | ▁It ▁has ▁be en ▁testifie d ▁some where ▁, ▁`` ▁What ▁is ▁man ▁, ▁tha t ▁you ▁are ▁mind ful ▁of ▁him ▁, ▁or ▁the ▁son ▁of ▁man ▁, ▁tha t ▁you ▁care ▁for ▁him ▁? 89 | ▁And ▁the y ▁assembl ed ▁them ▁at ▁the ▁place ▁tha t ▁in ▁Hebrew ▁is ▁call ed ▁Ar m aged don ▁. 90 | ▁The ▁servant ▁girl ▁at ▁the ▁door ▁ said ▁to ▁P eter ▁, ▁`` ▁You ▁a lso ▁are ▁not ▁one ▁of ▁this ▁man ▁' s ▁disciples ▁, ▁are ▁you ▁? ▁'' ▁He ▁ said ▁, ▁`` ▁I ▁am ▁not ▁. ▁'' 91 | ▁Ne ither ▁is ▁new ▁wine ▁put ▁in to ▁old ▁wine s kins ▁. ▁I f ▁it ▁is ▁, ▁the ▁skin s ▁bur st ▁ and ▁the ▁wine ▁is ▁sp ill ed ▁ and ▁the ▁skin s ▁are ▁destroy ed ▁. ▁Bu t ▁new ▁wine ▁is ▁put ▁in to ▁ fresh ▁wine s kins ▁, ▁ and ▁so ▁both ▁are ▁preserve d ▁. ▁'' 92 | ▁Rep ay ▁no ▁one ▁evil ▁for ▁evil ▁, ▁but ▁give ▁thought ▁to ▁do ▁what ▁is ▁honorabl e ▁in ▁the ▁sight ▁of ▁all ▁. 93 | ▁And ▁the y ▁ were ▁sell ing ▁their ▁possession s ▁ and ▁belonging s ▁ and ▁distribut ing ▁the ▁proceed s ▁to ▁all ▁, ▁as ▁any ▁ had ▁need ▁. 94 | ▁ and ▁when ▁he ▁ had ▁found ▁him ▁, ▁he ▁br ought ▁him ▁to ▁Anti och ▁. ▁For ▁a ▁who le ▁year ▁the y ▁me t ▁with ▁the ▁church ▁ and ▁t aught ▁a ▁great ▁man y ▁people ▁. ▁And ▁in ▁Anti och ▁the ▁disciples ▁ were ▁first ▁call ed ▁Christian s ▁. 95 | ▁ and ▁twist ing ▁to ge ther ▁a ▁crown ▁of ▁thorn s ▁, ▁the y ▁put ▁it ▁on ▁ his ▁head ▁ and ▁put ▁a ▁re ed ▁in ▁ his ▁right ▁hand ▁. ▁And ▁knee ling ▁before ▁him ▁, ▁the y ▁mock ed ▁him ▁, ▁saying ▁, ▁`` ▁Ha il ▁, ▁ K ing ▁of ▁the ▁Jew s ▁! ▁'' 96 | ▁W atch ▁ and ▁pray ▁tha t ▁you ▁ma y ▁not ▁enter ▁in to ▁tempt ation ▁. ▁The ▁spirit ▁in deed ▁is ▁will ing ▁, ▁but ▁the ▁flesh ▁is ▁weak ▁. ▁'' 97 | ▁W atch ▁ and ▁pray ▁tha t ▁you ▁ma y ▁not ▁enter ▁in to ▁tempt ation ▁. ▁The ▁spirit ▁in deed ▁is ▁will ing ▁, ▁but ▁the ▁flesh ▁is ▁weak ▁. ▁'' 98 | ▁P eter ▁ said ▁to ▁him ▁, ▁`` ▁Lord ▁, ▁wh y ▁can ▁I ▁not ▁follow ▁you ▁now ▁? ▁I ▁will ▁lay ▁down ▁my ▁life ▁for ▁you ▁. ▁'' 99 | ▁No ▁one ▁take s ▁it ▁fro m ▁me ▁, ▁but ▁I ▁lay ▁it ▁down ▁of ▁my ▁own ▁accord ▁. ▁I ▁ have ▁authorit y ▁to ▁lay ▁it ▁down ▁, ▁ and ▁I ▁ have ▁authorit y ▁to ▁take ▁it ▁up ▁again ▁. ▁Thi s ▁charge ▁I ▁ have ▁receive d ▁fro m ▁my ▁Father ▁. ▁'' 100 | ▁`` ▁Bless ed ▁are ▁the ▁meek ▁, ▁for ▁the y ▁ s hall ▁inherit ▁the ▁earth ▁. 101 | -------------------------------------------------------------------------------- /data/orm_data/set0-trainunfilt-head100.tok.piece.orm: -------------------------------------------------------------------------------- 1 | ▁i ▁like ▁my ▁st ▁eak ▁me ▁dium . 2 | ▁Yesuus ▁itti ▁fur ee ▁, ▁`` ▁S iif ▁magan ▁yaa ▁ K o raa z iin ▁, ▁siif ▁magan ▁yaa ▁Beetesayida a ▁, ▁hojiin ▁humna ▁Rabbii ▁, ▁tan ▁isin ▁keeysatti ▁godhamt e ▁, ▁odoo ▁ X i iroosii ▁fi ▁Sidoon aa ▁keeysatti ▁waan ▁godhamt e ▁taatee ▁, ▁umma nni ▁isaan ▁keeysa ▁jiraatu ▁si laa ▁wayta ▁tana ▁, ▁uffata ▁gadd aa ▁uffatee ▁, ▁daa raa ▁keeysaa s ▁taa ' ee ▁, ▁toowbachuu ▁isaa ▁ni ▁mullisa ▁ture ▁. 3 | ▁`` ▁S iif ▁magan ▁yaa ▁ K o raa z iin ▁, ▁siif ▁magan ▁yaa ▁Beetesayida a ▁! ▁Hojii n ▁quduraa ▁Rabbii ▁tan ▁isin ▁keeysatti ▁godhamt e ▁, ▁odoo ▁ X i iroosii ▁fi ▁Sidoon ▁keeysatti ▁waan ▁godhamt e ▁taatee ▁, ▁si laa ▁qa a ta ▁umma nni ▁isaan ▁keeysa ▁jiraatu ▁, ▁uffata ▁gadd aatii ▁fi ▁daa raa ▁dhaan ▁toowbachuu ▁isaanii ▁mullisa nii ▁turan ▁. 4 | ▁Hogguusuma ▁Yesuus ▁hanga ▁ifiif ▁ummata ▁san ▁gadi ▁dhiisuu ▁, ▁baratt oonni ▁isaa ▁doonii ▁seenanii ▁, ▁daf anii ▁isa ▁dura ▁Beetesayida tti ▁gama ▁akka ▁ceeh an ▁godhe ▁. 5 | ▁ H a dhaa ▁hundumaa ▁, ▁kanneen ▁akka ▁lolaa ▁, ▁aara a ▁, ▁iyyan s a ▁, ▁arrab s oo ▁fi ▁ammaa s ▁hamm een y i ▁hundin uu ▁isin ▁irraa ▁haa ▁fagaat anii ▁! 6 | ▁As ii ▁achi ▁anaa ▁miti ▁kan ▁jiraatu ▁, ▁Kiristoosii ▁tu ▁na ▁keeysa ▁jiraata ▁malee ▁; ▁jiruu n ▁amma ▁jasadaa n ▁jiraadh u ▁tun ▁, ▁tan iin ▁Ilma ▁Rabbitti ▁amanuu ▁dhaan ▁jiraadh uu ▁dha ▁; ▁inni ▁ niis ▁kan ▁na ▁jaalatee ▁if ▁dabarsee ▁anaaf ▁jecha ▁kannee ▁dha ▁. 7 | ▁dhiiy ni ▁Kiristoos ▁, ▁Ruuhaa ▁bara ▁baraa ▁tiin ▁aarsaa ▁illaa ▁hin ▁qabne ▁godhee ▁ifuma ▁isaa ▁kan ▁Rabbii f ▁dhiheeyse ▁sun ▁, ▁akka ▁Rabbii ▁Jiraat aa ▁kaadam uu ▁dandeeny uuf ▁, ▁gocha ▁du ' atti ▁nama ▁geeysu ▁irraa ▁, ▁xalalii ▁teenya ▁daran ▁cal chi see ▁xaliil eeysu u ▁ni ▁dandaya ▁. 8 | ▁Isaan ▁kun ▁Filiphoos ▁, ▁nama ▁Beetesayida a ▁kan ▁kutaa ▁Galiilaa ▁sana ▁bira ▁dhufanii ▁ti ▁, ▁`` ▁Yesuus ▁arkuu ▁dha ▁ f een a ▁, ▁Gooftaa ▁'' ▁jedhaniin ▁. 9 | ▁Inni ▁wayta ▁arrab f ame ▁, ▁deebisee ▁hin ▁arrab s in e ▁; ▁wayta ▁dararam ee s ▁, ▁Rabbi ▁kan ▁firdii ▁qaceelaa ▁kannu tti ▁kanna te ▁malee ▁, ▁warra ▁isa ▁darar aniis ▁hin ▁ s os s o daa chi f ne ▁. 10 | ▁Hogguusuma ▁, ▁dingat uma ▁kun oo ▁! ▁ J u n d iin ▁samii ▁dhaa ▁hedduun ▁maleeykich aan ▁wajjiin ▁mullata nii ▁ti ▁; 11 | ▁Qo o w l ichi ▁jasada ▁tahee ▁ti ▁, ▁nu ▁jidduu s ▁jiraate ▁. ▁ U lfina ▁isaa ▁kan ▁ni ' imaa ▁fi ▁dhugaa ▁dhaan ▁guutame e s ▁agarre ▁; ▁ulfinni ▁isaa ▁tiis ▁kan ▁Ilmi ▁tokk ichi ▁Abbaa ▁isaa ▁irraa ▁arkatu ▁ture ▁. 12 | ▁Hogguu n ▁dhaqee s ▁bakka ▁isiniif iin ▁qopheeys a ▁; ▁bakka an ▁ani ▁jiru ▁isin iis ▁akka ▁jiraattan iif ▁, ▁deebi ' ee ▁dhufee ▁tiin ▁gara ▁kiyya tti ▁isin ▁fudhadha ▁. 13 | ▁Yesuus ▁sadaffaa ▁ammaa s ▁, ▁`` ▁Simoon ▁, ▁yaa ▁ilma ▁Yohannis ▁, ▁ana ▁jaalatta a ▁? ▁'' ▁jedhee ni ▁. ▁Phexiroos ▁ ammoo ▁waan ▁inni ▁hoggu ▁sadih ▁, ▁`` ▁Ana ▁jaalatta a ▁? ▁'' ▁isaan ▁jedhee f ▁jecha ▁ni ▁na h e ▁; ▁deebisee ▁tiis ▁ammaa s ▁, ▁`` ▁Yaa ▁Gooftaa ▁, ▁ati ▁waan ▁hundumaa ▁ni ▁beeyta ▁; ▁akka an ▁si ▁jaaladhu u s ▁ atuu ▁ni ▁beeyta ▁'' ▁jedhee ni ▁; ▁Yesuus ▁ammaa s ▁, ▁`` ▁Hoolo ta ▁kiyya ▁tii y si ▁! ▁'' ▁isaan ▁jedhe ▁. 14 | ▁Jalqab a ▁qoowlicha a ▁tu ▁ture ▁; ▁qoo w l ichi ▁ niis ▁Rabbii ▁wajji ▁ture ▁; ▁qoo w l ichi nuu ▁Rabbi ▁ture ▁. 15 | ▁Arra ▁magaalaa ▁Daawwid itti ▁naja - baasoo n ▁isiniif ▁dhalate e ▁ti ▁jira ▁; ▁inni ▁ niis ▁Kiristoos ▁Gooftich a ▁. 16 | ▁Guyyaa ▁borumtii s ▁Yesuus ▁Galiilaa ▁dhaquuf ▁ka ' e ▁; ▁Filiphoos ▁agarra an ▁, ▁`` ▁Na ▁duukaa ▁bu ' i ▁! ▁'' ▁isaan ▁jedhe ▁. 17 | ▁Ergamoonni ▁ niis ▁deebi ' anii ▁, ▁waan ▁hojjatan ▁hundumaa ▁Yesuusitti ▁himan ▁; ▁inni ▁ niis ▁isaan ▁if ▁jala ▁fuudhee ▁ti ▁, ▁karaa ▁magaalaa ▁Beetesayida a ▁jedhamu ▁tokko ▁dhaquuf ▁kophaa ▁isaanii ▁miila ▁jalaa ▁baafat an ▁. 18 | ▁Ammaa s ▁abb ummaan ▁fedhee ▁huu ▁, ▁yoo ▁isin ▁simachuu ▁dhaa ▁baate ▁, ▁yookii n ▁dubbii ▁teeysan iis ▁dhagayuu ▁baate ▁, ▁mana ▁sanii ▁, ▁yookii n ▁mandaricha ▁sanii ▁hogguu ▁baatan ▁, ▁dhukkee ▁miila ▁keeysan ▁irraa ▁achitti ▁dhadhaw a dhaa ▁! 19 | ▁Sab oota ▁biyya ▁addunyaa ▁hundumaa ▁keeysaa ▁, ▁nam ni ▁Rabbi ▁sodaa tu ▁, ▁kan ▁waan ▁haqaa ▁dalaguu s ▁fuul a ▁Is aa ▁duratti ▁fudhatama a ▁dha ▁. 20 | ▁Filiphoos iis ▁akkuma ▁Indiriiyaas ii ▁fi ▁Phexiroos ▁, ▁nama ▁magaalaa ▁Beetesayida a ▁tii ▁ture ▁. 21 | ▁Kanaaf ▁jecha ▁, ▁qalbii n ▁teenya ▁, ▁jallina ▁xalalii ▁irraa ▁xaahar amtee ▁, ▁ammaa s ▁qaam ni ▁keenyaa s ▁bishaan ▁xaahara an ▁dhiq amee ▁, ▁koott aa ▁mee ▁gar uma a ▁fudhanne e ▁amantii ▁mirkana ▁qabduu n ▁gara ▁Rabbii ▁haa ▁dhihaann uu ▁! 22 | ▁Bakka ▁ayyaantichaa tti ▁sagad uuf ▁warra ▁ol ▁bahan ▁keeysaa ▁, ▁hamm i ▁tokko ▁Giriigota ▁turan ▁. 23 | ▁Dilii ▁teenya ▁yoo ▁isa tti ▁if ▁himne ▁, ▁inni ▁amanamaa ▁fi ▁qaceelaa ▁waan ▁taheef ▁jecha ▁, ▁dilii ▁teenya ▁nuuf ▁oomfii ▁godha ▁, ▁jallina ▁keenya ▁hundumaa ▁irraa hiis ▁nu ▁xaahara ▁. 24 | ▁Warra ▁amma ▁biyya ▁addunyaa ▁tana ▁irratti ▁taajir oota ▁tahan ▁, ▁akka ▁hin ▁dha y t oo y ne ▁isaan ▁ajaj i ▁! ▁Abdii ▁isaanii ▁tiis ▁ a d du n y a ▁nama ▁hin ▁ aman s iif ne ▁tana ▁irra ▁akka ▁hin ▁kee n y e ▁hubachiis i ▁! ▁Abdii ▁isaanii ▁ ammoo ▁Rabbi ▁kan ▁waa ▁hundumaa ▁haa ▁itti in ▁ga mmann uuf ▁jecha ▁arjummaa n ▁nu ▁badhaas e ▁irra ▁akka ▁kaaya n ▁gorsi ▁! 25 | ▁B a dii ▁teenya ▁hundumaa ▁jalaa ▁nu ▁eewal uu ▁fi ▁ummata ▁tokko ▁kan ▁matuma ▁isaa ▁qofaaf ▁tahe ▁, ▁ammaa s ▁kan ▁hojii ▁gaarii ▁hojjach utti ▁mirqaan u ▁qulqulleeys uuf ▁, ▁nuuf ▁jecha ▁dabarsee ▁ti ▁if ▁kanne ▁. 26 | ▁Akka ▁badhaadhummaa ▁ni ' imaa ▁Rabbitti ▁Kiristoos ▁dhiiga ▁isaa ▁nuuf ▁dhangalaas uu ▁isaa ▁tiin ▁, ▁najaata a ▁fi ▁oomfii ▁dilii ▁tiis ▁nu ▁arka chi is e ▁. 27 | ▁Kan ▁ol ▁if ▁kaasu ▁hundin uu ▁gadi ▁deebi fama ▁, ▁gadi ▁kan ▁if ▁qabu ▁hundin uu ▁ol ▁kaafam a ▁'' ▁jedhe ▁. 28 | ▁Kana ▁booddee ▁Beetesayida a ▁dha ▁dhufan ▁. ▁Achitti is ▁jarr een ▁tokko ▁namticha ▁jaamaa ▁tokko ▁gara ▁Yesuus ▁fidanii ▁ti ▁, ▁akka ▁inni ▁harka an ▁isa ▁tuq u ▁kadhatan ▁. 29 | ▁Nuti ▁dilii ▁jalaa ▁duune e ▁qaceelina af ▁akka ▁jiraannuu f ▁jecha ▁, ▁Kiristoos ▁dilii ▁teenya ▁jasada ▁isaa ▁tiin ▁fannoo ▁irratti ▁baadhat e ▁; ▁madaa ▁isaa ▁tiin ▁isin ▁aafih am tan ▁. 30 | ▁Nam oonni ▁ ammoo ▁tana ▁hubachuu ▁dhaan ▁isa ▁duukaa ▁bu ' an ▁; ▁inni ▁ niis ▁isaan ▁simat ee ▁ti ▁, ▁waa ' ee ▁Mootummaa ▁Rabbii ▁isaanitti ▁hime ▁; ▁warra ▁aafih amuun ▁isaan ▁barbaachise e s ▁ni ▁aafih e ▁. 31 | ▁Eega ▁Herodis ▁du ' e ▁booda ▁, ▁maleeykaa n ▁Gooytaa ▁tokko ▁Mi s ra tti ▁mana ama tti ▁Yoseef itti ▁mullatee ▁ti ▁, 32 | ▁Rabbi ▁Abbaan ▁quduraa ▁, ▁dandaya an ▁, ▁waan ▁guddaa ▁dha ▁anaaf ▁godhe ▁; ▁maqaa n ▁isaa ▁qulqulluu ▁dha ▁. 33 | ▁Isaan ▁hundaa nuu ▁, ▁`` ▁Eenyu ▁illeen ▁Na ▁duukaa ▁bu ' uu ▁yoo ▁fedhe ▁, ▁if ▁kaade e ▁, ▁fannoo ▁isaa ▁tiis ▁baadhat ee ▁, ▁guyy uu ▁ana ▁duukaa ▁haa ▁bu ' uu ▁; 34 | ▁B al l eeysaa ▁teeysanii f ▁jecha ▁ tuma m tanii ▁yoo ▁o b si tan ▁, ▁zaj uma ▁maali ▁qabdu ▁? ▁Gaar ii ▁dalaguu ▁keeysaniif ▁dararam a ▁isin ▁irra ▁gahu ▁yoo ▁o b si tan ▁ ammoo ▁, ▁Rabbi ▁biraa ▁ni ' imaa ▁ni ▁arkat tu ▁. 35 | ▁`` ▁ H am d iin ▁Rabbii f ▁samii ▁gubba tti ▁haa ▁taatu ▁, ▁nagay ni ▁warra ▁Rabbi ▁itti ▁gammade ef ▁, ▁lafa ▁irratti ▁haa ▁tahu ▁'' ▁jechaa ▁Gooyticha ▁ni ▁faars u ▁turan ▁. 36 | ▁Isin ▁ ammoo ▁wal ▁ c i nii uu ▁fi ▁wal ▁nyaachuu ▁dhaan ▁akka ▁wal ▁hin ▁fix ne ▁if ▁eeggadhaa ▁! 37 | ▁Warra ▁bobbaa ▁kana ▁irraa ▁hin ▁tahin ▁, ▁hoolota ▁biraa ▁tiis ▁ni in ▁qaba ▁; ▁isaanii s ▁qaceel c h ee ▁fi duun ▁ana ▁irra ▁jira ▁, ▁sagalee ▁tiyyaa s ▁ni ▁dhagayu ▁; ▁walitti ▁bobbaa ▁tokko ▁ni ▁tahu ▁, ▁ti ysit uun ▁isaanii ▁tiis ▁tokko ▁taha ▁. 38 | ▁Inni ▁ummata ▁danuu ▁barattoota ▁isaa ▁wajjiin ▁ifitti ▁yaamee ▁ti ▁, ▁`` ▁Nam ▁tokkoo ▁illeen ▁ana ▁duukaa ▁bu ' uu ▁yoo ▁barbaade ▁, ▁if ▁kaadu u ▁dha ▁qaba ▁; ▁fannoo ▁isaa ▁tiis ▁ol ▁fuudhee ▁baadhat ee ▁, ▁ana ▁duukaa ▁haa ▁bu ' u ▁! 39 | ▁Yesuus ▁lama d dii ▁ammaa s ▁, ▁`` ▁Simoon ▁, ▁yaa ▁ilma ▁Yohannis ▁, ▁ana ▁jaalatta a ▁? ▁'' ▁jedhee ni ▁; ▁hogguus ▁inni ▁, ▁`` ▁Eeyye ▁yaa ▁Gooftaa ▁! ▁Akka an ▁si ▁jaaladhu ▁ atuu ▁ni ▁beeyta ▁'' ▁jedhee ▁ti ▁deebiseef ▁. ▁Yesuus ▁ammaa s ▁, ▁`` ▁Hoolo ta ▁kiyya ▁eegi ▁! ▁'' ▁jedhee ni ▁. 40 | ▁Kana ▁booddee ▁ammaa s ▁, ▁ g ee b ichaa s ▁fuudhee ▁ti ▁, ▁galata ▁galche e ▁, ▁isaaniif ▁kanne ▁; ▁isaan ▁hundi ▁ niis ▁ g ee b icha ▁keeysaa ▁ni ▁dhug an ▁. 41 | ▁Yesuus ▁hanga ▁ifii ▁ummata ▁san ▁geggeeys uu ▁, ▁baratt oonni ▁isaa ▁doonii ▁seenanii ▁, ▁deemanii ▁isa ▁dura ▁gama ▁akka ▁ceeh an ▁godhe ▁. 42 | ▁Kanaaf ▁yaa ▁obboleeyyan ▁koo ▁jaalatam oo ▁, ▁warra an ▁yaada an ▁dha rra ' uuf ▁, ▁kanneen ▁na ▁gammachiif tanii ▁fi ▁ammaa s ▁ goo n f oo ▁koo ▁taatanii s ▁, ▁amantiin ▁haala ▁kanatti ▁Gooftatti ▁jabaadhaa ▁dhaabbadh aa ▁! 43 | ▁Jaalal a ▁jechuun ▁, ▁nuti ▁Rabbi ▁jaalanne ▁jechuu ▁miti ▁, ▁ ammoo ▁Rabbi ▁ifumaa ▁nu ▁jaalatee ▁, ▁Ilma ▁isaa ▁ gumaa ▁dilii ▁teenyaa ▁godhee ▁erge ▁jechu ▁. 44 | ▁ T ii ysit oonni ▁sun ▁, ▁waan ▁maleeyk oonni ▁Rabbii ▁itti ▁himan ▁hundumaa ▁dhagayanii ▁, ▁akkuma ▁itti ▁himam e ▁tahee ▁ija ▁isaanii ▁tiin ▁waan ▁arkanii f ▁jecha ▁, ▁Rabbii f ▁hamdii ▁dhiheeysa a ▁, ▁galateeyfach aa ▁tiis ▁bakka ▁isaanitti ▁deebi ' an ▁. 45 | ▁Ati ▁hoo ▁yaa ▁Qifirnaahom ▁, ▁hanga ▁sami tti ▁ol ▁kaafam uu ▁dha ▁barbaadde e ▁? ▁ H u n q uuf tuu ▁hoong aa ▁hin ▁qabne tti ▁gadi ▁darbam ta ▁; ▁hojiin ▁quduraa ▁Rabbii ▁tan ▁si ▁keeysatti ▁godhamt e ▁, ▁odoo ▁Sidoon ▁keeysatti ▁godhamtee ▁jirti ▁tahee ▁, ▁si laa ▁hanga ▁arraatti ▁jiraatte e ▁ti ▁haft i ▁turte ▁. 46 | ▁Kana ▁booda ▁bantii ▁samii ▁haaraya a ▁fi ▁dachii ▁haaraya an ▁arke ▁; ▁samii n ▁duraa ▁tii ▁fi ▁, ▁dachiin ▁duraa ▁tiis ▁dabra nii ▁ti ▁jiru ▁, ▁bahar iis ▁deebi ' ee ▁hin ▁arkam u ▁. 47 | ▁Geeba ▁fudhatee ▁ti ▁, ▁Rabbii f ▁galata ▁galche e ▁, ▁`` ▁ H oo dhaa ▁, ▁waliin ▁gaha a ▁dhugaa ▁! 48 | ▁Kun iis ▁dhaloota ▁jall aa ▁fi ▁mi c c ii ra maa ▁tahe ▁kana ▁keeysatti ▁, ▁isin ▁akka ▁urjii lee ▁bantii ▁samii ▁keeysaa tti ▁wal a q q i f tanii ▁, ▁joollee ▁Rabbii ▁nassab a ▁, ▁xaliil a ▁illaa ▁hin ▁qabne ▁isin ▁godha ▁. 49 | ▁Jaalal a ▁keeysa ▁sodaa n ▁hin ▁jirtu ▁; ▁jaalalti ▁dhugaa ▁, ▁sodaa ▁baaft ee ▁ti ▁darb i ti ▁; ▁sodaa n ▁adab aan ▁walitti ▁hidh a ▁, ▁namticha ▁sodaa tu ▁jaalalti ▁isaa ▁bakka ▁hin ▁geenye ▁. 50 | ▁Yesuus ▁, ▁`` ▁ T i ysit uun ▁gaarii n ▁Ana ▁, ▁ti ysit uun ▁gaarii n ▁lubbuu ▁isaa ▁hoolota af ▁dabarsee ▁ti ▁kanna ▁. 51 | ▁Achi ▁irratti ▁Yesuus ▁, ▁`` ▁Eegas uu ▁waan ▁Qeesaarii f ▁kannamuu ▁qabu ▁Qeesaarii f ▁kannaa ▁; ▁waan ▁Rabbii f ▁kannamuu ▁qabuu s ▁Rabbii f ▁kannaa ▁! ▁'' ▁isaaniin ▁jedhe ▁; ▁ j e chi ▁kun iis ▁isaan ▁ ajaa ' ibsi s e ▁. 52 | ▁Guyyaa ▁eeybifama a ▁keenya ▁san ▁ a fachaa ▁, ▁mu l lachuu ▁durrii ▁ulfina ▁Rabbii ▁keenya ▁guddaa ▁dhaa ▁, ▁ammaa s ▁Yesuus ▁Kiristoos ▁naja - baasoo ▁dha ▁eeggann a ▁. 53 | ▁`` ▁Eegas uu ▁waan ▁Qeesaarii f ▁kannamuu ▁qabu ▁Qeesaarii f ▁kannaa ▁, ▁ammaa s ▁waan ▁Rabbii f ▁kannamuu ▁qabuu s ▁Rabbii f ▁kannaa ▁! ▁'' ▁isaaniin ▁jedhe ▁. 54 | ▁Ani ▁isinitti in ▁hima ▁, ▁namticha ▁kaan ▁odoo ▁hin ▁tahin ▁, ▁namtichi ▁kun ▁Rabbi ▁duratti ▁qaceel atti ▁lakkaawa mee ▁ti ▁mana ▁isaatti ▁gala ▁; ▁kan ▁ol ▁if ▁kaasu ▁eenyu ▁illeen ▁gadi ▁qabam a ▁, ▁kan ▁if ▁gadi ▁qaqqab u ▁ ammoo ▁ol ▁kaafam a ▁'' ▁jedhe ▁. 55 | ▁Isin ii ▁miti ▁kan ▁ana ▁filate ▁; ▁deem tanii ▁akka ▁firii ▁gootan ▁, ▁firii n ▁teeysan iis ▁tan ▁ay aa ▁qabdu ▁akka ▁taatuu f ▁, ▁kan ▁isin ▁fi le ▁bar ▁anaa ▁; ▁eegas uu ▁, ▁Abbaan ▁kiyya ▁waanuma ▁isin ▁maqaa ▁kiyyaan ▁kadhattan ▁, ▁isiniif ▁ni ▁kanna ▁. 56 | ▁Yesuus ▁ammaa s ▁, ▁`` ▁Ani ▁ifa ▁addunyaa ▁ti ▁, ▁nam ni ▁ana ▁duukaa ▁bu ' u ▁ifa ▁jireenyaa ▁ti ▁qabaa ta ▁malee ▁, ▁dukkana ▁keeysa ▁abad an ▁hin ▁deemu ▁'' ▁jedhee ▁ti ▁namoota tti ▁dubbate ▁. 57 | ▁Nuti is ▁biyya ▁Yihudaa ▁tii ▁fi ▁magaalaa ▁Yerusaaleem ▁keeysatti is ▁, ▁waan ▁Yesuus ▁hojjate ▁hundumaaf ▁raga - baatuu ▁dha ▁; ▁isaanii s ▁muka ▁irratti ▁fannis anii ▁ti ▁isa ▁ajjeesan ▁. 58 | ▁Hogguus ▁, ▁Yesuus ▁barattoota ▁isaa ▁tiin ▁, ▁`` ▁Nam ni ▁tokko ▁illeen ▁ana ▁duukaa ▁bu ' uu ▁yoo ▁barbaade ▁, ▁if ▁kaade e ▁, ▁fannoo ▁isaa ▁tiis ▁baadhat ee ▁, ▁ana ▁duukaa ▁haa ▁bu ' uu ▁! 59 | ▁Rabbi ▁ni ' imaa ▁isaa ▁, ▁tan ▁Banii ▁Aadam ▁hundumaaf ▁najaata ▁kannitu ▁mullisee ▁ti ▁jira ▁. 60 | ▁Isaan iis ▁, ▁`` ▁Ta n ▁Qeesaar i ▁'' ▁isaan ▁jedhan ▁. ▁Hogguus ▁Yesuus ▁, ▁`` ▁Eegas uu ▁waan ▁Qeesaar ▁kanneen ▁tahan ▁, ▁Qeesaarii f ▁kannaa ▁, ▁waan ▁Rabbii ▁kanneen ▁tahanii s ▁, ▁Rabbuma af ▁kannaa ▁! ▁'' ▁isaaniin ▁jedhe ▁. 61 | ▁Rabbi ▁tokkum a ▁, ▁ammaa s ▁araar - qaboo n ▁Rabbii ▁fi ▁nama ▁walitti ▁fi duu s ▁tokko ▁; ▁inni ▁ niis ▁Yesuus ▁Kiristoos ▁kan ▁nama ▁tahee ▁mullate ▁san i ▁. 62 | ▁B a dhaa dhu mma an ▁, ▁o g ummaan ii ▁fi ▁beeku m si ▁hundin uu ▁, ▁isa ▁keeysatti ▁dhooyfam anii ▁ti ▁jiru ▁. 63 | ▁Yoo ▁jiraanne ▁, ▁akeeka ▁Gooftaa ▁tiin ▁jiraanna ▁, ▁yoo ▁duune e s ▁akeeka ▁Gooftichaa ▁tiin ▁ duun a ▁; ▁eegas uu ▁, ▁jiraannuu s ▁ duun uus ▁nuti ▁warruma ▁Gooftichaa ▁ti ▁. 64 | ▁Qalbii n ▁teeysan iis ▁bakk uma ▁addunyaa n ▁teeysan ▁jirtu ▁taa ti ▁'' ▁jedhe ▁. 65 | ▁ O b si ▁ niis ▁fitinam anii ▁bahuu ▁dha ▁fida ▁, ▁fitinam anii ▁bahuu niis ▁abdii ▁dha ▁fida ▁. 66 | ▁` ▁Ani ▁rahamata an ▁barbaada ▁malee ▁, ▁aarsaa ▁miti ▁' ▁jechuun ▁hiikkaa ▁maal ▁akka ▁qabu ▁, ▁deema a ▁tii ▁bara dhaa ▁! ▁Ani ▁warra ▁qaceelota ▁yaam uuf ▁hin ▁dhufne ▁, ▁warra ▁dilaawoota an ▁yaam uuf ▁dhufe ▁malee ▁'' ▁jedhe ▁. 67 | ▁ T ii ysit oonni ▁tokko ▁ ammoo ▁, ▁naann oo ▁sanitti ▁, ▁bobbaa ▁horii ▁isaanii ▁halkan ▁dirree ▁keeysatti ▁ni ▁eegu ▁turan ▁. 68 | ▁Yoo ▁ aa r tan ▁illee ▁ aa rri ▁keeysan ▁dil itti ▁isin ▁hin ▁na q in ▁! ▁ O duma ▁ aa r tanii ▁aduu n ▁isinitti ▁hin ▁dhih in ▁! 69 | ▁Bara ▁Herodis ▁mootichi ▁, ▁Yihudaa ▁bulchaa ▁ture ▁, ▁warra ▁luboota ▁murna ▁Ab i iyaa ▁keeysaa ▁kan ▁tahe ▁, ▁luba ▁ Z akkaariyaas ▁kan ▁jedhamu ▁tokkoo ▁tu ▁ture ▁; ▁niitii n ▁isaa ▁tiis ▁shanyii ▁luba ▁Ar oon ▁irraa hi ▁, ▁maqaa n ▁isii ▁tiis ▁Elsaa b e x ▁jedhama ▁. 70 | ▁Haa wwan ▁mucicha a ▁bar uma ▁bara an ▁, ▁Ayyaan a ▁Bira - D a b r uu ▁kabaju u f ▁, ▁Yerusaaleem ▁ni ▁dhaqu ▁turan ▁. 71 | ▁Yesuus iis ▁, ▁`` ▁Na ▁duukaa ▁bu ' aa ▁! ▁Ani is ▁akka ▁namoota ▁walitti ▁qabdanii n ▁isin ▁godha ▁! ▁'' ▁isaaniin ▁jedhe ▁. 72 | ▁Rabbi ▁oduma ▁beekanii ▁, ▁akka ▁Rabbummaa ▁isaatti ▁ulfina ▁hin ▁kannin eef ▁, ▁hamdii s ▁isaaf ▁hin ▁galchin e ▁; ▁ ammoo ▁yaada ▁isaanii ▁tiin ▁waan ▁maal aa - yaa nii ▁duukaa ▁dha ▁bu ' an ▁, ▁du u daa n ▁qalbii ▁isaanii ▁tiis ▁ni ▁du kk an eey f amte ▁. 73 | ▁Kanaaf ▁nuti ▁karaa ▁irraa ▁akka ▁hin ▁ma y neef ▁, ▁waan ▁kanneen ▁dhageenye ▁san ▁, ▁caalatti ▁itti ▁yaad n ee ▁fardeeyfa chuu ▁tu ▁nu ▁barbaachisa ▁. 74 | ▁Bakka ▁itti ▁guddat e ▁Naazireet ▁dhufee ▁ti ▁, ▁guyyaa ▁Sanbataa ▁, ▁akkuma ▁aadaa ▁isaa ▁mana ▁sagadaa ▁isaanii ▁dhaqee ▁, ▁Kitaaba ▁dubbisu u f ▁ol ▁ka ' e ▁. 75 | ▁Yesuus ▁ammaa s ▁fakkeenya an ▁isaanitti ▁dubbach utti ▁ka ' e ▁; ▁`` ▁Namtichi ▁tokko ▁mukkeen ▁inabaa ▁hedduu ▁lafa ▁isaa ▁irra ▁dhaab ee ▁, ▁naannawa ▁isaatti is ▁dallay a ▁dhagaa ▁itti ▁ijaar ee ▁, ▁boolla ▁firii ▁isaa ▁itti ▁cunfa n ▁qot ee ▁, ▁siree ▁irra ▁taa ' anii ▁eega niin iis ▁itti ▁ijaar e ▁. ▁Kana ▁booda ▁warra ▁hojii ▁inabaa ▁beeku tti ▁k i ree ysee ▁ti ▁, ▁imaltu u ▁takka ▁dhaqe ▁. 76 | ▁Yaa ▁obboleeyyan ▁koo ▁, ▁ani ▁akka ▁nama ▁duruun ▁fudhatee ▁, ▁harka an ▁gaha tee tti ▁if ▁hin ▁lakkaawu ▁. ▁Haa ▁tahu ▁malee ▁, ▁waan ▁takka ▁numa an ▁godha ▁; ▁waan ▁na ▁booddee tti ▁hafe ▁yaaduu ▁dhiisee ▁, ▁gara ▁waan ▁na ▁dura ▁jirutti in ▁hii xa dha ▁. 77 | ▁Eega ▁ f a xar a ▁fix an ▁booda ▁Yesuus ▁, ▁Simoon ▁Phexiroosii n ▁, ▁`` ▁Simoon ▁, ▁yaa ▁ilma ▁Yohannis ▁! ▁Isaan ▁kanneen ▁caalaa ▁ana ▁ni ▁jaalatta a ▁? ▁'' ▁jedhee ni ▁; ▁inni ▁ ammoo ▁, ▁`` ▁Eeyye ▁yaa ▁Gooftaa ▁! ▁Akka an ▁si ▁jaaladhu ▁ atuu ▁ni ▁beeyta ▁'' ▁jedhee ni ▁. ▁Yesuus ▁deebisee ▁, ▁`` ▁ I l moo lee ▁hoolota a ▁tiyya ▁tii y si ▁! ▁'' ▁jedhee ni ▁. 78 | ▁Warr i ▁araara ▁nagayaa ▁buus an ▁eeybifamoo ▁dha ▁, ▁isaan ▁joollee ▁Rabbii ▁ti ▁jedhamu ▁. 79 | ▁Kana tti ▁ aan see ▁hunduma ▁isaanii ▁tiin ▁, ▁`` ▁Jireeny i ▁namaa ▁hoddomin a ▁qabeenya ▁isaa ▁tiin ▁waan ▁hin ▁dhaab ban neef ▁jecha ▁if ▁eeggadhaa ▁, ▁bolol ummaa ▁hunda ▁irraa ▁if ▁ti y saa ▁! ▁'' ▁jedhee ni ▁. 80 | ▁qara f aa ▁, ▁waan ▁urg ooytuu ▁dib a tan ▁, ▁libaanat a ▁, ▁qu m bii ▁, ▁h a p h ee ▁, ▁dhugaati i ▁cuunfa a ▁inabaa ▁, ▁dha dhaa ▁ejersa a ▁, ▁fi c aa ▁ x iin a ▁, ▁qamadi i ▁, ▁ l oon ▁, ▁hoolaa ▁, ▁farda ▁, ▁konkolaat aa ▁farda an ▁harki f amu ▁, ▁gabroo taa ▁fi ▁lub b uma ▁namaa ▁illee ▁kan ▁isaan ▁irraa ▁bit u ▁hin ▁jiru ▁. 81 | ▁Hogguus ▁Phexiroos ▁jara ▁saniin ▁, ▁`` ▁Rabbi ▁akka ▁nama ▁wal ▁hin ▁ c aa l chi f ne ▁amma ▁anaaf ▁gal e ▁. 82 | ▁Maariyaam ▁ ammoo ▁, ▁dubbii ▁tana ▁hundumaa ▁garaa tti ▁qaba tt ee ▁, ▁itti ▁yaadd i ▁turte ▁. 83 | ▁Man a ▁seenanii ▁, ▁mucicha ▁haadha ▁isaa ▁Maariyaam ▁wajjiin ▁arkanii ▁ti ▁, ▁lafa ▁gayan ii ▁ti ▁Is aaf ▁sagadan ▁; ▁qandii ▁isaanii ▁tiis ▁ba n anii ▁, ▁ziqeeya ▁, ▁libaanat aa ▁fi ▁qu m bii ▁harka ▁fuudha a ▁kannanii f ▁. 84 | ▁Guyyaa ▁beela ma ▁isaanitti ▁Herodis ▁uffata ▁mootummaa ▁uffatee ▁, ▁bakka ▁ol - ka ' aa ▁isaaf ▁qophaaw e ▁irra ▁taa ' ee ▁ti ▁, ▁isaanitti ▁dubbate ▁. 85 | ▁Yaa ▁joollee ▁too ▁! ▁Akka ▁isin ▁dilii ▁hin ▁h o j j an neef iin ▁tana ▁isiniif ▁barreeysa ▁; ▁nam ni ▁tokko ▁illeen ▁yoo ▁dilii ▁godhe ▁ ammoo ▁, ▁nuti ▁magan taa ▁Abbaa ▁biraa ▁ni ▁qabna ▁; ▁inni ▁ niis ▁Yesuus ▁Kiristoos ▁qaceelaa ▁dha ▁. 86 | ▁Tun ▁hikmaa ▁dha ▁barbaachift i ▁; ▁lakkooysi ▁kun ▁lakkooysa ▁namaa ▁waan ▁taheef ▁jecha ▁, ▁nam ni ▁aaqil li ▁qabu ▁lakkooysa ▁hi saa bee ▁beekuu ▁ni ▁dandaya ▁. ▁L a kk oo y s ichi ▁ niis ▁dhibba ▁jaha a ▁fi ▁jahaatam ii ▁jaha ▁. 87 | ▁B oo y icha ▁isaanii ▁keeysatti ▁, ▁`` ▁Magaal aa ▁guddittii ▁, ▁tan ▁uffata ▁qu n c ee ▁qoo n xar ▁aslii ▁dhaa ▁uffata ▁diimaa ▁fi ▁hu llaa ▁diimaa ▁ u f fachaa ▁turte ef ▁, ▁tan ▁ziqeeya a ▁fi ▁lu ' ulu ' a ▁qa q qaalii ▁cal lee s ▁na qa chaa ▁turte ▁saniif ▁magan ▁! ▁Magan ▁! 88 | ▁Namtichi ▁tokko ▁Barreeyfam a ▁Qulqulluu ▁keeysaa ▁bakka ▁tokkotti ▁ragaa ▁bahuu n ▁, ▁`` ▁Yaa ▁Rabbi ▁, ▁nam ni ▁maali ▁, ▁akka ▁isa ▁yaada ttuu f ▁? ▁Yook iin ▁Ilmi ▁Namaa ▁maali ▁, ▁akka ▁isaaf ▁yaadd uuf ▁? 89 | ▁Ruuhaan onni ▁hamoon ▁sun ▁mootoota ▁san ▁bakka ▁afaan ▁Ibr ootaa tti ▁`` ▁Ar maa g e doo n ▁'' ▁jedhamtu tti ▁walitti ▁qaban ▁. 90 | ▁Nam ittiin ▁karra ▁eeydu ▁sun ▁Phexiroosii n ▁, ▁`` ▁Ati is ▁barattoota ▁namticha ▁kanaa ▁keeysaa ▁tokko ▁mitii ▁ree ▁? ▁'' ▁jette ▁; ▁inni ▁ ammoo ▁, ▁`` ▁Ana a ▁miti ▁'' ▁jedhe ▁. 91 | ▁Akkuma ▁kanatti ▁ammaa s ▁, ▁daadhii ▁inabaa ▁haaraya ▁qandii ▁moofaa tti ▁hin ▁naqan ▁; ▁ a k an a ▁yoo ▁godhan ▁, ▁qandii n ▁ni ▁dh oo ti ▁, ▁daadhii niis ▁ni ▁dhangala a ti ▁, ▁qa n d ichi ▁ niis ▁turaab ▁taha ▁. ▁Akka s ▁godhuu ▁mann aa ▁, ▁daadhii ▁inabaa ▁haaraya ▁, ▁qandii ▁haaraya tti ▁na q uu ▁dha ▁, ▁akkasitti ▁daadhii ▁fi ▁qa n di niis ▁ni ▁eewalam u ▁'' ▁isaaniin ▁jedhe ▁. 92 | ▁Nam ni ▁sharrii ▁yoo ▁isin ▁irratti ▁hojjate ▁, ▁deebi f tanii ▁sharrii ▁isa ▁irratti ▁hin ▁hojjat inaa ▁! ▁Akkasi i ▁mann aa ▁, ▁waan ▁fuul a ▁nama ▁hundumaa ▁duratti ▁fudhatama a ▁tahe ▁godhuuf ▁yaada a ▁! 93 | ▁Ar a dda a ▁isaanii ▁tii ▁fi ▁qabeenya ▁isaanii ▁gurgur anii ▁, ▁horii ▁isaa ▁tiis ▁nama ▁hundumaaf ▁, ▁akka ▁isa ▁barbaachise tti ▁ni ▁hi r u ▁turan ▁. 94 | ▁Hogguu ▁isa ▁arkee s ▁, ▁gara ▁Anxookiyaa tti ▁isa ▁fi de ▁; ▁bara ▁tokko ▁guutuu ▁waldaya ▁sanii ▁wajjiin ▁taa h anii ▁, ▁namoota ▁hedduu ▁barsiisaa ▁ti ▁turan ▁. ▁Baratt oonni ▁Anxookiyaa tti ▁jalqaba ▁duraa ▁tiif ▁, ▁kiristaan ota ▁jedhama nii ▁ti ▁mogga af aman ▁. 95 | ▁Go n f oo ▁qoree ▁tiis ▁hojjatan ii ▁ti ▁, ▁mataa ▁isaa ▁irra ▁kaaya nii fi ▁; ▁ulee ▁ g o bee n saa ▁tiis ▁harka ▁mirgaa ▁tiin ▁qaba chi is an ▁. ▁Isa ▁duratti is ▁jilbiiffat anii ▁, ▁`` ▁Akka m ▁ree ▁, ▁yaa ▁mootii ▁ummata ▁Yihudaa ▁! ▁'' ▁jedhanii ▁ti ▁itti ▁hanjam an ▁. 96 | ▁Fitina tti ▁akka ▁hin ▁galle ef ▁dammaqaa ▁tii ▁, ▁kadhadha a ▁! ▁Ruuhaan ▁illeen ▁q oph ee ▁dha ▁, ▁foon ▁namaa ▁ ammoo ▁dadhab aa ▁dha ▁'' ▁jedhe ▁. 97 | ▁Fitina tti ▁akka ▁hin ▁galle ef ▁dammaqaa ▁tii ▁kadhadha a ▁! ▁Ruuhaan ▁illeen ▁q oph ee ▁dha ▁, ▁foon ▁namaa ▁ ammoo ▁dadhab aa ▁dha ▁'' ▁jedhee ni ▁. 98 | ▁As ▁irratti ▁Phexiroos ▁, ▁`` ▁Gooftaa ▁, ▁maaliif iin ▁amma ▁si ▁duukaa ▁deemuu ▁hin ▁dandeenye ▁? ▁Ani ▁kun oo ▁, ▁siif ▁jedhee ▁lubbuu ▁tiyya ▁ni in ▁kanna ▁'' ▁jedhee ni ▁. 99 | ▁Nam ▁tokko ▁illeen ▁lubbuu ▁tiyya ▁na ▁irraa ▁hin ▁fuu dhu ▁, ▁anuma a ▁tu ▁fedhii ▁tiyya an ▁kanna ▁malee ▁. ▁Ani ▁lubbuu ▁tiyya ▁dabarsee ▁kannuuf ▁, ▁ammaa s ▁deebisee ▁fudhachuuf iis ▁amrii n ▁qaba ▁; ▁ajaja ▁kanaa s ▁Abbaa ▁kiyya ▁irraa ▁hii n ▁fudhadhe ▁'' ▁jedhe ▁. 100 | ▁Warr i ▁ija - laa f oon ▁eeybifamoo ▁dha ▁, ▁isaan ▁addunyaa ▁dhaa ▁ni ▁dhaalu ▁. 101 | -------------------------------------------------------------------------------- /data/orm_data/set0.null.tag: -------------------------------------------------------------------------------- 1 | ((S[b]\NP)/NP)/NP 2 | -------------------------------------------------------------------------------- /data/orm_data/set0.tag: -------------------------------------------------------------------------------- 1 | ((S[b]\NP)/NP)/NP 2 | ((NP\NP)/(S[dcl]\NP))\(NP/NP) 3 | S[qem]/(S[dcl]\NP) 4 | ((S[ng]\NP)/PR)/NP 5 | S[em]/S[dcl] 6 | (S[pss]\NP)/PR 7 | ((S/S)/(S[adj]\NP))/NP 8 | (S[pss]\NP)/PP 9 | NP/((S[to]\NP)/NP) 10 | (NP\NP)/NP 11 | S[ng]\NP 12 | (S[pt]\NP)/S[qem] 13 | S[qem]/(S[to]\NP) 14 | (N/N)/(S[adj]\NP) 15 | (S[ng]\NP)/(S[b]\NP) 16 | ((S[dcl]\NP)/(S[ng]\NP))/NP 17 | ((S[pt]\NP)/S[em])/NP 18 | (S[dcl]\S[wq])/NP 19 | (S\S)/NP 20 | (S/S)/PP 21 | ((S[dcl]\NP)/NP)/(S[adj]\NP) 22 | ((S[dcl]\NP)/S[dcl])/NP 23 | ((S[dcl]\NP)/PP)/NP 24 | ((S[b]\NP)/(S[pt]\NP))/NP 25 | (NP/(N/PP))\NP 26 | S/S[dcl] 27 | ((S[b]\NP)/(S[b]\NP))/NP 28 | (S[ng]\NP)/PR 29 | (S[ng]\NP)/PP 30 | (((S\NP)\(S\NP))/((S\NP)\(S\NP)))/NP 31 | (S[dcl]\PP)/NP 32 | (S[adj]\NP)/S[em] 33 | (N\N)/(S[b]\NP) 34 | (((S[dcl]\NP)/PP)/NP)/PR 35 | ((S[ng]\NP)/(S[ng]\NP))/NP 36 | ((N/N)/(N/N))\(S[adj]\NP) 37 | NP/PP 38 | (S[to]\NP)/(S[b]\NP) 39 | (NP/(S[dcl]\NP))/N 40 | ((S[b]\NP)/PP)/NP 41 | ; 42 | PP/NP 43 | (S[pss]\NP)/NP 44 | ((S[pt]\NP)/PP)/NP 45 | (S[adj]\NP)\NP 46 | S[qem]/(S[dcl]/NP) 47 | (S[ng]\NP)/S[qem] 48 | ((S[b]\NP)/NP)/PR 49 | ((S[b]\NP)/NP)/PP 50 | S[em]/S[b] 51 | (S[adj]\NP)/(S[adj]\NP) 52 | (((S[dcl]\NP)/(S[to]\NP))/PP)/NP 53 | ((S[dcl]\NP[expl])/S[em])/PP 54 | ((S[dcl]\NP[thr])/PP)/NP 55 | (S[pt]\NP)/(S[pss]\NP) 56 | (N\N)/S[inv] 57 | ((S[b]\NP)/(S[adj]\NP))/NP 58 | (PP/(S[ng]\NP))/NP 59 | (S[pt]\NP)/S[dcl] 60 | (S[pss]\NP)/(S[to]\NP) 61 | (S[dcl]\NP)/S[dcl] 62 | (S[b]\NP)/NP 63 | ((S[b]\NP)/PP)/(S[adj]\NP) 64 | N/N 65 | ((S[pt]\NP)/(S[to]\NP))/NP 66 | (((S\NP)\(S\NP))/S[dcl])/N 67 | ((S[dcl]\NP)/(S[adj]\NP))/NP 68 | ((S[ng]\NP)/PP)/NP 69 | (((S\NP)\(S\NP))/(S[ng]\NP))/NP 70 | (S/S)/(S[dcl]\NP) 71 | (NP/NP)\(S[adj]\NP) 72 | ((S\NP)/(S\NP))\NP 73 | N/PP 74 | ((S[dcl]\NP)/(S[b]\NP))/NP 75 | (S[dcl]/NP)/NP 76 | (NP\NP)/(S[dcl]/NP) 77 | S/(S[dcl]\NP) 78 | (N/S[em])/PP 79 | (NP/(S[dcl]/NP))/N 80 | ((S[pt]\NP)/(S[b]\NP))/NP 81 | (N/N)\(S[adj]\NP) 82 | ((S[adj]\NP)/PP)/PP 83 | ((S\NP)\(S\NP))/S[inv] 84 | (S[dcl]\(S[adj]\NP))/NP 85 | ((S\NP)/(S\NP))/NP 86 | (((S\NP)\(S\NP))\NP)/S[dcl] 87 | (N\N)/(S[ng]\NP) 88 | (NP\NP)/(NP\NP) 89 | (NP\NP)/(S[b]\NP) 90 | (S[dcl]\S[dcl])/NP 91 | (S[ng]\NP)/(S[adj]\NP) 92 | ((S\NP)\(S\NP))\NP 93 | N/(S[to]\NP) 94 | N/S[dcl] 95 | (S[pt]\NP)/(S[adj]\NP) 96 | ((S[pt]\NP)/NP)/PR 97 | ((S[pt]\NP)/NP)/PP 98 | ((S[qem]/(S[dcl]/NP))/N)/(S[adj]\NP) 99 | ((S\NP)\(S\NP))/(S[b]\NP) 100 | (S[dcl]\NP[thr])/(S[b]\NP) 101 | ((S[dcl]\NP)/PP)/(S[adj]\NP) 102 | ((S[b]\NP)/S[em])/NP 103 | (S[dcl]\NP)/(S[ng]\NP) 104 | (S[adj]\NP)/S[dcl] 105 | (S[dcl]\NP)/(S[pss]\NP) 106 | (S/S)\NP 107 | ((S[dcl]\NP)/S[em])/NP 108 | N/(S[adj]\NP) 109 | NP/N 110 | (N\N)/NP 111 | (S[pt]\NP)/S[em] 112 | ((S[ng]\NP)/(S[b]\NP))/NP 113 | N/S[for] 114 | ((S\NP)\(S\NP))/N 115 | ((S[pss]\NP)/PP)/NP 116 | ((S[pt]\NP)/(S[adj]\NP))/NP 117 | S[poss]/S[dcl] 118 | , 119 | ((S\NP)\(S\NP))/(S[adj]\NP) 120 | (S\NP)\(S\NP) 121 | ((S[ng]\NP)/(S[to]\NP))/NP 122 | ((N/PP)\(N/PP))/(S[ng]\NP) 123 | (S[b]\NP)/(S[to]\NP) 124 | (S[pss]\NP)/S[em] 125 | ((NP\NP)\NP)/NP 126 | (N/N)/(N/N) 127 | ((NP\NP)/(S[dcl]\NP))/N 128 | (S[pss]\NP)/(S[adj]\NP) 129 | ((S[ng]\NP)/(S[adj]\NP))/NP 130 | ((S\NP)\(S\NP))/PP 131 | ((S\NP)/(S\NP))/PP 132 | NP[thr] 133 | (S[dcl]/S[dcl])/NP 134 | (N/N)\NP 135 | (S[qem]/S[dcl])/(S[adj]\NP) 136 | ((S[ng]\NP)/PP)/PP 137 | ((S[ng]\NP)/PP)/PR 138 | (S[dcl]\S[dcl])\NP 139 | (N\N)/N 140 | NP\NP 141 | (S[pt]\NP)/(S[b]\NP) 142 | ((S\NP)\(S\NP))/S[em] 143 | (S[qem]/(S[dcl]\NP))/N 144 | (S[ng]\NP)/S[dcl] 145 | (NP\NP)/PP 146 | ((S[dcl]\NP[expl])/S[em])/(S[adj]\NP) 147 | (((S[b]\NP)/PP)/NP)/NP 148 | (S[b]\NP)/S[qem] 149 | ((S[adj]\NP)\(S[adj]\NP))/PP 150 | (((S\NP)\(S\NP))/(S[pt]\NP))/NP 151 | (S[dcl]\NP)/S[qem] 152 | (N\N)/(N\N) 153 | ((N/PP)\(N/PP))/N 154 | PP/(S[ng]\NP) 155 | S[dcl]/NP 156 | (((S\NP)\(S\NP))\((S\NP)\(S\NP)))/NP 157 | (S[dcl]/(S[pt]\NP))/NP 158 | (S[adj]\NP)/S[for] 159 | (S[q]/(S[b]\NP))/NP 160 | (((S[b]\NP)/PP)/NP)/PR 161 | ((S[dcl]\NP)/NP)/NP 162 | ((S[b]\NP)/(S[pss]\NP))/NP 163 | (NP/NP)/(S[adj]\NP) 164 | (S[b]\NP)/(S[pt]\NP) 165 | (S[dcl]\NP)/(S[dcl]\NP) 166 | ((S[adj]\NP)\(S[adj]\NP))/(S[adj]\NP) 167 | NP/NP 168 | (N\N)/(S[adj]\NP) 169 | NP[expl] 170 | ((S[b]\NP)/S[qem])/NP 171 | ((S\NP)/(S\NP))/(S[ng]\NP) 172 | (NP\NP)/(S[dcl]\NP) 173 | (((S[b]\NP)/S[for])/(S[adj]\NP))/NP[expl] 174 | (S[b]\NP)/S[em] 175 | S[dcl]\NP 176 | (((S\NP)\(S\NP))/(S[to]\NP))/N 177 | S[wq]/(S[q]/NP) 178 | (S[dcl]\NP[thr])/(S[pt]\NP) 179 | (S[dcl]\(S[b]\NP))/NP 180 | ((S/S)/(S/S))/NP 181 | S[qem]/S[dcl] 182 | N/(S[ng]\NP) 183 | ((S/S)\(S/S))/NP 184 | S[b]\NP 185 | (S[qem]/(S[dcl]/NP))/N 186 | (((S[b]\NP)/PP)/PP)/NP 187 | (S[dcl]\NP)/(S[pt]\NP) 188 | (S[dcl]\NP)/NP 189 | ((S[ng]\NP)/NP)/PR 190 | ((S[ng]\NP)/NP)/PP 191 | ((S[b]\NP)/NP)/(S[adj]\NP) 192 | (N\N)/(S[pss]\NP) 193 | ((S\NP)\(S\NP))/((S\NP)\(S\NP)) 194 | ((S[dcl]\NP)/PR)/NP 195 | ((N\N)/S[dcl])\((N\N)/NP) 196 | (S[adj]\NP)/NP 197 | ((S\NP)\(S\NP))/S[poss] 198 | ((S[dcl]\NP)/(S[pss]\NP))/NP 199 | S[pt]\NP 200 | PP/PP 201 | (((S\NP)\(S\NP))\NP)/NP 202 | (N/PP)/S[em] 203 | ((S[b]\NP)/(S[to]\NP))/NP 204 | (S[dcl]\NP)/PP 205 | (S[dcl]\NP)/PR 206 | (S[pt]\NP)/(S[ng]\NP) 207 | LQU 208 | (S/S)/(S[adj]\NP) 209 | NP/(S[dcl]\NP) 210 | (((S\NP)\(S\NP))\((S\NP)\(S\NP)))/S[dcl] 211 | (S[adj]\NP)/((S[to]\NP)/NP) 212 | ((S[pss]\NP)/(S[to]\NP))/PP 213 | (N\N)/S[dcl] 214 | ((S/S)\NP)/S[dcl] 215 | ((S[dcl]\NP)/(S[to]\NP))/NP 216 | (S[dcl]\NP[thr])/NP 217 | ((S[dcl]\NP)/(S[to]\NP))/(S[adj]\NP) 218 | (S/S)\(S/S) 219 | S[asup]\NP 220 | (S[adj]\NP)/(S[ng]\NP) 221 | (NP\NP)/S[dcl] 222 | (S[pss]\NP)/(S[pss]\NP) 223 | (N\N)\(N\N) 224 | (S[dcl]\NP)/S[b] 225 | (S[dcl]\NP)/S[em] 226 | ((S[adj]\NP)/S[em])/(S[adj]\NP) 227 | S/S 228 | ((S/S)\NP)/NP 229 | (S[pss]\NP)/S[dcl] 230 | PR 231 | PP 232 | ((S\NP)/(S\NP))/N 233 | ((S\NP)\(S\NP))/(S[ng]\NP) 234 | PP/(S[dcl]\NP) 235 | ((S[ng]\NP)/(S[to]\NP))/PP 236 | ((S/S)\(S/S))/S[dcl] 237 | ((S[pt]\NP)/PP)/PP 238 | PP/(S[adj]\NP) 239 | (S\S)/S[q] 240 | NP/(S[dcl]/NP) 241 | ((S[pss]\NP)/PP)/(S[adj]\NP) 242 | NP/(N/PP) 243 | (N/N)/NP 244 | ((S[dcl]\NP)/PP)/PP 245 | ((S[dcl]\NP)/PP)/PR 246 | (S[dcl]\(S[pss]\NP))/NP 247 | (S\S)/PP 248 | S 249 | ((S[dcl]\NP)/(S[to]\NP))/PP 250 | (S[q]/(S[adj]\NP))/NP 251 | (S[dcl]\NP)/(S[adj]\NP) 252 | (NP/N)\NP 253 | PP/S[dcl] 254 | ((S\NP)\(S\NP))/(S[to]\NP) 255 | ((S\NP)\(S\NP))/S[dcl] 256 | ((S[pt]\NP)/PP)/PR 257 | (N/PP)/(S[to]\NP) 258 | . 259 | (S[dcl]\NP)/(S[to]\NP) 260 | (S[pt]\NP)/NP 261 | ((S[b]\NP)/(S[ng]\NP))/NP 262 | ((N/PP)\(N/PP))/S[dcl] 263 | (((S[ng]\NP)/PP)/PP)/NP 264 | ((NP\NP)/PP)/NP 265 | N 266 | (S/S)/S[dcl] 267 | (NP\NP)/N 268 | S[inv]/NP 269 | (S\S)/(S[b]\NP) 270 | (NP/NP)/NP 271 | ((S[pt]\NP)/NP)/NP 272 | (S[q]/NP)/NP 273 | ((S[dcl]\NP)/NP)/PR 274 | ((S[dcl]\NP)/NP)/PP 275 | (S[pt]\NP)/(S[to]\NP) 276 | ((S[dcl]\NP)/S[qem])/NP 277 | (S[ng]\NP)/S[for] 278 | (S[b]\NP)/S[dcl] 279 | conj 280 | ((S[dcl]\NP)/S[em])/PP 281 | ((S[dcl]\NP)/S[em])/PR 282 | : 283 | (S[dcl]\NP[expl])/(S[b]\NP) 284 | (S[pss]\NP)/(S[ng]\NP) 285 | ((S\NP)/(S\NP))/S[dcl] 286 | ((S[adj]\NP)/(S[adj]\NP))/N 287 | ((N/PP)\(N/PP))/(S[dcl]\NP) 288 | PP/(S[dcl]/NP) 289 | (N/N)/N 290 | (S[dcl]\NP)/(S[b]\NP) 291 | ((S\NP)/(S\NP))/((S\NP)/(S\NP)) 292 | (N/PP)\(N/PP) 293 | (S[pt]\NP)/PR 294 | (S[pt]\NP)/PP 295 | ((S\NP)\(S\NP))/NP 296 | S[pss]\NP 297 | (S[ng]\NP)/S[em] 298 | ((S[adj]\NP)/(S[to]\NP))/(S[adj]\NP) 299 | (S[adj]\NP)\(S[adj]\NP) 300 | ((S\NP)\(S\NP))/(S[dcl]\NP) 301 | (((S[dcl]\NP)/(S[to]\NP))/(S[adj]\NP))/NP[expl] 302 | (S[dcl]/(S[b]\NP))/NP 303 | (N/(S[to]\NP))/PP 304 | ((S[adj]\NP)\(S[adj]\NP))/NP 305 | ((S[b]\NP)/S[for])/(S[adj]\NP) 306 | ((S[ng]\NP)/S[em])/NP 307 | (S/S)/(S[asup]\NP) 308 | (((S\NP)\(S\NP))/((S\NP)\(S\NP)))/(S[asup]\NP) 309 | RQU 310 | ((S[b]\NP)/PP)/PP 311 | ((S[b]\NP)/PP)/PR 312 | ((N/PP)\(N/PP))/NP 313 | ((S[pss]\NP)/PP)/PR 314 | (S[b]\NP)/(S[ng]\NP) 315 | ((S[pt]\NP)/(S[ng]\NP))/NP 316 | N/S[em] 317 | (S[dcl]\NP)/S[wq] 318 | ((S[dcl]\NP[expl])/S[em])/NP 319 | ((S[dcl]\S[dcl])\NP)/NP 320 | S[wq] 321 | ((S/S)/(S[ng]\NP))/NP 322 | (S/S)/(S/S) 323 | ((S\NP)\(S\NP))\((S\NP)\(S\NP)) 324 | (S[b]\NP)/PP 325 | (S[b]\NP)/PR 326 | ((S[dcl]\NP[expl])/(S[to]\NP))/(S[adj]\NP) 327 | ((N/PP)\(N/PP))/PP 328 | conj\conj 329 | (S[dcl]\NP)/S[bem] 330 | (S/S)/(S[pss]\NP) 331 | (N\N)/(S[dcl]/NP) 332 | (S[b]\NP)/(S[adj]\NP) 333 | (N/PP)/(S[adj]\NP) 334 | S[bem]/S[b] 335 | (S[pss]\NP)/S[qem] 336 | ((S[dcl]\NP[expl])/S[for])/(S[adj]\NP) 337 | (NP/NP)/(NP/NP) 338 | PP\PP 339 | (PP/(S[to]\NP))/NP 340 | LRB 341 | (S/S)/NP 342 | PP/S[qem] 343 | (S[ng]\NP)/(S[pt]\NP) 344 | (S[wq]/(S[q]/NP))/N 345 | (S/S)/(S[ng]\NP) 346 | ((S[pss]\NP)/PP)/PP 347 | ((NP\NP)/S[dcl])\((N\N)/NP) 348 | (((S[dcl]\NP)/PP)/PP)/NP 349 | (PP\PR)/NP 350 | ((S[b]\NP)/(S[to]\NP))/PP 351 | S[to]\NP 352 | S[adj]\NP 353 | ((S[ng]\NP)/NP)/NP 354 | ((N\N)/(S[dcl]\NP))/N 355 | RRB 356 | ((S[dcl]\NP)/NP)/(S[dcl]\NP) 357 | NP 358 | (S\NP)/(S\NP) 359 | ((S/S)/(S[b]\NP))/NP 360 | S\S 361 | (PP\NP)/NP 362 | ((N\N)/PP)/N 363 | (S[ng]\NP)/(S[pss]\NP) 364 | (S[ng]\NP)/NP 365 | S[dcl]/S[inv] 366 | ((S\NP)/(S\NP))/(S[adj]\NP) 367 | (S[q]/(S[ng]\NP))/NP 368 | (((S\NP)\(S\NP))\((S\NP)\(S\NP)))/PP 369 | ((S\NP)\(S\NP))/S[qem] 370 | ((S/S)/(S[pt]\NP))/NP 371 | N\N 372 | (((S[dcl]\NP)/S[em])/(S[adj]\NP))/NP[expl] 373 | (N/PP)/PP 374 | NP/S[dcl] 375 | (NP\NP)/(S[adj]\NP) 376 | ((S[adj]\NP)\(S[adj]\NP))/S[dcl] 377 | (S[for]/(S[to]\NP))/NP 378 | (S\S)/S[dcl] 379 | (S[adj]\NP)/PP 380 | ((S[ng]\NP)/(S[pss]\NP))/NP 381 | S[wq]/S[q] 382 | ((S[b]\NP)/S[dcl])/NP 383 | ((S\NP)/(S\NP))/(S[pss]\NP) 384 | (((S[ng]\NP)/PP)/NP)/PR 385 | (((S\NP)\(S\NP))/(S[to]\NP))/NP 386 | (S[b]\NP)/(S[pss]\NP) 387 | (S[adj]\NP)/(S[to]\NP) 388 | ((S\NP)\(S\NP))/(S[dcl]/NP) 389 | (S[dcl]\NP)/S[for] 390 | (N\N)/(S[dcl]\NP) 391 | ((S\NP)\(S\NP))/(S[pss]\NP) 392 | (S/S)/N 393 | (N\N)/PP 394 | (S[b]\NP)/(S[b]\NP) 395 | conj/conj 396 | ((N\N)/(N\N))/NP 397 | ((S[b]\NP)/PR)/NP 398 | ((S[dcl]\NP[expl])/(S[to]\NP))/NP 399 | S[wq]/(S[dcl]\NP) 400 | (S[ng]\NP)/(S[to]\NP) 401 | -------------------------------------------------------------------------------- /data/orm_data/vocab.bina_rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT ROOT|||ROOT ROOT 6 | ROOT|||*|||* 7 | -------------------------------------------------------------------------------- /data/orm_data/vocab.no-lhs-rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT|||ROOT 6 | ROOT|||ROOT ROOT *|||ROOT ROOT * 7 | ROOT|||*|||* 8 | ROOT|||* ROOT|||* ROOT 9 | ROOT|||* ROOT * ROOT *|||* ROOT * ROOT * 10 | ROOT|||* ROOT * ROOT|||* ROOT * ROOT 11 | ROOT|||* ROOT *|||* ROOT * 12 | ROOT|||* ROOT * ROOT ROOT|||* ROOT * ROOT ROOT 13 | ROOT|||ROOT ROOT|||ROOT ROOT 14 | ROOT|||* ROOT ROOT|||* ROOT ROOT 15 | ROOT|||ROOT ROOT ROOT *|||ROOT ROOT ROOT * 16 | ROOT|||* ROOT * ROOT * ROOT|||* ROOT * ROOT * ROOT 17 | ROOT|||ROOT * ROOT|||ROOT * ROOT 18 | ROOT|||ROOT *|||ROOT * 19 | ROOT|||* ROOT ROOT * ROOT|||* ROOT ROOT * ROOT 20 | ROOT|||ROOT ROOT * ROOT ROOT * ROOT *|||ROOT ROOT * ROOT ROOT * ROOT * 21 | ROOT|||ROOT ROOT ROOT|||ROOT ROOT ROOT 22 | ROOT|||ROOT * ROOT *|||ROOT * ROOT * 23 | ROOT|||* ROOT ROOT ROOT|||* ROOT ROOT ROOT 24 | ROOT|||ROOT ROOT ROOT ROOT *|||ROOT ROOT ROOT ROOT * 25 | ROOT|||ROOT * ROOT ROOT|||ROOT * ROOT ROOT 26 | ROOT|||* ROOT ROOT ROOT *|||* ROOT ROOT ROOT * 27 | ROOT|||* ROOT * ROOT ROOT *|||* ROOT * ROOT ROOT * 28 | ROOT|||ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * 29 | ROOT|||ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT 30 | ROOT|||* ROOT ROOT * ROOT *|||* ROOT ROOT * ROOT * 31 | ROOT|||* ROOT ROOT *|||* ROOT ROOT * 32 | ROOT|||ROOT ROOT ROOT * ROOT *|||ROOT ROOT ROOT * ROOT * 33 | ROOT|||ROOT * ROOT ROOT *|||ROOT * ROOT ROOT * 34 | ROOT|||ROOT ROOT * ROOT *|||ROOT ROOT * ROOT * 35 | ROOT|||* ROOT ROOT ROOT * ROOT|||* ROOT ROOT ROOT * ROOT 36 | ROOT|||ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * 37 | ROOT|||* ROOT * ROOT * ROOT ROOT *|||* ROOT * ROOT * ROOT ROOT * 38 | ROOT|||* ROOT ROOT * ROOT * ROOT * ROOT *|||* ROOT ROOT * ROOT * ROOT * ROOT * 39 | ROOT|||ROOT ROOT ROOT * ROOT ROOT *|||ROOT ROOT ROOT * ROOT ROOT * 40 | ROOT|||ROOT ROOT * ROOT|||ROOT ROOT * ROOT 41 | ROOT|||ROOT ROOT ROOT * ROOT|||ROOT ROOT ROOT * ROOT 42 | ROOT|||ROOT ROOT * ROOT ROOT *|||ROOT ROOT * ROOT ROOT * 43 | ROOT|||ROOT * ROOT ROOT * ROOT|||ROOT * ROOT ROOT * ROOT 44 | ROOT|||* ROOT ROOT * ROOT ROOT *|||* ROOT ROOT * ROOT ROOT * 45 | ROOT|||ROOT * ROOT * ROOT ROOT|||ROOT * ROOT * ROOT ROOT 46 | ROOT|||ROOT * ROOT ROOT ROOT *|||ROOT * ROOT ROOT ROOT * 47 | ROOT|||ROOT * ROOT ROOT * ROOT *|||ROOT * ROOT ROOT * ROOT * 48 | ROOT|||* ROOT ROOT * ROOT * ROOT *|||* ROOT ROOT * ROOT * ROOT * 49 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * 50 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT 51 | ROOT|||* ROOT * ROOT * ROOT *|||* ROOT * ROOT * ROOT * 52 | ROOT|||ROOT ROOT * ROOT * ROOT|||ROOT ROOT * ROOT * ROOT 53 | ROOT|||* ROOT * ROOT * ROOT ROOT|||* ROOT * ROOT * ROOT ROOT 54 | ROOT|||ROOT ROOT * ROOT * ROOT * ROOT *|||ROOT ROOT * ROOT * ROOT * ROOT * 55 | ROOT|||ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT 56 | ROOT|||ROOT * ROOT ROOT ROOT|||ROOT * ROOT ROOT ROOT 57 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT ROOT ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT ROOT ROOT * 58 | ROOT|||ROOT ROOT * ROOT ROOT * ROOT ROOT *|||ROOT ROOT * ROOT ROOT * ROOT ROOT * 59 | ROOT|||* ROOT * ROOT ROOT * ROOT ROOT ROOT *|||* ROOT * ROOT ROOT * ROOT ROOT ROOT * 60 | ROOT|||ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT * ROOT 61 | ROOT|||* ROOT * ROOT ROOT * ROOT ROOT *|||* ROOT * ROOT ROOT * ROOT ROOT * 62 | ROOT|||ROOT ROOT * ROOT ROOT|||ROOT ROOT * ROOT ROOT 63 | ROOT|||ROOT * ROOT * ROOT ROOT *|||ROOT * ROOT * ROOT ROOT * 64 | ROOT|||ROOT * ROOT * ROOT * ROOT ROOT *|||ROOT * ROOT * ROOT * ROOT ROOT * 65 | ROOT|||ROOT * ROOT ROOT ROOT * ROOT *|||ROOT * ROOT ROOT ROOT * ROOT * 66 | ROOT|||ROOT ROOT ROOT * ROOT ROOT * ROOT ROOT *|||ROOT ROOT ROOT * ROOT ROOT * ROOT ROOT * 67 | ROOT|||* ROOT ROOT * ROOT ROOT|||* ROOT ROOT * ROOT ROOT 68 | ROOT|||* ROOT * ROOT ROOT * ROOT * ROOT *|||* ROOT * ROOT ROOT * ROOT * ROOT * 69 | ROOT|||* ROOT ROOT * ROOT ROOT * ROOT * ROOT *|||* ROOT ROOT * ROOT ROOT * ROOT * ROOT * 70 | ROOT|||ROOT * ROOT * ROOT ROOT ROOT *|||ROOT * ROOT * ROOT ROOT ROOT * 71 | ROOT|||ROOT * ROOT ROOT * ROOT ROOT|||ROOT * ROOT ROOT * ROOT ROOT 72 | ROOT|||* ROOT ROOT * ROOT * ROOT|||* ROOT ROOT * ROOT * ROOT 73 | ROOT|||ROOT ROOT ROOT * ROOT * ROOT *|||ROOT ROOT ROOT * ROOT * ROOT * 74 | ROOT|||* ROOT ROOT ROOT * ROOT ROOT|||* ROOT ROOT ROOT * ROOT ROOT 75 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * 76 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT 77 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT 78 | ROOT|||ROOT ROOT * ROOT * ROOT *|||ROOT ROOT * ROOT * ROOT * 79 | ROOT|||* ROOT ROOT ROOT ROOT|||* ROOT ROOT ROOT ROOT 80 | ROOT|||* ROOT * ROOT ROOT * ROOT *|||* ROOT * ROOT ROOT * ROOT * 81 | ROOT|||* ROOT ROOT ROOT * ROOT ROOT *|||* ROOT ROOT ROOT * ROOT ROOT * 82 | ROOT|||ROOT ROOT * ROOT * ROOT ROOT|||ROOT ROOT * ROOT * ROOT ROOT 83 | ROOT|||ROOT ROOT ROOT ROOT * ROOT *|||ROOT ROOT ROOT ROOT * ROOT * 84 | ROOT|||ROOT ROOT ROOT * ROOT ROOT|||ROOT ROOT ROOT * ROOT ROOT 85 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * 86 | ROOT|||* ROOT ROOT * ROOT * ROOT ROOT *|||* ROOT ROOT * ROOT * ROOT ROOT * 87 | ROOT|||* ROOT * ROOT * ROOT * ROOT|||* ROOT * ROOT * ROOT * ROOT 88 | ROOT|||ROOT * ROOT ROOT ROOT ROOT *|||ROOT * ROOT ROOT ROOT ROOT * 89 | ROOT|||* ROOT * ROOT * ROOT * ROOT *|||* ROOT * ROOT * ROOT * ROOT * 90 | ROOT|||* ROOT ROOT ROOT * ROOT *|||* ROOT ROOT ROOT * ROOT * 91 | ROOT|||* ROOT ROOT * ROOT * ROOT ROOT * ROOT *|||* ROOT ROOT * ROOT * ROOT ROOT * ROOT * 92 | ROOT|||ROOT ROOT * ROOT * ROOT ROOT *|||ROOT ROOT * ROOT * ROOT ROOT * 93 | ROOT|||* ROOT ROOT * ROOT ROOT * ROOT *|||* ROOT ROOT * ROOT ROOT * ROOT * 94 | ROOT|||* ROOT ROOT ROOT * ROOT * ROOT * ROOT * ROOT *|||* ROOT ROOT ROOT * ROOT * ROOT * ROOT * ROOT * 95 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT * 96 | ROOT|||* ROOT * ROOT ROOT ROOT ROOT *|||* ROOT * ROOT ROOT ROOT ROOT * 97 | ROOT|||ROOT ROOT * ROOT * ROOT * ROOT|||ROOT ROOT * ROOT * ROOT * ROOT 98 | ROOT|||ROOT * ROOT ROOT * ROOT * ROOT *|||ROOT * ROOT ROOT * ROOT * ROOT * 99 | ROOT|||* ROOT * ROOT ROOT ROOT|||* ROOT * ROOT ROOT ROOT 100 | ROOT|||ROOT ROOT ROOT * ROOT * ROOT * ROOT *|||ROOT ROOT ROOT * ROOT * ROOT * ROOT * 101 | ROOT|||* ROOT * ROOT ROOT ROOT *|||* ROOT * ROOT ROOT ROOT * 102 | ROOT|||* ROOT ROOT ROOT ROOT ROOT * ROOT ROOT ROOT|||* ROOT ROOT ROOT ROOT ROOT * ROOT ROOT ROOT 103 | ROOT|||* ROOT ROOT ROOT ROOT *|||* ROOT ROOT ROOT ROOT * 104 | ROOT|||ROOT * ROOT * ROOT * ROOT ROOT|||ROOT * ROOT * ROOT * ROOT ROOT 105 | ROOT|||* ROOT * ROOT ROOT * ROOT|||* ROOT * ROOT ROOT * ROOT 106 | ROOT|||ROOT ROOT ROOT * ROOT * ROOT * ROOT ROOT *|||ROOT ROOT ROOT * ROOT * ROOT * ROOT ROOT * 107 | ROOT|||ROOT * ROOT ROOT * ROOT ROOT *|||ROOT * ROOT ROOT * ROOT ROOT * 108 | ROOT|||* ROOT ROOT ROOT * ROOT * ROOT|||* ROOT ROOT ROOT * ROOT * ROOT 109 | ROOT|||ROOT ROOT * ROOT ROOT ROOT ROOT ROOT * ROOT ROOT *|||ROOT ROOT * ROOT ROOT ROOT ROOT ROOT * ROOT ROOT * 110 | ROOT|||* ROOT ROOT * ROOT * ROOT * ROOT * ROOT *|||* ROOT ROOT * ROOT * ROOT * ROOT * ROOT * 111 | ROOT|||ROOT ROOT ROOT ROOT * ROOT ROOT *|||ROOT ROOT ROOT ROOT * ROOT ROOT * 112 | ROOT|||* ROOT ROOT * ROOT * ROOT * ROOT|||* ROOT ROOT * ROOT * ROOT * ROOT 113 | ROOT|||* ROOT * ROOT ROOT * ROOT * ROOT ROOT *|||* ROOT * ROOT ROOT * ROOT * ROOT ROOT * 114 | ROOT|||* ROOT ROOT * ROOT ROOT ROOT ROOT *|||* ROOT ROOT * ROOT ROOT ROOT ROOT * 115 | ROOT|||ROOT ROOT ROOT ROOT * ROOT * ROOT ROOT *|||ROOT ROOT ROOT ROOT * ROOT * ROOT ROOT * 116 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * 117 | ROOT|||ROOT ROOT ROOT * ROOT ROOT ROOT *|||ROOT ROOT ROOT * ROOT ROOT ROOT * 118 | ROOT|||ROOT * ROOT * ROOT * ROOT ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT ROOT * ROOT * 119 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT ROOT * 120 | ROOT|||* ROOT * ROOT ROOT * ROOT * ROOT|||* ROOT * ROOT ROOT * ROOT * ROOT 121 | ROOT|||ROOT ROOT ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT ROOT ROOT * ROOT * ROOT * ROOT * ROOT * 122 | ROOT|||* ROOT ROOT * ROOT ROOT ROOT *|||* ROOT ROOT * ROOT ROOT ROOT * 123 | ROOT|||* ROOT * ROOT * ROOT ROOT * ROOT ROOT *|||* ROOT * ROOT * ROOT ROOT * ROOT ROOT * 124 | ROOT|||ROOT ROOT * ROOT ROOT * ROOT|||ROOT ROOT * ROOT ROOT * ROOT 125 | ROOT|||* ROOT ROOT ROOT ROOT ROOT|||* ROOT ROOT ROOT ROOT ROOT 126 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT ROOT * ROOT * ROOT * ROOT * 127 | ROOT|||* ROOT ROOT ROOT ROOT * ROOT|||* ROOT ROOT ROOT ROOT * ROOT 128 | ROOT|||* ROOT ROOT ROOT * ROOT * ROOT *|||* ROOT ROOT ROOT * ROOT * ROOT * 129 | ROOT|||ROOT ROOT ROOT ROOT ROOT * ROOT *|||ROOT ROOT ROOT ROOT ROOT * ROOT * 130 | ROOT|||ROOT * ROOT ROOT * ROOT * ROOT ROOT *|||ROOT * ROOT ROOT * ROOT * ROOT ROOT * 131 | ROOT|||* ROOT * ROOT * ROOT * ROOT ROOT *|||* ROOT * ROOT * ROOT * ROOT ROOT * 132 | ROOT|||* ROOT ROOT * ROOT ROOT * ROOT ROOT *|||* ROOT ROOT * ROOT ROOT * ROOT ROOT * 133 | ROOT|||ROOT ROOT * ROOT ROOT ROOT ROOT ROOT *|||ROOT ROOT * ROOT ROOT ROOT ROOT ROOT * 134 | ROOT|||ROOT ROOT * ROOT * ROOT ROOT * ROOT *|||ROOT ROOT * ROOT * ROOT ROOT * ROOT * 135 | ROOT|||ROOT * ROOT ROOT * ROOT ROOT ROOT|||ROOT * ROOT ROOT * ROOT ROOT ROOT 136 | ROOT|||ROOT * ROOT ROOT * ROOT * ROOT|||ROOT * ROOT ROOT * ROOT * ROOT 137 | ROOT|||ROOT ROOT ROOT ROOT ROOT *|||ROOT ROOT ROOT ROOT ROOT * 138 | ROOT|||* ROOT * ROOT ROOT ROOT ROOT|||* ROOT * ROOT ROOT ROOT ROOT 139 | ROOT|||* ROOT ROOT * ROOT ROOT * ROOT ROOT|||* ROOT ROOT * ROOT ROOT * ROOT ROOT 140 | ROOT|||ROOT * ROOT ROOT ROOT * ROOT * ROOT *|||ROOT * ROOT ROOT ROOT * ROOT * ROOT * 141 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT ROOT *|||ROOT * ROOT * ROOT ROOT * ROOT ROOT * 142 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * 143 | ROOT|||* ROOT * ROOT * ROOT ROOT ROOT|||* ROOT * ROOT * ROOT ROOT ROOT 144 | ROOT|||* ROOT * ROOT * ROOT ROOT * ROOT|||* ROOT * ROOT * ROOT ROOT * ROOT 145 | ROOT|||* ROOT ROOT ROOT ROOT ROOT *|||* ROOT ROOT ROOT ROOT ROOT * 146 | ROOT|||* ROOT ROOT ROOT ROOT * ROOT *|||* ROOT ROOT ROOT ROOT * ROOT * 147 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT 148 | ROOT|||ROOT * ROOT ROOT * ROOT ROOT ROOT *|||ROOT * ROOT ROOT * ROOT ROOT ROOT * 149 | ROOT|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * ROOT * 150 | ROOT|||* ROOT ROOT * ROOT ROOT ROOT|||* ROOT ROOT * ROOT ROOT ROOT 151 | ROOT|||ROOT ROOT ROOT ROOT * ROOT ROOT|||ROOT ROOT ROOT ROOT * ROOT ROOT 152 | ROOT|||ROOT ROOT * ROOT * ROOT * ROOT * ROOT ROOT *|||ROOT ROOT * ROOT * ROOT * ROOT * ROOT ROOT * 153 | ROOT|||ROOT * ROOT * ROOT ROOT ROOT|||ROOT * ROOT * ROOT ROOT ROOT 154 | ROOT|||ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT 155 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT * ROOT|||ROOT * ROOT * ROOT ROOT * ROOT * ROOT * ROOT 156 | ROOT|||ROOT ROOT ROOT * ROOT ROOT * ROOT *|||ROOT ROOT ROOT * ROOT ROOT * ROOT * 157 | ROOT|||ROOT ROOT * ROOT ROOT ROOT|||ROOT ROOT * ROOT ROOT ROOT 158 | ROOT|||ROOT ROOT * ROOT * ROOT * ROOT * ROOT *|||ROOT ROOT * ROOT * ROOT * ROOT * ROOT * 159 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT *|||ROOT ROOT ROOT ROOT ROOT ROOT * 160 | ROOT|||ROOT ROOT ROOT ROOT * ROOT|||ROOT ROOT ROOT ROOT * ROOT 161 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT|||ROOT * ROOT * ROOT ROOT * ROOT 162 | ROOT|||ROOT * ROOT ROOT ROOT ROOT|||ROOT * ROOT ROOT ROOT ROOT 163 | ROOT|||ROOT ROOT * ROOT ROOT ROOT *|||ROOT ROOT * ROOT ROOT ROOT * 164 | ROOT|||* ROOT * ROOT * ROOT * ROOT * ROOT *|||* ROOT * ROOT * ROOT * ROOT * ROOT * 165 | ROOT|||ROOT * ROOT * ROOT ROOT ROOT ROOT *|||ROOT * ROOT * ROOT ROOT ROOT ROOT * 166 | ROOT|||ROOT ROOT ROOT * ROOT ROOT ROOT ROOT *|||ROOT ROOT ROOT * ROOT ROOT ROOT ROOT * 167 | ROOT|||* ROOT ROOT ROOT ROOT ROOT * ROOT * ROOT|||* ROOT ROOT ROOT ROOT ROOT * ROOT * ROOT 168 | ROOT|||ROOT * ROOT * ROOT ROOT * ROOT *|||ROOT * ROOT * ROOT ROOT * ROOT * 169 | ROOT|||* ROOT * ROOT * ROOT ROOT * ROOT *|||* ROOT * ROOT * ROOT ROOT * ROOT * 170 | ROOT|||ROOT ROOT * ROOT * ROOT * ROOT ROOT *|||ROOT ROOT * ROOT * ROOT * ROOT ROOT * 171 | ROOT|||ROOT * ROOT ROOT ROOT * ROOT|||ROOT * ROOT ROOT ROOT * ROOT 172 | -------------------------------------------------------------------------------- /data/orm_data/vocab.phrase_rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT ROOT|||ROOT ROOT 6 | ROOT|||*|||* 7 | -------------------------------------------------------------------------------- /data/orm_data/vocab.pos-no-lhs-rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT|||ROOT 6 | ROOT|||ROOT ROOT ROOT|||ROOT ROOT ROOT 7 | ROOT|||*|||* 8 | ROOT|||ROOT ROOT|||ROOT ROOT 9 | ROOT|||ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT 10 | ROOT|||ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT 11 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT 12 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 13 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 14 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT 15 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 16 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 17 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 18 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 19 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 20 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 21 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 22 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 23 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 24 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 25 | ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT|||ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT ROOT 26 | -------------------------------------------------------------------------------- /data/orm_data/vocab.random_bina_rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT ROOT|||ROOT ROOT 6 | ROOT|||*|||* 7 | -------------------------------------------------------------------------------- /data/orm_data/vocab.right_branch_rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||* ROOT|||* ROOT 6 | ROOT|||*|||* 7 | -------------------------------------------------------------------------------- /data/orm_data/vocab.tri_rule.eng: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ROOT|||ROOT ROOT ROOT|||ROOT ROOT ROOT 6 | ROOT|||ROOT ROOT|||ROOT ROOT 7 | ROOT|||*|||* 8 | -------------------------------------------------------------------------------- /multi-bleu.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | # $Id$ 4 | use strict; 5 | 6 | my $lowercase = 0; 7 | if ($ARGV[0] eq "-lc") { 8 | $lowercase = 1; 9 | shift; 10 | } 11 | 12 | my $stem = $ARGV[0]; 13 | if (!defined $stem) { 14 | print STDERR "usage: multi-bleu.pl [-lc] reference < hypothesis\n"; 15 | print STDERR "Reads the references from reference or reference0, reference1, ...\n"; 16 | exit(1); 17 | } 18 | 19 | $stem .= ".ref" if !-e $stem && !-e $stem."0" && -e $stem.".ref0"; 20 | 21 | my @REF; 22 | my $ref=0; 23 | while(-e "$stem$ref") { 24 | &add_to_ref("$stem$ref",\@REF); 25 | $ref++; 26 | } 27 | &add_to_ref($stem,\@REF) if -e $stem; 28 | die("ERROR: could not find reference file $stem") unless scalar @REF; 29 | 30 | sub add_to_ref { 31 | my ($file,$REF) = @_; 32 | my $s=0; 33 | open(REF,$file) or die "Can't read $file"; 34 | while() { 35 | chop; 36 | push @{$$REF[$s++]}, $_; 37 | } 38 | close(REF); 39 | } 40 | 41 | my(@CORRECT,@TOTAL,$length_translation,$length_reference); 42 | my $s=0; 43 | while() { 44 | chop; 45 | $_ = lc if $lowercase; 46 | my @WORD = split; 47 | my %REF_NGRAM = (); 48 | my $length_translation_this_sentence = scalar(@WORD); 49 | my ($closest_diff,$closest_length) = (9999,9999); 50 | foreach my $reference (@{$REF[$s]}) { 51 | # print "$s $_ <=> $reference\n"; 52 | $reference = lc($reference) if $lowercase; 53 | my @WORD = split(' ',$reference); 54 | my $length = scalar(@WORD); 55 | my $diff = abs($length_translation_this_sentence-$length); 56 | if ($diff < $closest_diff) { 57 | $closest_diff = $diff; 58 | $closest_length = $length; 59 | # print STDERR "$s: closest diff ".abs($length_translation_this_sentence-$length)." = abs($length_translation_this_sentence-$length), setting len: $closest_length\n"; 60 | } elsif ($diff == $closest_diff) { 61 | $closest_length = $length if $length < $closest_length; 62 | # from two references with the same closeness to me 63 | # take the *shorter* into account, not the "first" one. 64 | } 65 | for(my $n=1;$n<=4;$n++) { 66 | my %REF_NGRAM_N = (); 67 | for(my $start=0;$start<=$#WORD-($n-1);$start++) { 68 | my $ngram = "$n"; 69 | for(my $w=0;$w<$n;$w++) { 70 | $ngram .= " ".$WORD[$start+$w]; 71 | } 72 | $REF_NGRAM_N{$ngram}++; 73 | } 74 | foreach my $ngram (keys %REF_NGRAM_N) { 75 | if (!defined($REF_NGRAM{$ngram}) || 76 | $REF_NGRAM{$ngram} < $REF_NGRAM_N{$ngram}) { 77 | $REF_NGRAM{$ngram} = $REF_NGRAM_N{$ngram}; 78 | # print "$i: REF_NGRAM{$ngram} = $REF_NGRAM{$ngram}
\n"; 79 | } 80 | } 81 | } 82 | } 83 | $length_translation += $length_translation_this_sentence; 84 | $length_reference += $closest_length; 85 | for(my $n=1;$n<=4;$n++) { 86 | my %T_NGRAM = (); 87 | for(my $start=0;$start<=$#WORD-($n-1);$start++) { 88 | my $ngram = "$n"; 89 | for(my $w=0;$w<$n;$w++) { 90 | $ngram .= " ".$WORD[$start+$w]; 91 | } 92 | $T_NGRAM{$ngram}++; 93 | } 94 | foreach my $ngram (keys %T_NGRAM) { 95 | $ngram =~ /^(\d+) /; 96 | my $n = $1; 97 | # my $corr = 0; 98 | # print "$i e $ngram $T_NGRAM{$ngram}
\n"; 99 | $TOTAL[$n] += $T_NGRAM{$ngram}; 100 | if (defined($REF_NGRAM{$ngram})) { 101 | if ($REF_NGRAM{$ngram} >= $T_NGRAM{$ngram}) { 102 | $CORRECT[$n] += $T_NGRAM{$ngram}; 103 | # $corr = $T_NGRAM{$ngram}; 104 | # print "$i e correct1 $T_NGRAM{$ngram}
\n"; 105 | } 106 | else { 107 | $CORRECT[$n] += $REF_NGRAM{$ngram}; 108 | # $corr = $REF_NGRAM{$ngram}; 109 | # print "$i e correct2 $REF_NGRAM{$ngram}
\n"; 110 | } 111 | } 112 | # $REF_NGRAM{$ngram} = 0 if !defined $REF_NGRAM{$ngram}; 113 | # print STDERR "$ngram: {$s, $REF_NGRAM{$ngram}, $T_NGRAM{$ngram}, $corr}\n" 114 | } 115 | } 116 | $s++; 117 | } 118 | my $brevity_penalty = 1; 119 | my $bleu = 0; 120 | 121 | my @bleu=(); 122 | 123 | for(my $n=1;$n<=4;$n++) { 124 | if (defined ($TOTAL[$n])){ 125 | $bleu[$n]=($TOTAL[$n])?$CORRECT[$n]/$TOTAL[$n]:0; 126 | # print STDERR "CORRECT[$n]:$CORRECT[$n] TOTAL[$n]:$TOTAL[$n]\n"; 127 | }else{ 128 | $bleu[$n]=0; 129 | } 130 | } 131 | 132 | if ($length_reference==0){ 133 | printf "BLEU = 0, 0/0/0/0 (BP=0, ratio=0, hyp_len=0, ref_len=0)\n"; 134 | exit(1); 135 | } 136 | 137 | if ($length_translation<$length_reference) { 138 | $brevity_penalty = exp(1-$length_reference/$length_translation); 139 | } 140 | $bleu = $brevity_penalty * exp((my_log( $bleu[1] ) + 141 | my_log( $bleu[2] ) + 142 | my_log( $bleu[3] ) + 143 | my_log( $bleu[4] ) ) / 4) ; 144 | printf "BLEU = %.2f, %.1f/%.1f/%.1f/%.1f (BP=%.3f, ratio=%.3f, hyp_len=%d, ref_len=%d)\n", 145 | 100*$bleu, 146 | 100*$bleu[1], 147 | 100*$bleu[2], 148 | 100*$bleu[3], 149 | 100*$bleu[4], 150 | $brevity_penalty, 151 | $length_translation / $length_reference, 152 | $length_translation, 153 | $length_reference; 154 | 155 | sub my_log { 156 | return -9999999999 unless $_[0]; 157 | return log($_[0]); 158 | } 159 | -------------------------------------------------------------------------------- /scripts/orm_trdec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PYTHONPATH="$(pwd)" 3 | export CUDA_VISIBLE_DEVICES="2" 4 | 5 | # The experiment that produce the best performing TrDec in the paper 6 | # Use concatenation of two kinds of synthesized binary tree 7 | 8 | python3.6 src/main.py \ 9 | --trdec \ 10 | --no_lhs \ 11 | --pos=1 \ 12 | --output_dir="outputs_orm_trdec_s0" \ 13 | --log_every=20 \ 14 | --eval_every=200 \ 15 | --reset_output_dir \ 16 | --d_word_vec=512 \ 17 | --d_model=512 \ 18 | --data_path="data/orm_data/" \ 19 | --target_tree_vocab="vocab.bina_rule.eng" \ 20 | --target_word_vocab="vocab.bina_word.eng" \ 21 | --target_tree_train="set0-trainunfilt.tok.eng.bina+w" \ 22 | --target_tree_valid="set0-dev.tok.eng.bina" \ 23 | --target_tree_test="set0-test.tok.eng.bina" \ 24 | --source_train="set0-trainunfilt.tok.piece.orm.bina+w" \ 25 | --target_train="set0-trainunfilt.tok.piece.eng.bina+w" \ 26 | --source_valid="set0-dev.tok.piece.orm" \ 27 | --target_valid="set0-dev.tok.piece.eng" \ 28 | --source_vocab="vocab.orm" \ 29 | --source_test="set0-test.tok.piece.orm" \ 30 | --target_test="set0-test.tok.piece.eng" \ 31 | --batch_size=800 \ 32 | --batcher="word" \ 33 | --n_train_sents=200000 \ 34 | --max_len=2000 \ 35 | --n_train_steps=5000 \ 36 | --cuda \ 37 | "$@" 38 | 39 | 40 | -------------------------------------------------------------------------------- /scripts/orm_trdec_translate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PYTHONPATH="$(pwd)" 3 | export CUDA_VISIBLE_DEVICES="1" 4 | 5 | 6 | python3.6 src/translate.py \ 7 | --trdec \ 8 | --model_dir="outputs_orm_trdec_s0" \ 9 | --data_path="data/orm_data/" \ 10 | --source_vocab="vocab.orm" \ 11 | --target_word_vocab="vocab.bina_word.eng" \ 12 | --target_tree_vocab="vocab.bina_rule.eng" \ 13 | --source_test="set0-test.tok.piece.orm" \ 14 | --target_test="set0-test.tok.piece.eng" \ 15 | --target_tree_test="set0-test.tok.eng.bina" \ 16 | --beam_size=45 \ 17 | --max_len=500 \ 18 | --n_train_sents=10000 \ 19 | --merge_bpe \ 20 | --poly_norm_m=1 \ 21 | --out_file="beam45_m1" \ 22 | --max_tree_len=3000 \ 23 | --no_lhs \ 24 | --pos=1 \ 25 | --cuda \ 26 | "$@" 27 | 28 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cindyxinyiwang/TrDec_pytorch/877bb2a7949828b3da13ac3345a8a26650177697/src/__init__.py -------------------------------------------------------------------------------- /src/clean_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | data_dir = "data/orm_data/" 5 | input_files = ["setE-500.tok.eng", "setE-500.tok.orm"] 6 | output_files = ["setE-500.clean.tok.eng", "setE-500.clean.tok.orm"] 7 | 8 | for i in range(len(input_files)): 9 | input_files[i] = os.path.join(data_dir, input_files[i]) 10 | output_files[i] = os.path.join(data_dir, output_files[i]) 11 | 12 | in_lines_1 = open(input_files[0], 'r').readlines() 13 | in_lines_2 = open(input_files[1], 'r').readlines() 14 | 15 | out_file_1 = open(output_files[0], 'w') 16 | out_file_2 = open(output_files[1], 'w') 17 | 18 | for i1, i2, in zip(in_lines_1, in_lines_2): 19 | if "#untranslated" in i1: 20 | continue 21 | out_file_1.write(i1) 22 | out_file_2.write(i2) 23 | 24 | out_file_1.close() 25 | out_file_2.close() 26 | -------------------------------------------------------------------------------- /src/correct_dep.py: -------------------------------------------------------------------------------- 1 | from tree_utils import * 2 | import sys 3 | 4 | def remove_pre(tree): 5 | if tree.is_preterminal(): 6 | tree.label = '*' 7 | return 8 | for c in tree.children: 9 | if not type(c) == str: 10 | remove_pre(c) 11 | 12 | for line in sys.stdin: 13 | tree = Tree(parse_root(tokenize(line))).root 14 | remove_pre(tree) 15 | print(tree.to_parse_string()) 16 | -------------------------------------------------------------------------------- /src/get_rule_vocab.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from tree_utils import * 3 | from hparams import HParams 4 | 5 | piece_file = "data/orm_data/set0-trainunfilt.tok.piece.eng" 6 | tree_file = "data/orm_data/set0-trainunfilt.tok.eng.dep" 7 | rule_vocab_file = "data/orm_data/vocab.dep_rule.eng" 8 | word_vocab_file = "data/orm_data/vocab.dep_word.eng" 9 | 10 | hp = HParams() 11 | rule_vocab = RuleVocab(hparams=hp, frozen=False) 12 | word_vocab = Vocab(hparams=hp, frozen=False) 13 | 14 | piece_file = open(piece_file, 'r', encoding='utf-8') 15 | tree_file = open(tree_file, 'r', encoding='utf-8') 16 | for piece_line, tree_line in zip(piece_file, tree_file): 17 | tree = Tree(parse_root(tokenize(tree_line))) 18 | #remove_preterminal_POS(tree.root) 19 | #merge_depth(tree.root, 4, 0) 20 | pieces = sent_piece_segs(piece_line) 21 | #print(pieces) 22 | #print(tree.root.to_string()) 23 | split_sent_piece(tree.root, pieces, 0) 24 | add_preterminal_wordswitch(tree.root, add_eos=True) 25 | #remove_lhs(tree.root, 'ROOT') 26 | tree.root.label = "XXX" 27 | tree.reset_timestep() 28 | tree.get_data_root(rule_vocab, word_vocab) 29 | 30 | binarize = False 31 | del_preterm_POS=True 32 | replace_pos=False 33 | read_word=True 34 | merge=False 35 | merge_level=-1 36 | add_eos=True 37 | bpe_post=True 38 | 39 | with open(rule_vocab_file, 'w', encoding='utf-8') as myfile: 40 | for r in rule_vocab: 41 | myfile.write(str(r) + '\n') 42 | 43 | if word_vocab_file: 44 | with open(word_vocab_file, 'w', encoding='utf-8') as myfile: 45 | for w in word_vocab: 46 | myfile.write(w + '\n') 47 | -------------------------------------------------------------------------------- /src/get_vocab.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import argparse 6 | #import cPickle as pickle 7 | import os 8 | import re 9 | import shutil 10 | import sys 11 | import time 12 | 13 | import numpy as np 14 | 15 | DATA_PATH = "data/orm_data/" 16 | INP_NAMES = ["set0-trainunfilt.interleave.null.eng"] 17 | OUT_NAMES = ["vocab.ccg.null.eng"] 18 | 19 | def main(): 20 | for inp_name, out_name in zip(INP_NAMES, OUT_NAMES): 21 | inp_file_name = os.path.join(DATA_PATH, inp_name) 22 | with open(inp_file_name, "r", encoding="utf-8") as finp: 23 | lines = finp.read().split("\n") 24 | 25 | vocab = { 26 | "": 0, 27 | "": 1, 28 | "": 2, 29 | "": 3, 30 | } 31 | num_lines = 0 32 | for line in lines: 33 | line = line.strip() 34 | if not line: 35 | continue 36 | tokens = line.split() 37 | for token in tokens: 38 | if token not in vocab: 39 | index = len(vocab) 40 | vocab[token] = index 41 | num_lines += 1 42 | if num_lines % 50000 == 0: 43 | print("Read {0:>6d} lines".format(num_lines)) 44 | sys.stdout.flush() 45 | print("Read {0:>6d} lines. vocab_size={1}".format(num_lines, len(vocab))) 46 | sys.stdout.flush() 47 | 48 | log_string = "" 49 | for word, idx in vocab.items(): 50 | log_string += "{0} {1}\n".format(word, idx) 51 | 52 | out_name = os.path.join(DATA_PATH, out_name) 53 | print("Saving vocab to '{0}'".format(out_name)) 54 | sys.stdout.flush() 55 | with open(out_name, "w", encoding="utf-8") as fout: 56 | fout.write(log_string) 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /src/hparams.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | 4 | class HParams(object): 5 | def __init__(self, **kwargs): 6 | #super(Iwslt16EnDeBpe32SharedParams, self).__init__(**kwargs) 7 | for name, value in six.iteritems(kwargs): 8 | self.add_param(name, value) 9 | 10 | self.dataset = "Hparams" 11 | 12 | self.unk = "" 13 | self.bos = "" 14 | self.eos = "" 15 | self.pad = "" 16 | 17 | self.unk_id = None 18 | self.eos_id = None 19 | self.bos_id = None 20 | self.pad_id = None 21 | 22 | self.tiny = 0. 23 | self.inf = float("inf") 24 | 25 | 26 | def add_param(self, name, value): 27 | setattr(self, name, value) 28 | -------------------------------------------------------------------------------- /src/make_tree.py: -------------------------------------------------------------------------------- 1 | from tree_utils import * 2 | import argparse 3 | import os 4 | import random 5 | import re 6 | import numpy as np 7 | import math 8 | 9 | def bina_list(node_list, left, right): 10 | if left == right: 11 | return node_list[left] 12 | root = TreeNode("ROOT", []) 13 | mid = int((left + right) / 2) 14 | left = bina_list(node_list, left, mid) 15 | right = bina_list(node_list, mid+1, right) 16 | root.children = [left, right] 17 | return root 18 | 19 | def make_r_binary_tree(word_list, left, right): 20 | ## make if fully binary except for the end of tree 21 | def _bina_list(node_list, left, right): 22 | if left == right: 23 | return node_list[left] 24 | root = TreeNode("ROOT", []) 25 | mid = int((left + right) / 2) 26 | left = _bina_list(node_list, left, mid) 27 | right = _bina_list(node_list, mid+1, right) 28 | root.children = [left, right] 29 | return root 30 | l = len(word_list) 31 | num_preterm = int(pow(2, int(np.log2(l))) / 2) 32 | preterms = [] 33 | for i in range(num_preterm-1): 34 | lc = TreeNode("ROOT", [word_list[i*2]]) 35 | rc = TreeNode("ROOT", [word_list[i*2+1]]) 36 | preterms.append(TreeNode("ROOT", [lc, rc])) 37 | preterms.append(make_binary_tree(word_list, (num_preterm-1)*2, right)) 38 | return _bina_list(preterms, 0, len(preterms)-1) 39 | 40 | def make_w_binary_tree(word_list): 41 | ## first combine words then make trees 42 | l = len(word_list) 43 | nodes = [] 44 | i = 0 45 | while i < l-1: 46 | lc = TreeNode("ROOT", [word_list[i]]) 47 | rc = TreeNode("ROOT", [word_list[i+1]]) 48 | nodes.append(TreeNode("ROOT", [lc, rc])) 49 | i += 2 50 | if l % 2 == 1: 51 | nodes.append(TreeNode("ROOT", [word_list[-1]])) 52 | return bina_list(nodes, 0, len(nodes)-1) 53 | 54 | def make_right_binary_tree(word_list, left, right): 55 | if left == right: 56 | return TreeNode("ROOT", word_list[left]) 57 | root = TreeNode("ROOT", []) 58 | mid = int((left + right) / 2) 59 | if mid == left: mid += 1 60 | left = make_right_binary_tree(word_list, left, mid-1) 61 | right = make_right_binary_tree(word_list, mid, right) 62 | root.children = [left, right] 63 | return root 64 | 65 | def make_binary_tree(word_list, left, right): 66 | if left == right: 67 | return TreeNode("ROOT", word_list[left]) 68 | root = TreeNode("ROOT", []) 69 | mid = int((left + right) / 2) 70 | left = make_binary_tree(word_list, left, mid) 71 | right = make_binary_tree(word_list, mid+1, right) 72 | root.children = [left, right] 73 | return root 74 | 75 | def make_tri_tree(word_list, left, right): 76 | if left == right: 77 | return TreeNode("ROOT", word_list[left]) 78 | if left == right-1: 79 | c1 = TreeNode("ROOT", word_list[left]) 80 | c2 = TreeNode("ROOT", word_list[right]) 81 | return TreeNode("ROOT", [c1, c2]) 82 | root = TreeNode("ROOT", []) 83 | s1 = int((2*left + right) / 3) 84 | s2 = int((left + 2*right) / 3) 85 | c1 = make_tri_tree(word_list, left, s1) 86 | c2 = make_tri_tree(word_list, s1+1, s2) 87 | c3 = make_tri_tree(word_list, s2+1, right) 88 | root.children = [c1, c2, c3] 89 | return root 90 | 91 | def make_random_binary_tree(word_list, left, right): 92 | if left == right: 93 | return TreeNode("ROOT", word_list[left]) 94 | root = TreeNode("ROOT", []) 95 | mid = random.randint(left, right-1) 96 | left = make_binary_tree(word_list, left, mid) 97 | right = make_binary_tree(word_list, mid+1, right) 98 | root.children = [left, right] 99 | return root 100 | 101 | def make_phrase_tree(string): 102 | def _bina_list(node_list, left, right): 103 | if left == right: 104 | return node_list[left] 105 | root = TreeNode("ROOT", []) 106 | mid = int((left + right) / 2) 107 | left = _bina_list(node_list, left, mid) 108 | right = _bina_list(node_list, mid+1, right) 109 | root.children = [left, right] 110 | return root 111 | 112 | #words = re.findall(r"[\w]+|[^\s\w]", string) 113 | #puncs = re.findall(r"[^\s\w]", string) 114 | #print(string) 115 | split_points = [] 116 | for match in re.finditer(r"[\s\w]+", string): 117 | if match.group().strip(): 118 | #print(match.span(), match.group()) 119 | split_points.append(match.span()[0]) 120 | split_points = split_points[1:] 121 | nodes = [] 122 | start = 0 123 | for s in split_points: 124 | cur_str = string[start:s].split() 125 | if string[s] != " ": 126 | if cur_str[-1] == "'": 127 | if string[s] == "s": 128 | continue 129 | else: 130 | cur_str[-1] = cur_str[-1][:-1] 131 | s -= 1 132 | else: 133 | continue 134 | nodes.append(make_binary_tree(cur_str, 0, len(cur_str)-1)) 135 | #print(string[start:s]) 136 | start = s 137 | cur_str = string[start:].split() 138 | nodes.append(make_binary_tree(cur_str, 0, len(cur_str)-1)) 139 | #print(string[start:]) 140 | root = _bina_list(nodes, 0, len(nodes)-1) 141 | return root 142 | 143 | parser = argparse.ArgumentParser(description="build trees") 144 | 145 | parser.add_argument("--data_dir", type=str, help="directory of the data") 146 | parser.add_argument("--file_name",type=str, help="name of the file to parse") 147 | parser.add_argument("--tree_type",type=str, help="[phrase|random_bina|tri|bina|right_branch]") 148 | parser.add_argument("--parse_file_name",type=str, help="name of the file to parse") 149 | 150 | #tree_type = "w_bina" 151 | #data_dir="data/kftt_data/" 152 | #input_files = ["kyoto-train.lower.en", "kyoto-dev.lower.en", "kyoto-test.lower.en"] 153 | #data_dir="data/orm_data/" 154 | #input_files = ["set0-trainunfilt.tok.eng", "set0-dev.tok.eng", "set0-test.tok.eng"] 155 | #input_files = ["debug.tok.eng"] 156 | tree_type = "right_bina" 157 | #data_dir="data/kftt_data/" 158 | #input_files = ["kyoto-train.lower.en", "kyoto-dev.lower.en", "kyoto-test.lower.en"] 159 | #data_dir="data/orm_data/" 160 | #input_files = ["set0-trainunfilt.tok.eng", "set0-dev.tok.eng", "set0-test.tok.eng"] 161 | data_dir="data/de16_data/" 162 | input_files = ["train.tok.en", "tst2013.tok.en", "tst2014.tok.en"] 163 | output_files = [] 164 | for f in input_files: 165 | output_files.append(f + "." + tree_type) 166 | 167 | for in_file, out_file in zip(input_files, output_files): 168 | in_file = os.path.join(data_dir, in_file) 169 | out_file = os.path.join(data_dir, out_file) 170 | print("creating parse file {}".format(out_file)) 171 | out_file = open(out_file, 'w') 172 | with open(in_file, encoding='utf-8') as myfile: 173 | for line in myfile: 174 | words = line.split() 175 | if tree_type == "right_branch": 176 | root = TreeNode("ROOT", []) 177 | c_n = root 178 | i = 0 179 | while i < len(words): 180 | c_n.children.append(TreeNode("*", words[i])) 181 | i += 1 182 | if i < len(words): 183 | n_n = TreeNode("ROOT", []) 184 | c_n.children.append(n_n) 185 | c_n = n_n 186 | out_file.write(root.to_parse_string() + '\n') 187 | elif tree_type == "bina" : 188 | root = make_binary_tree(words, 0, len(words)-1) 189 | out_file.write(root.to_parse_string() + '\n') 190 | elif tree_type == "right_bina": 191 | root = make_right_binary_tree(words, 0, len(words)-1) 192 | out_file.write(root.to_parse_string() + '\n') 193 | elif tree_type == "tri": 194 | root = make_tri_tree(words, 0, len(words)-1) 195 | out_file.write(root.to_parse_string() + '\n') 196 | elif tree_type == "random_bina": 197 | root = make_random_binary_tree(words, 0, len(words)-1) 198 | out_file.write(root.to_parse_string() + '\n') 199 | elif tree_type == "phrase": 200 | root = make_phrase_tree(line) 201 | out_file.write(root.to_parse_string() + '\n') 202 | elif tree_type == "r_bina": 203 | root = make_r_binary_tree(words, 0, len(words)-1) 204 | out_file.write(root.to_parse_string() + '\n') 205 | elif tree_type == "w_bina": 206 | root = make_w_binary_tree(words) 207 | out_file.write(root.to_parse_string() + '\n') 208 | else: 209 | print("Not implemented") 210 | out_file.close() 211 | -------------------------------------------------------------------------------- /src/models.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.init as init 3 | from torch.autograd import Variable 4 | from torch import nn 5 | from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | from utils import * 10 | 11 | class MlpAttn(nn.Module): 12 | def __init__(self, hparams): 13 | super(MlpAttn, self).__init__() 14 | self.hparams = hparams 15 | #self.dropout = nn.Dropout(hparams.dropout) 16 | self.w_trg = nn.Linear(self.hparams.d_model, self.hparams.d_model) 17 | self.w_att = nn.Linear(self.hparams.d_model, 1) 18 | if self.hparams.cuda: 19 | self.w_trg = self.w_trg.cuda() 20 | self.w_att = self.w_att.cuda() 21 | 22 | def forward(self, q, k, v, attn_mask=None): 23 | batch_size, d_q = q.size() 24 | batch_size, len_k, d_k = k.size() 25 | batch_size, len_v, d_v = v.size() 26 | # v is bi-directional encoding of source 27 | assert d_k == d_q 28 | #assert 2*d_k == d_v 29 | assert len_k == len_v 30 | # (batch_size, len_k, d_k) 31 | att_src_hidden = nn.functional.tanh(k + self.w_trg(q).unsqueeze(1)) 32 | # (batch_size, len_k) 33 | att_src_weights = self.w_att(att_src_hidden).squeeze(2) 34 | if not attn_mask is None: 35 | att_src_weights.data.masked_fill_(attn_mask, -self.hparams.inf) 36 | att_src_weights = F.softmax(att_src_weights, dim=-1) 37 | #att_src_weights = self.dropout(att_src_weights) 38 | ctx = torch.bmm(att_src_weights.unsqueeze(1), v).squeeze(1) 39 | return ctx 40 | 41 | 42 | class LayerNormalization(nn.Module): 43 | def __init__(self, d_hid, eps=1e-9): 44 | super(LayerNormalization, self).__init__() 45 | 46 | self.d_hid = d_hid 47 | self.eps = eps 48 | self.scale = nn.Parameter(torch.ones(self.d_hid), requires_grad=True) 49 | self.offset= nn.Parameter(torch.zeros(self.d_hid), requires_grad=True) 50 | 51 | def forward(self, x): 52 | assert x.dim() >= 2 53 | mean = x.mean(dim=-1, keepdim=True) 54 | std = x.std(dim=-1, keepdim=True) 55 | return self.scale * (x - mean) / (std + self.eps) + self.offset 56 | 57 | class DotProdAttn(nn.Module): 58 | def __init__(self, hparams): 59 | super(DotProdAttn, self).__init__() 60 | self.dropout = nn.Dropout(hparams.dropout) 61 | #self.src_enc_linear = nn.Linear(hparams.d_model * 2, hparams.d_model) 62 | self.softmax = nn.Softmax(dim=-1) 63 | self.hparams = hparams 64 | self.temp = np.power(hparams.d_model, 0.5) 65 | 66 | def forward(self, q, k, v, attn_mask = None): 67 | """ 68 | dot prodct attention: (q * k.T) * v 69 | Args: 70 | q: [batch_size, d_q] (target state) 71 | k: [batch_size, len_k, d_k] (source enc key vectors) 72 | v: [batch_size, len_v, d_v] (source encoding vectors) 73 | attn_mask: [batch_size, len_k] (source mask) 74 | Return: 75 | attn: [batch_size, d_v] 76 | """ 77 | batch_size, d_q = q.size() 78 | batch_size, len_k, d_k = k.size() 79 | batch_size, len_v, d_v = v.size() 80 | # v is bi-directional encoding of source 81 | assert d_k == d_q 82 | #assert 2*d_k == d_v 83 | assert len_k == len_v 84 | # [batch_size, len_k, d_model] 85 | #k_vec = self.src_enc_linear(k) 86 | # [batch_size, len_k] 87 | attn_weight = torch.bmm(k, q.unsqueeze(2)).squeeze(2) / self.temp 88 | if not attn_mask is None: 89 | attn_weight.data.masked_fill_(attn_mask, -self.hparams.inf) 90 | attn_weight = self.softmax(attn_weight) 91 | attn_weight = self.dropout(attn_weight) 92 | # [batch_size, d_v] 93 | ctx = torch.bmm(attn_weight.unsqueeze(1), v).squeeze(1) 94 | return ctx 95 | 96 | class MultiHeadAttn(nn.Module): 97 | def __init__(self, hparams): 98 | super(MultiHeadAttn, self).__init__() 99 | 100 | self.hparams = hparams 101 | 102 | self.attention = DotProdAttn(hparams) 103 | self.layer_norm = LayerNormalization(hparams.d_model) 104 | 105 | # projection of concatenated attn 106 | n_heads = self.hparams.n_heads 107 | d_model = self.hparams.d_model 108 | d_q = self.hparams.d_k 109 | d_k = self.hparams.d_k 110 | d_v = self.hparams.d_v 111 | 112 | Q, K, V = [], [], [] 113 | for head_id in range(n_heads): 114 | q = nn.Linear(d_model, d_q, bias=False) 115 | k = nn.Linear(d_model, d_k, bias=False) 116 | v = nn.Linear(d_model, d_v, bias=False) 117 | init_param(q.weight, init_type="uniform", init_range=hparams.init_range) 118 | init_param(k.weight, init_type="uniform", init_range=hparams.init_range) 119 | init_param(v.weight, init_type="uniform", init_range=hparams.init_range) 120 | Q.append(q) 121 | K.append(k) 122 | V.append(v) 123 | self.Q = nn.ModuleList(Q) 124 | self.K = nn.ModuleList(K) 125 | self.V = nn.ModuleList(V) 126 | if self.hparams.cuda: 127 | self.Q = self.Q.cuda() 128 | self.K = self.K.cuda() 129 | self.V = self.V.cuda() 130 | 131 | self.w_proj = nn.Linear(n_heads * d_v, d_model, bias=False) 132 | init_param(self.w_proj.weight, init_type="uniform", init_range=hparams.init_range) 133 | if self.hparams.cuda: 134 | self.w_proj = self.w_proj.cuda() 135 | 136 | def forward(self, q, k, v, attn_mask=None): 137 | """Performs the following computations: 138 | head[i] = Attention(q * w_q[i], k * w_k[i], v * w_v[i]) 139 | outputs = concat(all head[i]) * self.w_proj 140 | Args: 141 | q: [batch_size, len_q, d_q]. 142 | k: [batch_size, len_k, d_k]. 143 | v: [batch_size, len_v, d_v]. 144 | Must have: len_k == len_v 145 | Note: This batch_size is in general NOT the training batch_size, as 146 | both sentences and time steps are batched together for efficiency. 147 | Returns: 148 | outputs: [batch_size, len_q, d_model]. 149 | """ 150 | 151 | residual = q 152 | 153 | n_heads = self.hparams.n_heads 154 | d_model = self.hparams.d_model 155 | d_q = self.hparams.d_k 156 | d_k = self.hparams.d_k 157 | d_v = self.hparams.d_v 158 | batch_size = q.size(0) 159 | 160 | heads = [] 161 | for Q, K, V in zip(self.Q, self.K, self.V): 162 | head_q, head_k, head_v = Q(q), K(k), V(v) 163 | head = self.attention(head_q, head_k, head_v, attn_mask=attn_mask) 164 | heads.append(head) 165 | 166 | outputs = torch.cat(heads, dim=-1).contiguous().view(batch_size, n_heads * d_v) 167 | outputs = self.w_proj(outputs) 168 | if not hasattr(self.hparams, "residue") or self.hparams.residue == 1: 169 | outputs = outputs + residual 170 | if not hasattr(self.hparams, "layer_norm") or self.hparams.layer_norm == 1: 171 | outputs = self.layer_norm(outputs) 172 | 173 | return outputs 174 | 175 | class Encoder(nn.Module): 176 | def __init__(self, hparams, *args, **kwargs): 177 | super(Encoder, self).__init__() 178 | 179 | self.hparams = hparams 180 | #print("d_word_vec", self.hparams.d_word_vec) 181 | self.word_emb = nn.Embedding(self.hparams.source_vocab_size, 182 | self.hparams.d_word_vec, 183 | padding_idx=hparams.pad_id) 184 | 185 | self.layer = nn.LSTM(self.hparams.d_word_vec, 186 | self.hparams.d_model, 187 | bidirectional=True, 188 | dropout=hparams.dropout) 189 | 190 | # bridge from encoder state to decoder init state 191 | self.bridge = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 192 | 193 | self.dropout = nn.Dropout(self.hparams.dropout) 194 | if self.hparams.cuda: 195 | self.word_emb = self.word_emb.cuda() 196 | self.layer = self.layer.cuda() 197 | self.dropout = self.dropout.cuda() 198 | self.bridge = self.bridge.cuda() 199 | 200 | def forward(self, x_train, x_len): 201 | """Performs a forward pass. 202 | 203 | Args: 204 | x_train: Torch Tensor of size [batch_size, max_len] 205 | x_mask: Torch Tensor of size [batch_size, max_len]. 1 means to ignore a 206 | position. 207 | x_len: [batch_size,] 208 | 209 | Returns: 210 | enc_output: Tensor of size [batch_size, max_len, d_model]. 211 | """ 212 | batch_size, max_len = x_train.size() 213 | #print("x_train", x_train) 214 | #print("x_len", x_len) 215 | x_train = x_train.transpose(0, 1) 216 | # [batch_size, max_len, d_word_vec] 217 | word_emb = self.word_emb(x_train) 218 | word_emb = self.dropout(word_emb) 219 | packed_word_emb = pack_padded_sequence(word_emb, x_len) 220 | enc_output, (ht, ct) = self.layer(packed_word_emb) 221 | enc_output, _ = pad_packed_sequence(enc_output, padding_value=self.hparams.pad_id) 222 | enc_output = enc_output.permute(1, 0, 2) 223 | 224 | dec_init_cell = self.bridge(torch.cat([ct[0], ct[1]], 1)) 225 | dec_init_state = F.tanh(dec_init_cell) 226 | dec_init = (dec_init_state, dec_init_cell) 227 | return enc_output, dec_init 228 | 229 | class Decoder(nn.Module): 230 | def __init__(self, hparams): 231 | super(Decoder, self).__init__() 232 | self.hparams = hparams 233 | 234 | #self.attention = DotProdAttn(hparams) 235 | self.attention = MlpAttn(hparams) 236 | # transform [ctx, h_t] to readout state vectors before softmax 237 | self.ctx_to_readout = nn.Linear(hparams.d_model * 2 + hparams.d_model, hparams.d_model, bias=False) 238 | self.readout = nn.Linear(hparams.d_model, hparams.target_vocab_size, bias=False) 239 | self.word_emb = nn.Embedding(self.hparams.target_vocab_size, 240 | self.hparams.d_word_vec, 241 | padding_idx=hparams.pad_id) 242 | # input: [y_t-1, input_feed] 243 | self.layer = nn.LSTMCell(hparams.d_word_vec + hparams.d_model * 2, 244 | hparams.d_model) 245 | self.dropout = nn.Dropout(hparams.dropout) 246 | if self.hparams.cuda: 247 | self.ctx_to_readout = self.ctx_to_readout.cuda() 248 | self.readout = self.readout.cuda() 249 | self.word_emb = self.word_emb.cuda() 250 | self.layer = self.layer.cuda() 251 | self.dropout = self.dropout.cuda() 252 | 253 | def forward(self, x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask): 254 | # get decoder init state and cell, use x_ct 255 | """ 256 | x_enc: [batch_size, max_x_len, d_model * 2] 257 | 258 | """ 259 | batch_size_x = x_enc.size()[0] 260 | batch_size, y_max_len = y_train.size() 261 | assert batch_size_x == batch_size 262 | hidden = dec_init 263 | input_feed = Variable(torch.zeros(batch_size, self.hparams.d_model * 2), requires_grad=False) 264 | #input_feed = Variable(dec_init[1][1].data.new(batch_size, self.hparams.d_model).zero_(), requires_grad=False) 265 | if self.hparams.cuda: 266 | input_feed = input_feed.cuda() 267 | # [batch_size, y_len, d_word_vec] 268 | trg_emb = self.word_emb(y_train) 269 | pre_readouts = [] 270 | logits = [] 271 | for t in range(y_max_len): 272 | y_emb_tm1 = trg_emb[:, t, :] 273 | y_input = torch.cat([y_emb_tm1, input_feed], dim=1) 274 | 275 | h_t, c_t = self.layer(y_input, hidden) 276 | ctx = self.attention(h_t, x_enc_k, x_enc, attn_mask=x_mask) 277 | pre_readout = F.tanh(self.ctx_to_readout(torch.cat([h_t, ctx], dim=1))) 278 | pre_readout = self.dropout(pre_readout) 279 | pre_readouts.append(pre_readout) 280 | 281 | input_feed = ctx 282 | hidden = (h_t, c_t) 283 | # [len_y, batch_size, trg_vocab_size] 284 | logits = self.readout(torch.stack(pre_readouts)).transpose(0, 1).contiguous() 285 | return logits 286 | 287 | def step(self, x_enc, x_enc_k, y_tm1, dec_state, ctx_t): 288 | y_emb_tm1 = self.word_emb(y_tm1) 289 | y_input = torch.cat([y_emb_tm1, ctx_t], dim=1) 290 | #print (y_input.size()) 291 | #print (dec_state[0].size()) 292 | #print (dec_state[1].size()) 293 | h_t, c_t = self.layer(y_input, dec_state) 294 | ctx = self.attention(h_t, x_enc_k, x_enc) 295 | pre_readout = F.tanh(self.ctx_to_readout(torch.cat([h_t, ctx], dim=1))) 296 | logits = self.readout(pre_readout) 297 | 298 | return logits, (h_t, c_t), ctx 299 | 300 | class Seq2Seq(nn.Module): 301 | 302 | def __init__(self, hparams): 303 | super(Seq2Seq, self).__init__() 304 | self.encoder = Encoder(hparams) 305 | self.decoder = Decoder(hparams) 306 | # transform encoder state vectors into attention key vector 307 | self.enc_to_k = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 308 | self.hparams = hparams 309 | if self.hparams.cuda: 310 | self.enc_to_k = self.enc_to_k.cuda() 311 | 312 | def forward(self, x_train, x_mask, x_len, y_train, y_mask, y_len): 313 | # [batch_size, x_len, d_model * 2] 314 | #print("x_train", x_train) 315 | #print("x_mask", x_mask) 316 | #print("x_len", x_len) 317 | x_enc, dec_init = self.encoder(x_train, x_len) 318 | x_enc_k = self.enc_to_k(x_enc) 319 | # [batch_size, y_len-1, trg_vocab_size] 320 | logits = self.decoder(x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask) 321 | return logits 322 | 323 | def translate(self, x_train, max_len=100, beam_size=5, poly_norm_m=0): 324 | hyps = [] 325 | for x in x_train: 326 | x = Variable(torch.LongTensor(x), volatile=True) 327 | if self.hparams.cuda: 328 | x = x.cuda() 329 | hyp = self.translate_sent(x, max_len=max_len, beam_size=beam_size, poly_norm_m=poly_norm_m)[0] 330 | hyps.append(hyp.y[1:-1]) 331 | return hyps 332 | 333 | def translate_sent(self, x_train, max_len=100, beam_size=5, poly_norm_m=0): 334 | assert len(x_train.size()) == 1 335 | x_len = [x_train.size(0)] 336 | x_train = x_train.unsqueeze(0) 337 | x_enc, dec_init = self.encoder(x_train, x_len) 338 | x_enc_k = self.enc_to_k(x_enc) 339 | length = 0 340 | completed_hyp = [] 341 | input_feed = Variable(torch.zeros(1, self.hparams.d_model * 2), requires_grad=False) 342 | if self.hparams.cuda: 343 | input_feed = input_feed.cuda() 344 | active_hyp = [Hyp(state=dec_init, y=[self.hparams.bos_id], ctx_tm1=input_feed, score=0.)] 345 | while len(completed_hyp) < beam_size and length < max_len: 346 | length += 1 347 | new_hyp_score_list = [] 348 | #hyp_num = len(active_hyp) 349 | #cur_x_enc = x_enc.repeat(hyp_num, 1, 1) 350 | #cur_x_enc_k = x_enc_k.repeat(hyp_num, 1, 1) 351 | #y_tm1 = Variable(torch.LongTensor([hyp.y[-1] for hyp in active_hyp]), volatile=True) 352 | #if self.hparams.cuda: 353 | # y_tm1 = y_tm1.cuda() 354 | #logits = self.decoder.step(cur_x_enc, cur_x_enc_k, y_tm1, ) 355 | for i, hyp in enumerate(active_hyp): 356 | y_tm1 = Variable(torch.LongTensor([int(hyp.y[-1])] ), volatile=True) 357 | if self.hparams.cuda: 358 | y_tm1 = y_tm1.cuda() 359 | logits, dec_state, ctx = self.decoder.step(x_enc, x_enc_k, y_tm1, hyp.state, hyp.ctx_tm1) 360 | hyp.state = dec_state 361 | hyp.ctx_tm1 = ctx 362 | 363 | p_t = F.log_softmax(logits, -1).data 364 | if poly_norm_m > 0 and length > 1: 365 | new_hyp_scores = (hyp.score * pow(length-1, poly_norm_m) + p_t) / pow(length, poly_norm_m) 366 | else: 367 | new_hyp_scores = hyp.score + p_t 368 | #print(new_hyp_scores) 369 | #print(p_t) 370 | new_hyp_score_list.append(new_hyp_scores) 371 | #print(hyp.y) 372 | #print(dec_state) 373 | #if len(active_hyp) > i+1: 374 | # print(active_hyp[i+1].state) 375 | #print() 376 | #exit(0) 377 | live_hyp_num = beam_size - len(completed_hyp) 378 | new_hyp_scores = np.concatenate(new_hyp_score_list).flatten() 379 | new_hyp_pos = (-new_hyp_scores).argsort()[:live_hyp_num] 380 | prev_hyp_ids = new_hyp_pos / self.hparams.target_vocab_size 381 | word_ids = new_hyp_pos % self.hparams.target_vocab_size 382 | new_hyp_scores = new_hyp_scores[new_hyp_pos] 383 | 384 | new_hypotheses = [] 385 | for prev_hyp_id, word_id, hyp_score in zip(prev_hyp_ids, word_ids, new_hyp_scores): 386 | prev_hyp = active_hyp[int(prev_hyp_id)] 387 | hyp = Hyp(state=prev_hyp.state, y=prev_hyp.y+[word_id], ctx_tm1=prev_hyp.ctx_tm1, score=hyp_score) 388 | if word_id == self.hparams.eos_id: 389 | completed_hyp.append(hyp) 390 | else: 391 | new_hypotheses.append(hyp) 392 | #print(word_id, hyp_score) 393 | #exit(0) 394 | active_hyp = new_hypotheses 395 | 396 | if len(completed_hyp) == 0: 397 | completed_hyp.append(active_hyp[0]) 398 | return sorted(completed_hyp, key=lambda x: x.score, reverse=True) 399 | 400 | class Hyp(object): 401 | def __init__(self, state, y, ctx_tm1, score): 402 | self.state = state 403 | self.y = y 404 | self.ctx_tm1 = ctx_tm1 405 | self.score = score 406 | -------------------------------------------------------------------------------- /src/null_ccg.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | #ccg_tag_file = "data/orm_data/set0.tag" 5 | #input_files = ["data/orm_data/set0-dev.interleave.eng"] 6 | #output_files = ["data/orm_data/set0-dev.interleave.null.eng"] 7 | 8 | ccg_tag_file = "data/kftt_data/kyoto.tag" 9 | input_files = ["data/kftt_data/kyoto-train.interleave.en.filt", "data/kftt_data/kyoto-dev.interleave.en"] 10 | output_files = ["data/kftt_data/kyoto-train.interleave.null.en.filt", "data/kftt_data/kyoto-dev.interleave.null.en"] 11 | 12 | 13 | ccg_tags = [] 14 | with open(ccg_tag_file, 'r') as tag_file: 15 | for line in tag_file: 16 | ccg_tags.append(line.strip()) 17 | rep_tag = ccg_tags[0] 18 | ccg_tags = set(ccg_tags) 19 | 20 | for infile, outfile in zip(input_files, output_files): 21 | infile = open(infile, 'r', encoding='utf-8') 22 | outfile = open(outfile, 'w', encoding='utf-8') 23 | for line in infile: 24 | toks = line.split() 25 | for i in range(len(toks)): 26 | if toks[i] in ccg_tags: 27 | toks[i] = rep_tag 28 | outfile.write(' '.join(toks) + '\n') 29 | 30 | infile.close() 31 | outfile.close() 32 | -------------------------------------------------------------------------------- /src/translate.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import argparse 6 | import _pickle as pickle 7 | import shutil 8 | import gc 9 | import os 10 | import sys 11 | import time 12 | 13 | import numpy as np 14 | 15 | from data_utils import DataLoader 16 | from hparams import * 17 | from utils import * 18 | from models import * 19 | from trdec import * 20 | from trdec_single import * 21 | from trdec_attn import * 22 | from tree_utils import * 23 | 24 | import torch 25 | import torch.nn as nn 26 | from torch.autograd import Variable 27 | 28 | class TranslationHparams(HParams): 29 | dataset = "Translate dataset" 30 | 31 | parser = argparse.ArgumentParser(description="Neural MT translator") 32 | 33 | add_argument(parser, "trdec", type="bool", default=False, help="whether to use tree model or not") 34 | add_argument(parser, "root_label", type="str", default="ROOT", help="name of the nonterminal to start a tree") 35 | add_argument(parser, "cuda", type="bool", default=False, help="GPU or not") 36 | add_argument(parser, "data_path", type="str", default=None, help="path to all data") 37 | add_argument(parser, "model_dir", type="str", default="outputs", help="root directory of saved model") 38 | add_argument(parser, "source_test", type="str", default=None, help="name of source test file") 39 | add_argument(parser, "target_test", type="str", default=None, help="name of target test file") 40 | add_argument(parser, "target_tree_test", type="str", default=None, help="name of target test parse file") 41 | add_argument(parser, "beam_size", type="int", default=None, help="beam size") 42 | add_argument(parser, "max_len", type="int", default=300, help="maximum len considered on the target side") 43 | add_argument(parser, "max_tree_len", type="int", default=1000, help="maximum len of rule derivations considered on the target side") 44 | add_argument(parser, "poly_norm_m", type="float", default=0, help="m in polynormial normalization") 45 | add_argument(parser, "non_batch_translate", type="bool", default=False, help="use non-batched translation") 46 | add_argument(parser, "batch_size", type="int", default=1, help="") 47 | add_argument(parser, "merge_bpe", type="bool", default=True, help="") 48 | add_argument(parser, "source_vocab", type="str", default=None, help="name of source vocab file") 49 | add_argument(parser, "target_vocab", type="str", default=None, help="name of target vocab file") 50 | add_argument(parser, "target_tree_vocab", type="str", default=None, help="name of target vocab file") 51 | add_argument(parser, "target_word_vocab", type="str", default=None, help="name of target vocab file") 52 | add_argument(parser, "n_train_sents", type="int", default=None, help="max number of training sentences to load") 53 | add_argument(parser, "out_file", type="str", default="trans", help="output file for hypothesis") 54 | add_argument(parser, "debug", type="bool", default=False, help="output file for hypothesis") 55 | add_argument(parser, "ccg_tag_file", type="str", default=None, help="name of the file that contains ccg tags to be filtered") 56 | 57 | add_argument(parser, "max_tree_depth", type="int", default=0, help="") 58 | add_argument(parser, "no_lhs", type="bool", default=False, help="whether to use no lhs rules") 59 | add_argument(parser, "pos", type="int", default=0, help="whether to keep pos tag. 0 if remove pos tag [1|0]") 60 | add_argument(parser, "ignore_rule_len", type="bool", default=False, help="whether to ignore rules when doing length norm") 61 | add_argument(parser, "nbest", type="bool", default=False, help="whether to return the nbest list") 62 | add_argument(parser, "force_rule", type="bool", default=False, help="whether to force rule selection for the first timestep") 63 | add_argument(parser, "force_rule_step", type="int", default=1, help="the depth to force rule") 64 | args = parser.parse_args() 65 | 66 | model_file_name = os.path.join(args.model_dir, "model.pt") 67 | if not args.cuda: 68 | model = torch.load(model_file_name, map_location=lambda storage, loc: storage) 69 | else: 70 | model = torch.load(model_file_name) 71 | model.eval() 72 | 73 | out_file = os.path.join(args.model_dir, args.out_file) 74 | if args.trdec: 75 | out_parse_file = os.path.join(args.model_dir, args.out_file+".parse") 76 | print("writing translation to " + out_file) 77 | hparams = TranslationHparams( 78 | data_path=args.data_path, 79 | source_vocab=args.source_vocab, 80 | target_vocab=args.target_vocab, 81 | source_test = args.source_test, 82 | target_test = args.target_test, 83 | target_tree_test = args.target_tree_test, 84 | cuda=args.cuda, 85 | beam_size=args.beam_size, 86 | max_len=args.max_len, 87 | max_tree_len=args.max_tree_len, 88 | batch_size=args.batch_size, 89 | n_train_sents=args.n_train_sents, 90 | merge_bpe=args.merge_bpe, 91 | out_file=out_file, 92 | trdec=args.trdec, 93 | target_tree_vocab=args.target_tree_vocab, 94 | target_word_vocab=args.target_word_vocab, 95 | max_tree_depth=args.max_tree_depth, 96 | no_lhs=args.no_lhs, 97 | root_label=args.root_label, 98 | pos=args.pos, 99 | ignore_rule_len=args.ignore_rule_len, 100 | nbest=args.nbest, 101 | force_rule=args.force_rule, 102 | force_rule_step=args.force_rule_step, 103 | ) 104 | 105 | hparams.add_param("pad_id", model.hparams.pad_id) 106 | hparams.add_param("bos_id", model.hparams.bos_id) 107 | hparams.add_param("eos_id", model.hparams.eos_id) 108 | hparams.add_param("unk_id", model.hparams.unk_id) 109 | model.hparams.cuda = hparams.cuda 110 | if not hasattr(model.hparams, "parent_feed"): 111 | model.hparams.parent_feed = 1 112 | if not hasattr(model.hparams, "rule_parent_feed"): 113 | model.hparams.rule_parent_feed = 1 114 | model.hparams.root_label = args.root_label 115 | model.hparams.ignore_rule_len = args.ignore_rule_len 116 | model.hparams.nbest = args.nbest 117 | model.hparams.force_rule = args.force_rule 118 | model.hparams.force_rule_step = args.force_rule_step 119 | 120 | data = DataLoader(hparams=hparams, decode=True) 121 | filts = [model.hparams.pad_id, model.hparams.eos_id, model.hparams.bos_id] 122 | if args.ccg_tag_file: 123 | ccg_tag_file = os.path.join(args.data_path, args.ccg_tag_file) 124 | with open(ccg_tag_file, 'r') as tag_file: 125 | for line in tag_file: 126 | line = line.strip() 127 | if line in data.target_word_to_index: 128 | f_id = data.target_word_to_index[line] 129 | filts.append(f_id) 130 | hparams.add_param("filtered_tokens", set(filts)) 131 | if args.debug: 132 | hparams.add_param("target_word_vocab_size", data.target_word_vocab_size) 133 | hparams.add_param("target_rule_vocab_size", data.target_rule_vocab_size) 134 | crit = get_criterion(hparams) 135 | 136 | out_file = open(hparams.out_file, 'w', encoding='utf-8') 137 | if args.trdec: 138 | out_parse_file = open(out_parse_file, 'w', encoding='utf-8') 139 | 140 | end_of_epoch = False 141 | num_sentences = 0 142 | 143 | x_test = data.x_test.tolist() 144 | if args.debug: 145 | y_test = data.y_test.tolist() 146 | else: 147 | y_test = None 148 | #print(x_test) 149 | if args.trdec: 150 | hyps, scores = model.translate( 151 | x_test, target_rule_vocab=data.target_tree_vocab, 152 | beam_size=args.beam_size, max_len=args.max_len, y_label=y_test, poly_norm_m=args.poly_norm_m) 153 | else: 154 | hyps = model.translate( 155 | x_test, beam_size=args.beam_size, max_len=args.max_len, poly_norm_m=args.poly_norm_m) 156 | 157 | if args.debug: 158 | forward_scores = [] 159 | while not end_of_epoch: 160 | ((x_test, x_mask, x_len, x_count), 161 | (y_test, y_mask, y_len, y_count), 162 | batch_size, end_of_epoch) = data.next_test(test_batch_size=hparams.batch_size, sort_by_x=True) 163 | 164 | num_sentences += batch_size 165 | logits = model.forward(x_test, x_mask, x_len, y_test[:,:-1,:], y_mask[:,:-1], y_len, y_test[:,1:,2]) 166 | logits = logits.view(-1, hparams.target_rule_vocab_size+hparams.target_word_vocab_size) 167 | labels = y_test[:,1:,0].contiguous().view(-1) 168 | val_loss, val_acc, rule_loss, word_loss, eos_loss, rule_count, word_count, eos_count = \ 169 | get_performance(crit, logits, labels, hparams, sum_loss=False) 170 | print("train forward:", val_loss.data) 171 | print("train label:", labels.data) 172 | logit_score = [] 173 | for i,l in enumerate(labels): logit_score.append(logits[i][l].data[0]) 174 | print("train_logit", logit_score) 175 | #print("train_label", labels) 176 | forward_scores.append(val_loss.sum().data[0]) 177 | # The normal, correct way: 178 | #hyps = model.translate( 179 | # x_test, x_len, beam_size=args.beam_size, max_len=args.max_len) 180 | # For debugging: 181 | # model.debug_translate_batch( 182 | # x_test, x_mask, x_pos_emb_indices, hparams.beam_size, hparams.max_len, 183 | # y_test, y_mask, y_pos_emb_indices) 184 | # sys.exit(0) 185 | print("translate_score:", sum(scores)) 186 | print("forward_score:", sum(forward_scores)) 187 | exit(0) 188 | 189 | if args.nbest: 190 | for h_list in hyps: 191 | for h in h_list: 192 | if args.trdec: 193 | deriv = [] 194 | for w in h: 195 | if w < data.target_word_vocab_size: 196 | deriv.append([data.target_word_vocab[w], False]) 197 | else: 198 | deriv.append([data.target_tree_vocab[w], False]) 199 | tree = Tree.from_rule_deriv(deriv) 200 | line = tree.to_string() 201 | if hparams.merge_bpe: 202 | line = line.replace(' ', '') 203 | line = line.replace('▁', ' ').strip() 204 | out_file.write(line + '\n') 205 | out_file.flush() 206 | out_parse_file.write(tree.to_parse_string() + '\n') 207 | out_parse_file.flush() 208 | else: 209 | h_best_words = map(lambda wi: data.target_index_to_word[wi], 210 | filter(lambda wi: wi not in hparams.filtered_tokens, h)) 211 | if hparams.merge_bpe: 212 | line = ''.join(h_best_words) 213 | line = line.replace('▁', ' ') 214 | else: 215 | line = ' '.join(h_best_words) 216 | line = line.strip() 217 | out_file.write(line + '\n') 218 | out_file.flush() 219 | out_file.write('\n') 220 | if args.trdec: 221 | out_parse_file.write('\n') 222 | else: 223 | for h in hyps: 224 | if args.trdec: 225 | deriv = [] 226 | for w in h: 227 | if w < data.target_word_vocab_size: 228 | deriv.append([data.target_word_vocab[w], False]) 229 | else: 230 | deriv.append([data.target_tree_vocab[w], False]) 231 | tree = Tree.from_rule_deriv(deriv) 232 | line = tree.to_string() 233 | if hparams.merge_bpe: 234 | line = line.replace(' ', '') 235 | line = line.replace('▁', ' ').strip() 236 | out_file.write(line + '\n') 237 | out_file.flush() 238 | out_parse_file.write(tree.to_parse_string() + '\n') 239 | out_parse_file.flush() 240 | else: 241 | h_best_words = map(lambda wi: data.target_index_to_word[wi], 242 | filter(lambda wi: wi not in hparams.filtered_tokens, h)) 243 | if hparams.merge_bpe: 244 | line = ''.join(h_best_words) 245 | line = line.replace('▁', ' ') 246 | else: 247 | line = ' '.join(h_best_words) 248 | line = line.strip() 249 | out_file.write(line + '\n') 250 | out_file.flush() 251 | 252 | print("Translated {0} sentences".format(num_sentences)) 253 | sys.stdout.flush() 254 | 255 | out_file.close() 256 | if args.trdec: 257 | out_parse_file.close() 258 | -------------------------------------------------------------------------------- /src/trdec.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.init as init 3 | from torch.autograd import Variable 4 | from torch import nn 5 | from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | from utils import * 10 | from models import * 11 | import gc 12 | 13 | class TreeDecoder(nn.Module): 14 | def __init__(self, hparams): 15 | super(TreeDecoder, self).__init__() 16 | self.hparams = hparams 17 | self.target_vocab_size = self.hparams.target_word_vocab_size+self.hparams.target_rule_vocab_size 18 | self.emb = nn.Embedding(self.target_vocab_size, 19 | self.hparams.d_word_vec, 20 | padding_idx=hparams.pad_id) 21 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 22 | if self.hparams.attn == "mlp": 23 | self.attention = MlpAttn(hparams) 24 | else: 25 | self.attention = DotProdAttn(hparams) 26 | else: 27 | if self.hparams.attn == "mlp": 28 | self.rule_attention = MlpAttn(hparams) 29 | self.word_attention = MlpAttn(hparams) 30 | else: 31 | self.rule_attention = DotProdAttn(hparams) 32 | self.word_attention = DotProdAttn(hparams) 33 | # transform [word_ctx, word_h_t, rule_ctx, rule_h_t] to readout state vectors before softmax 34 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 35 | inp_dim = hparams.d_model * 3 36 | else: 37 | inp_dim = hparams.d_model * 6 38 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 39 | self.ctx_to_readout = nn.Linear(inp_dim, hparams.d_model, bias=False) 40 | else: 41 | self.rule_ctx_to_readout = nn.Linear(inp_dim, hparams.d_model, bias=False) 42 | self.word_ctx_to_readout = nn.Linear(hparams.d_model * 6, hparams.d_model, bias=False) 43 | 44 | self.readout = nn.Linear(hparams.d_model, 45 | self.target_vocab_size, 46 | bias=False) 47 | if self.hparams.share_emb_softmax: 48 | self.emb.weight = self.readout.weight 49 | # input: [y_t-1, parent_state, input_feed, word_state] 50 | rule_inp = hparams.d_word_vec + hparams.d_model * 3 51 | word_inp = hparams.d_word_vec + hparams.d_model * 2 52 | #if hparams.parent_feed: 53 | word_inp += hparams.d_model 54 | if hparams.rule_parent_feed: 55 | rule_inp += hparams.d_model 56 | self.rule_lstm_cell = nn.LSTMCell(rule_inp, hparams.d_model) 57 | # input: [y_t-1, parent_state, input_feed] 58 | self.word_lstm_cell = nn.LSTMCell(word_inp, hparams.d_model) 59 | self.dropout = nn.Dropout(hparams.dropout) 60 | 61 | vocab_mask = torch.zeros(1, 1, self.target_vocab_size) 62 | self.word_vocab_mask = vocab_mask.index_fill_(2, torch.arange(self.hparams.target_word_vocab_size).long(), 1) 63 | self.rule_vocab_mask = 1 - self.word_vocab_mask 64 | if self.hparams.cuda: 65 | self.rule_vocab_mask = self.rule_vocab_mask.cuda() 66 | self.word_vocab_mask = self.word_vocab_mask.cuda() 67 | self.emb = self.emb.cuda() 68 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 69 | self.attention = self.attention.cuda() 70 | else: 71 | self.rule_attention = self.rule_attention.cuda() 72 | self.word_attention = self.word_attention.cuda() 73 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 74 | self.ctx_to_readout = self.ctx_to_readout.cuda() 75 | else: 76 | self.rule_ctx_to_readout = self.rule_ctx_to_readout.cuda() 77 | self.word_ctx_to_readout = self.word_ctx_to_readout.cuda() 78 | self.readout = self.readout.cuda() 79 | self.rule_lstm_cell = self.rule_lstm_cell.cuda() 80 | self.word_lstm_cell = self.word_lstm_cell.cuda() 81 | self.dropout = self.dropout.cuda() 82 | 83 | def forward(self, x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label=None): 84 | # get decoder init state and cell, use x_ct 85 | """ 86 | x_enc: [batch_size, max_x_len, d_model * 2] 87 | """ 88 | batch_size_x = x_enc.size()[0] 89 | batch_size, y_max_len, data_len = y_train.size() 90 | assert batch_size_x == batch_size 91 | #print(y_train) 92 | input_feed_zeros = torch.zeros(batch_size, self.hparams.d_model * 2) 93 | if self.hparams.cuda: 94 | input_feed_zeros = input_feed_zeros.cuda() 95 | rule_hidden = dec_init 96 | word_hidden = dec_init 97 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 98 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 99 | if self.hparams.cuda: 100 | rule_input_feed = rule_input_feed.cuda() 101 | word_input_feed = word_input_feed.cuda() 102 | # [batch_size, y_len, d_word_vec] 103 | trg_emb = self.emb(y_train[:, :, 0]) 104 | logits = [] 105 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 106 | pre_readouts = [] 107 | else: 108 | rule_pre_readouts = [] 109 | word_pre_readouts = [] 110 | if self.hparams.parent_feed: 111 | all_state = Variable(torch.zeros(batch_size, self.hparams.d_model), requires_grad=False) 112 | offset = torch.arange(batch_size).long() 113 | if self.hparams.cuda: 114 | offset = offset.cuda() 115 | if self.hparams.parent_feed: 116 | all_state = all_state.cuda() 117 | for t in range(y_max_len): 118 | y_emb_tm1 = trg_emb[:, t, :] 119 | 120 | if self.hparams.parent_feed: 121 | state_idx_t = t 122 | state_idx_t += 1 123 | parent_t = y_train.data[:, t, 1] + state_idx_t * offset # [batch_size,] 124 | parent_t = Variable(parent_t, requires_grad=False) 125 | parent_state = torch.index_select(all_state.view(state_idx_t*batch_size, self.hparams.d_model), dim=0, index=parent_t) # [batch_size, d_model] 126 | 127 | word_mask = y_train[:, t, 2].unsqueeze(1).float() # (1 is word, 0 is rule) 128 | 129 | if self.hparams.parent_feed: 130 | word_input = torch.cat([y_emb_tm1, parent_state, word_input_feed], dim=1) 131 | else: 132 | word_input = torch.cat([y_emb_tm1, rule_hidden[0], word_input_feed], dim=1) 133 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 134 | 135 | word_h_t = word_h_t * word_mask + word_hidden[0] * (1-word_mask) 136 | word_c_t = word_c_t * word_mask + word_hidden[1] * (1-word_mask) 137 | 138 | if self.hparams.rule_parent_feed: 139 | rule_input = torch.cat([y_emb_tm1, parent_state, rule_input_feed, word_h_t], dim=1) 140 | else: 141 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, word_h_t], dim=1) 142 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 143 | 144 | if hasattr(self.hparams, "no_word_to_rule") and self.hparams.no_word_to_rule: 145 | eos_mask = (y_train[:, t, 0] == self.hparams.eos_id).unsqueeze(1).float() 146 | word_mask = word_mask - eos_mask 147 | rule_h_t = rule_h_t * (1-word_mask) + rule_hidden[0] * word_mask 148 | rule_c_t = rule_c_t * (1-word_mask) + rule_hidden[1] * word_mask 149 | 150 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 151 | rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 152 | word_ctx = self.attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 153 | else: 154 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 155 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 156 | 157 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 158 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(torch.cat([rule_h_t, rule_ctx], dim=1))) 159 | word_pre_readout = F.tanh(self.word_ctx_to_readout(torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1))) 160 | else: 161 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 162 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 163 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 164 | 165 | rule_pre_readout = self.dropout(rule_pre_readout) 166 | word_pre_readout = self.dropout(word_pre_readout) 167 | 168 | rule_pre_readouts.append(rule_pre_readout) 169 | word_pre_readouts.append(word_pre_readout) 170 | 171 | #inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 172 | #if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 173 | # pre_readout = F.tanh(self.ctx_to_readout(r_inp)) 174 | # pre_readout = self.dropout(pre_readout) 175 | # pre_readouts.append(pre_readout) 176 | #else: 177 | # rule_pre_readout = F.tanh(self.rule_ctx_to_readout(r_inp)) 178 | # word_pre_readout = F.tanh(self.word_ctx_to_readout(w_inp)) 179 | # 180 | # rule_pre_readout = self.dropout(rule_pre_readout) 181 | # word_pre_readout = self.dropout(word_pre_readout) 182 | # 183 | # rule_pre_readouts.append(rule_pre_readout) 184 | # word_pre_readouts.append(word_pre_readout) 185 | 186 | rule_input_feed = rule_ctx 187 | word_input_feed = word_ctx 188 | 189 | rule_hidden = (rule_h_t, rule_c_t) 190 | word_hidden = (word_h_t, word_c_t) 191 | if self.hparams.parent_feed: 192 | all_state = torch.cat([all_state, rule_h_t], dim=1) 193 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 194 | readouts = self.readout(torch.stack(pre_readouts)) 195 | logits = readouts.transpose(0, 1).contiguous() 196 | else: 197 | # [len_y, batch_size, trg_vocab_size] 198 | rule_readouts = self.readout(torch.stack(rule_pre_readouts))[:,:,-self.hparams.target_rule_vocab_size:] 199 | if hasattr(self.hparams, "rule_tanh") and self.hparams.rule_tanh > 0: 200 | rule_readouts = self.hparams.rule_tanh * torch.tanh(rule_readouts) 201 | word_readouts = self.readout(torch.stack(word_pre_readouts))[:,:,:self.hparams.target_word_vocab_size] 202 | if self.hparams.label_smooth > 0: 203 | smooth = self.hparams.label_smooth 204 | rule_probs = (1.0 - smooth) * F.softmax(rule_readouts, dim=2) + smooth / self.hparams.target_rule_vocab_size 205 | rule_readouts = torch.log(rule_probs) 206 | # [batch_size, len_y, trg_vocab_size] 207 | logits = torch.cat([word_readouts, rule_readouts], dim=2).transpose(0, 1).contiguous() 208 | score_mask = score_mask.unsqueeze(2).float().data 209 | mask_t = self.word_vocab_mask * (1-score_mask) + self.rule_vocab_mask * score_mask 210 | logits.data.masked_fill_(mask_t.byte(), -float("inf")) 211 | return logits 212 | 213 | def step(self, x_enc, x_enc_k, hyp, target_rule_vocab): 214 | y_tm1 = torch.LongTensor([int(hyp.y[-1])]) 215 | if self.hparams.cuda: 216 | y_tm1 = y_tm1.cuda() 217 | y_tm1 = Variable(y_tm1, volatile=True) 218 | open_nonterms = hyp.open_nonterms 219 | rule_input_feed = hyp.rule_ctx_tm1 220 | word_input_feed = hyp.word_ctx_tm1 221 | rule_hidden = hyp.rule_hidden 222 | word_hidden = hyp.word_hidden 223 | word_h_t, word_c_t = word_hidden 224 | rule_h_t, rule_c_t = rule_hidden 225 | 226 | y_emb_tm1 = self.emb(y_tm1) 227 | cur_nonterm = open_nonterms[-1] 228 | parent_state = cur_nonterm.parent_state 229 | 230 | if hyp.y[-1] < self.hparams.target_word_vocab_size: 231 | # word 232 | if self.hparams.parent_feed: 233 | word_input = torch.cat([y_emb_tm1, parent_state, word_input_feed], dim=1) 234 | else: 235 | word_input = torch.cat([y_emb_tm1, rule_h_t, word_input_feed], dim=1) 236 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 237 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 238 | word_ctx = self.attention(word_h_t, x_enc_k, x_enc) 239 | else: 240 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc) 241 | else: 242 | word_ctx = hyp.word_ctx_tm1 243 | 244 | update_rule_rnn = True 245 | if hasattr(self.hparams, "no_word_to_rule") and self.hparams.no_word_to_rule and hyp.y[-1] < self.hparams.target_word_vocab_size: 246 | if hyp.y[-1] != self.hparams.eos_id: 247 | update_rule_rnn = False 248 | 249 | if update_rule_rnn: 250 | if self.hparams.rule_parent_feed: 251 | rule_input = torch.cat([y_emb_tm1, parent_state, rule_input_feed, word_h_t], dim=1) 252 | else: 253 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, word_h_t], dim=1) 254 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 255 | # if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 256 | # rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc) 257 | # else: 258 | # rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc) 259 | #else: 260 | # rule_ctx = hyp.rule_ctx_tm1 261 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 262 | rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc) 263 | else: 264 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc) 265 | 266 | mask = torch.ones(1, self.target_vocab_size).byte() 267 | if cur_nonterm.label == '*': 268 | word_index = torch.arange(self.hparams.target_word_vocab_size).long() 269 | mask.index_fill_(1, word_index, 0) 270 | #if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 271 | # inp = torch.cat([word_h_t, word_ctx], dim=1) 272 | #else: 273 | # inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 274 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 275 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 276 | word_pre_readout = F.tanh(self.ctx_to_readout(inp)) 277 | else: 278 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 279 | #word_pre_readout = self.dropout(word_pre_readout) 280 | score_t = self.readout(word_pre_readout) 281 | num_rule_index = -1 282 | rule_select_index = [] 283 | else: 284 | rule_with_lhs = target_rule_vocab.rule_index_with_lhs(cur_nonterm.label) 285 | rule_select_index = [] 286 | for i in rule_with_lhs: rule_select_index.append(i + self.hparams.target_word_vocab_size) 287 | num_rule_index = len(rule_with_lhs) 288 | rule_index = torch.arange(self.hparams.target_rule_vocab_size).long() + self.hparams.target_word_vocab_size 289 | mask.index_fill_(1, rule_index, 0) 290 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 291 | inp = torch.cat([rule_h_t, rule_ctx], dim=1) 292 | else: 293 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 294 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 295 | rule_pre_readout = F.tanh(self.ctx_to_readout(inp)) 296 | else: 297 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 298 | #rule_pre_readout = self.dropout(rule_pre_readout) 299 | score_t = self.readout(rule_pre_readout) 300 | if self.hparams.cuda: 301 | mask = mask.cuda() 302 | score_t.data.masked_fill_(mask, -float("inf")) 303 | rule_hidden = (rule_h_t, rule_c_t) 304 | word_hidden = (word_h_t, word_c_t) 305 | return score_t, rule_hidden, word_hidden, rule_ctx, word_ctx, num_rule_index, rule_select_index 306 | 307 | class TrDec(nn.Module): 308 | 309 | def __init__(self, hparams): 310 | super(TrDec, self).__init__() 311 | self.encoder = Encoder(hparams) 312 | self.decoder = TreeDecoder(hparams) 313 | # transform encoder state vectors into attention key vector 314 | self.enc_to_k = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 315 | self.hparams = hparams 316 | if self.hparams.cuda: 317 | self.enc_to_k = self.enc_to_k.cuda() 318 | 319 | def forward(self, x_train, x_mask, x_len, y_train, y_mask, y_len, score_mask, y_label=None): 320 | # [batch_size, x_len, d_model * 2] 321 | x_enc, dec_init = self.encoder(x_train, x_len) 322 | x_enc_k = self.enc_to_k(x_enc) 323 | # [batch_size, y_len-1, trg_vocab_size] 324 | logits = self.decoder(x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label) 325 | return logits 326 | 327 | def translate(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=0): 328 | hyps = [] 329 | scores = [] 330 | i = 0 331 | self.logsoftmax = nn.LogSoftmax(dim=1) 332 | for x in x_train: 333 | x = Variable(torch.LongTensor(x), volatile=True) 334 | if y_label: 335 | y = y_label[i][1:] 336 | else: 337 | y = None 338 | if self.hparams.cuda: 339 | x = x.cuda() 340 | hyp, nll_score = self.translate_sent(x, target_rule_vocab, max_len=max_len, beam_size=beam_size, y_label=y, poly_norm_m=poly_norm_m) 341 | hyp = hyp[0] 342 | hyps.append(hyp.y[1:]) 343 | scores.append(sum(nll_score)) 344 | #print(hyp.y) 345 | #print("trans score:", nll_score) 346 | #print("trans label:", y) 347 | i += 1 348 | gc.collect() 349 | print(i) 350 | return hyps, scores 351 | 352 | def translate_sent(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=1.): 353 | assert len(x_train.size()) == 1 354 | x_len = [x_train.size(0)] 355 | x_train = x_train.unsqueeze(0) 356 | x_enc, dec_init = self.encoder(x_train, x_len) 357 | x_enc_k = self.enc_to_k(x_enc) 358 | length = 0 359 | completed_hyp = [] 360 | input_feed_zeros = torch.zeros(1, self.hparams.d_model * 2) 361 | state_zeros = torch.zeros(1, self.hparams.d_model) 362 | if self.hparams.cuda: 363 | input_feed_zeros = input_feed_zeros.cuda() 364 | state_zeros = state_zeros.cuda() 365 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 366 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 367 | active_hyp = [TrHyp(rule_hidden=dec_init, 368 | word_hidden=dec_init, 369 | y=[self.hparams.bos_id], 370 | rule_ctx_tm1=rule_input_feed, 371 | word_ctx_tm1=word_input_feed, 372 | open_nonterms=[OpenNonterm(label=self.hparams.root_label, 373 | parent_state=Variable(state_zeros, requires_grad=False))], 374 | score=0.)] 375 | nll_score = [] 376 | if y_label is not None: 377 | max_len = len(y_label) 378 | while len(completed_hyp) < beam_size and length < max_len: 379 | length += 1 380 | print(length, len(completed_hyp)) 381 | new_active_hyp = [] 382 | for i, hyp in enumerate(active_hyp): 383 | logits, rule_hidden, word_hidden, rule_ctx, word_ctx, num_rule_index, rule_index = self.decoder.step(x_enc, 384 | x_enc_k, hyp, target_rule_vocab) 385 | hyp.rule_hidden = rule_hidden 386 | hyp.word_hidden = word_hidden 387 | hyp.rule_ctx_tm1 = rule_ctx 388 | hyp.word_ctx_tm1 = word_ctx 389 | 390 | rule_index = set(rule_index) 391 | logits = logits.view(-1) 392 | p_t = F.log_softmax(logits, dim=0).data 393 | if hasattr(self.hparams, "ignore_rule_len") and self.hparams.ignore_rule_len: 394 | l = (np.array(hyp.y) < self.hparams.target_word_vocab_size).sum() 395 | if poly_norm_m > 0 and l > 1: 396 | new_hyp_scores = (hyp.score * pow(l-1, poly_norm_m) + p_t) / pow(l, poly_norm_m) 397 | else: 398 | new_hyp_scores = hyp.score + p_t 399 | else: 400 | if poly_norm_m > 0 and length > 1: 401 | new_hyp_scores = (hyp.score * pow(length-1, poly_norm_m) + p_t) / pow(length, poly_norm_m) 402 | else: 403 | new_hyp_scores = hyp.score + p_t 404 | if y_label is not None: 405 | top_ids = [y_label[length-1][0]] 406 | nll = -(p_t[top_ids[0]]) 407 | nll_score.append(nll) 408 | print(logits[top_ids[0]]) 409 | else: 410 | num_select = beam_size 411 | if num_rule_index >= 0: num_select = min(num_select, num_rule_index) 412 | top_ids = (-new_hyp_scores).cpu().numpy().argsort()[:num_select] 413 | #print("top ids" ) 414 | for word_id in top_ids: 415 | #print(word_id) 416 | if y_label is None and len(rule_index) > 0 and word_id not in rule_index: continue 417 | #print("pass through") 418 | open_nonterms = hyp.open_nonterms[:] 419 | if word_id >= self.hparams.target_word_vocab_size: 420 | rule = target_rule_vocab[word_id] 421 | # force the first rule to be not preterminal 422 | if hasattr(self.hparams, "force_rule") and self.hparams.force_rule: 423 | #if length <= self.hparams.force_rule_step and rule.rhs[0] == "*": 424 | if length <= self.hparams.force_rule_step and len(rule.rhs) == 1: 425 | continue 426 | cur_nonterm = open_nonterms.pop() 427 | parent_state = hyp.rule_hidden[0] 428 | for c in reversed(rule.rhs): 429 | open_nonterms.append(OpenNonterm(label=c, parent_state=parent_state)) 430 | else: 431 | if open_nonterms[-1].label != '*': 432 | print(open_nonterms[-1].label, word_id, new_hyp_scores[word_id]) 433 | print(target_rule_vocab.rule_index_with_lhs(open_nonterms[-1].label)) 434 | print(top_ids) 435 | assert open_nonterms[-1].label == '*' 436 | if word_id == self.hparams.eos_id: 437 | open_nonterms.pop() 438 | new_hyp = TrHyp(rule_hidden=(hyp.rule_hidden[0], hyp.rule_hidden[1]), 439 | word_hidden=(hyp.word_hidden[0], hyp.word_hidden[1]), 440 | y=hyp.y+[word_id], 441 | rule_ctx_tm1=hyp.rule_ctx_tm1, 442 | word_ctx_tm1=hyp.word_ctx_tm1, 443 | open_nonterms=open_nonterms, 444 | score=new_hyp_scores[word_id], 445 | c_p=p_t[word_id]) 446 | new_active_hyp.append(new_hyp) 447 | if y_label is None: 448 | live_hyp_num = beam_size - len(completed_hyp) 449 | new_active_hyp = sorted(new_active_hyp, key=lambda x:x.score, reverse=True)[:min(beam_size, live_hyp_num)] 450 | active_hyp = [] 451 | for hyp in new_active_hyp: 452 | if len(hyp.open_nonterms) == 0: 453 | #if poly_norm_m <= 0: 454 | # hyp.score = hyp.score / len(hyp.y) 455 | completed_hyp.append(hyp) 456 | else: 457 | active_hyp.append(hyp) 458 | else: 459 | active_hyp = new_active_hyp 460 | 461 | if len(completed_hyp) == 0: 462 | completed_hyp.append(active_hyp[0]) 463 | return sorted(completed_hyp, key=lambda x: x.score, reverse=True), nll_score 464 | 465 | class TrHyp(object): 466 | def __init__(self, rule_hidden, word_hidden, y, rule_ctx_tm1, word_ctx_tm1, score, open_nonterms, c_p=0): 467 | self.rule_hidden = rule_hidden 468 | self.word_hidden = word_hidden 469 | # [length_y, 2], each element (index, is_word) 470 | self.y = y 471 | self.rule_ctx_tm1 = rule_ctx_tm1 472 | self.word_ctx_tm1 = word_ctx_tm1 473 | self.score = score 474 | self.open_nonterms = open_nonterms 475 | self.c_p = c_p 476 | 477 | class OpenNonterm(object): 478 | def __init__(self, label, parent_state): 479 | self.label = label 480 | self.parent_state = parent_state 481 | -------------------------------------------------------------------------------- /src/trdec_attn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.init as init 3 | from torch.autograd import Variable 4 | from torch import nn 5 | from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | from utils import * 10 | from models import * 11 | import gc 12 | 13 | class TreeDecoderAttn(nn.Module): 14 | def __init__(self, hparams): 15 | super(TreeDecoderAttn, self).__init__() 16 | self.hparams = hparams 17 | self.target_vocab_size = self.hparams.target_word_vocab_size+self.hparams.target_rule_vocab_size 18 | self.emb = nn.Embedding(self.target_vocab_size, 19 | self.hparams.d_word_vec, 20 | padding_idx=hparams.pad_id) 21 | if self.hparams.attn == "mlp": 22 | self.rule_attention = MlpAttn(hparams) 23 | self.word_attention = MlpAttn(hparams) 24 | else: 25 | self.rule_attention = DotProdAttn(hparams) 26 | self.word_attention = DotProdAttn(hparams) 27 | 28 | if self.hparams.self_attn == "mlp": 29 | self.rule_to_word_attn = MlpAttn(hparams) 30 | self.word_to_rule_attn = MlpAttn(hparams) 31 | elif self.hparams.self_attn == "dot_prod": 32 | self.rule_to_word_attn = DotProdAttn(hparams) 33 | self.word_to_rule_attn = DotProdAttn(hparams) 34 | else: 35 | print("mult head attn") 36 | self.rule_to_word_attn = MultiHeadAttn(hparams) 37 | self.word_to_rule_attn = MultiHeadAttn(hparams) 38 | 39 | readout_in_dim = hparams.d_model * 6 40 | if hasattr(self.hparams, "self_attn_input_feed") and self.hparams.self_attn_input_feed: 41 | readout_in_dim += hparams.d_model * 2 42 | # transform [word_ctx, word_h_t, rule_ctx, rule_h_t] to readout state vectors before softmax 43 | self.rule_ctx_to_readout = nn.Linear(readout_in_dim, hparams.d_model, bias=False) 44 | self.word_ctx_to_readout = nn.Linear(readout_in_dim, hparams.d_model, bias=False) 45 | 46 | self.readout = nn.Linear(hparams.d_model, 47 | self.target_vocab_size, 48 | bias=False) 49 | if self.hparams.share_emb_softmax: 50 | self.emb.weight = self.readout.weight 51 | # input: [y_t-1, input_feed, word_state_ctx] 52 | rule_inp = hparams.d_word_vec + hparams.d_model * 3 53 | # input: [y_t-1, input_feed, rule_state_ctx] 54 | word_inp = hparams.d_word_vec + hparams.d_model * 3 55 | if self.hparams.parent_feed: 56 | word_inp += hparams.d_model 57 | self.rule_lstm_cell = nn.LSTMCell(rule_inp, hparams.d_model) 58 | self.word_lstm_cell = nn.LSTMCell(word_inp, hparams.d_model) 59 | self.dropout = nn.Dropout(hparams.dropout) 60 | 61 | vocab_mask = torch.zeros(1, 1, self.target_vocab_size) 62 | self.word_vocab_mask = vocab_mask.index_fill_(2, torch.arange(self.hparams.target_word_vocab_size).long(), 1) 63 | self.rule_vocab_mask = 1 - self.word_vocab_mask 64 | if self.hparams.cuda: 65 | self.rule_vocab_mask = self.rule_vocab_mask.cuda() 66 | self.word_vocab_mask = self.word_vocab_mask.cuda() 67 | self.emb = self.emb.cuda() 68 | self.rule_attention = self.rule_attention.cuda() 69 | self.word_attention = self.word_attention.cuda() 70 | self.rule_to_word_attn = self.rule_to_word_attn.cuda() 71 | self.word_to_rule_attn = self.word_to_rule_attn.cuda() 72 | 73 | self.rule_ctx_to_readout = self.rule_ctx_to_readout.cuda() 74 | self.word_ctx_to_readout = self.word_ctx_to_readout.cuda() 75 | self.readout = self.readout.cuda() 76 | self.rule_lstm_cell = self.rule_lstm_cell.cuda() 77 | self.word_lstm_cell = self.word_lstm_cell.cuda() 78 | self.dropout = self.dropout.cuda() 79 | 80 | def forward(self, x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label=None): 81 | # get decoder init state and cell, use x_ct 82 | """ 83 | x_enc: [batch_size, max_x_len, d_model * 2] 84 | """ 85 | batch_size_x, x_max_len, d_x = x_enc.size() 86 | batch_size, y_max_len, data_len = y_train.size() 87 | assert batch_size_x == batch_size 88 | #print(y_train) 89 | input_feed_zeros = torch.zeros(batch_size, self.hparams.d_model * 2) 90 | input_feed_zeros_d = torch.zeros(batch_size, self.hparams.d_model) 91 | rule_states = Variable(torch.zeros(batch_size, y_max_len, self.hparams.d_model) ) 92 | word_states = Variable(torch.zeros(batch_size, y_max_len, self.hparams.d_model) ) 93 | rule_states_mask = torch.ones(batch_size, y_max_len).byte() 94 | word_states_mask = torch.ones(batch_size, y_max_len).byte() 95 | # avoid attn nan 96 | word_states_mask.index_fill_(1, torch.LongTensor([0]), 0) 97 | rule_states_mask.index_fill_(1, torch.LongTensor([0]), 0) 98 | if self.hparams.cuda: 99 | input_feed_zeros = input_feed_zeros.cuda() 100 | rule_states = rule_states.cuda() 101 | word_states = word_states.cuda() 102 | rule_states_mask = rule_states_mask.cuda() 103 | word_states_mask = word_states_mask.cuda() 104 | 105 | rule_hidden = dec_init 106 | word_hidden = dec_init 107 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 108 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 109 | rule_to_word_input_feed = Variable(input_feed_zeros_d, requires_grad=False) 110 | word_to_rule_input_feed = Variable(input_feed_zeros_d, requires_grad=False) 111 | if self.hparams.cuda: 112 | rule_input_feed = rule_input_feed.cuda() 113 | word_input_feed = word_input_feed.cuda() 114 | rule_to_word_input_feed = rule_to_word_input_feed.cuda() 115 | word_to_rule_input_feed = word_to_rule_input_feed.cuda() 116 | # [batch_size, y_len, d_word_vec] 117 | trg_emb = self.emb(y_train[:, :, 0]) 118 | logits = [] 119 | rule_pre_readouts = [] 120 | word_pre_readouts = [] 121 | if self.hparams.parent_feed: 122 | all_state = Variable(torch.zeros(batch_size, self.hparams.d_model), requires_grad=False) 123 | offset = torch.arange(batch_size).long() 124 | if self.hparams.cuda: 125 | offset = offset.cuda() 126 | if self.hparams.parent_feed: 127 | all_state = all_state.cuda() 128 | for t in range(y_max_len): 129 | y_emb_tm1 = trg_emb[:, t, :] 130 | 131 | if self.hparams.parent_feed: 132 | state_idx_t = t 133 | state_idx_t += 1 134 | parent_t = y_train.data[:, t, 1] + state_idx_t * offset # [batch_size,] 135 | parent_t = Variable(parent_t, requires_grad=False) 136 | parent_state = torch.index_select(all_state.view(state_idx_t*batch_size, self.hparams.d_model), dim=0, index=parent_t) # [batch_size, d_model] 137 | 138 | word_mask = y_train[:, t, 2].unsqueeze(1).float() # (1 is word, 0 is rule) 139 | 140 | if self.hparams.parent_feed: 141 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed, parent_state], dim=1) 142 | else: 143 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed], dim=1) 144 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 145 | 146 | word_h_t = word_h_t * word_mask + word_hidden[0] * (1-word_mask) 147 | word_c_t = word_c_t * word_mask + word_hidden[1] * (1-word_mask) 148 | 149 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, rule_to_word_input_feed], dim=1) 150 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 151 | 152 | eos_mask = (y_train[:, t, 0] == self.hparams.eos_id).unsqueeze(1).float() 153 | word_mask = word_mask - eos_mask 154 | rule_h_t = rule_h_t * (1-word_mask) + rule_hidden[0] * word_mask 155 | rule_c_t = rule_c_t * (1-word_mask) + rule_hidden[1] * word_mask 156 | 157 | rule_states.data[:, t, :] = rule_h_t.data 158 | word_states.data[:, t, :] = word_h_t.data 159 | # word_mask: 1 is word, 0 is rule 160 | if t > 0: 161 | rule_states_mask[:, t] = word_mask.byte().data 162 | word_states_mask[:, t] = (1-word_mask).byte().data 163 | 164 | rule_to_word_ctx = self.rule_to_word_attn(rule_h_t, word_states, word_states, attn_mask=word_states_mask) 165 | word_to_rule_ctx = self.word_to_rule_attn(word_h_t, rule_states, rule_states, attn_mask=rule_states_mask) 166 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 167 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 168 | 169 | if hasattr(self.hparams, "self_attn_input_feed") and self.hparams.self_attn_input_feed: 170 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx, rule_to_word_input_feed, word_to_rule_input_feed], dim=1) 171 | else: 172 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 173 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 174 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 175 | 176 | rule_pre_readout = self.dropout(rule_pre_readout) 177 | word_pre_readout = self.dropout(word_pre_readout) 178 | 179 | rule_pre_readouts.append(rule_pre_readout) 180 | word_pre_readouts.append(word_pre_readout) 181 | 182 | rule_input_feed = rule_ctx 183 | word_input_feed = word_ctx 184 | rule_to_word_input_feed = rule_to_word_ctx 185 | word_to_rule_input_feed = word_to_rule_ctx 186 | 187 | rule_hidden = (rule_h_t, rule_c_t) 188 | word_hidden = (word_h_t, word_c_t) 189 | if self.hparams.parent_feed: 190 | all_state = torch.cat([all_state, rule_h_t], dim=1) 191 | # [len_y, batch_size, trg_vocab_size] 192 | rule_readouts = self.readout(torch.stack(rule_pre_readouts))[:,:,-self.hparams.target_rule_vocab_size:] 193 | word_readouts = self.readout(torch.stack(word_pre_readouts))[:,:,:self.hparams.target_word_vocab_size] 194 | if self.hparams.label_smooth > 0: 195 | smooth = self.hparams.label_smooth 196 | rule_probs = (1.0 - smooth) * F.softmax(rule_readouts, dim=2) + smooth / self.hparams.target_rule_vocab_size 197 | rule_readouts = torch.log(rule_probs) 198 | # [batch_size, len_y, trg_vocab_size] 199 | logits = torch.cat([word_readouts, rule_readouts], dim=2).transpose(0, 1).contiguous() 200 | score_mask = score_mask.unsqueeze(2).float().data 201 | mask_t = self.word_vocab_mask * (1-score_mask) + self.rule_vocab_mask * score_mask 202 | logits.data.masked_fill_(mask_t.byte(), -float("inf")) 203 | return logits 204 | 205 | def step(self, x_enc, x_enc_k, hyp, target_rule_vocab): 206 | y_tm1 = torch.LongTensor([int(hyp.y[-1])]) 207 | if self.hparams.cuda: 208 | y_tm1 = y_tm1.cuda() 209 | y_tm1 = Variable(y_tm1, volatile=True) 210 | open_nonterms = hyp.open_nonterms 211 | rule_input_feed = hyp.rule_ctx_tm1 212 | word_input_feed = hyp.word_ctx_tm1 213 | rule_to_word_input_feed = hyp.rule_to_word_ctx_tm1 214 | word_to_rule_input_feed = hyp.word_to_rule_ctx_tm1 215 | rule_hidden = hyp.rule_hidden 216 | word_hidden = hyp.word_hidden 217 | word_h_t, word_c_t = word_hidden 218 | rule_h_t, rule_c_t = rule_hidden 219 | 220 | y_emb_tm1 = self.emb(y_tm1) 221 | cur_nonterm = open_nonterms[-1] 222 | 223 | if hyp.y[-1] < self.hparams.target_word_vocab_size: 224 | # word 225 | if self.hparams.parent_feed: 226 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed, hyp.parent_state], dim=1) 227 | else: 228 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed], dim=1) 229 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 230 | if hyp.rule_states is None: 231 | hyp.rule_states = rule_h_t.unsqueeze(1) 232 | if hyp.word_states is None: 233 | # [1, 1, d_model] 234 | hyp.word_states = word_h_t.unsqueeze(1) 235 | else: 236 | if hyp.y[-1] != self.hparams.eos_id: 237 | hyp.word_states = torch.cat([hyp.word_states, word_h_t.unsqueeze(1)], dim=1) 238 | 239 | if hyp.y[-1] == self.hparams.eos_id or hyp.y[-1] >= self.hparams.target_word_vocab_size: 240 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, rule_to_word_input_feed], dim=1) 241 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 242 | hyp.rule_states = torch.cat([hyp.rule_states, rule_h_t.unsqueeze(1)], dim=1) 243 | 244 | hyp.rule_ctx_tm1 = self.rule_attention(rule_h_t, x_enc_k, x_enc) 245 | hyp.word_ctx_tm1 = self.word_attention(word_h_t, x_enc_k, x_enc) 246 | hyp.rule_to_word_ctx_tm1 = self.rule_to_word_attn(rule_h_t, hyp.word_states, hyp.word_states) 247 | hyp.word_to_rule_ctx_tm1 = self.word_to_rule_attn(word_h_t, hyp.rule_states, hyp.rule_states) 248 | 249 | if hasattr(self.hparams, "self_attn_input_feed") and self.hparams.self_attn_input_feed: 250 | inp = torch.cat([rule_h_t, hyp.rule_ctx_tm1, word_h_t, hyp.word_ctx_tm1, hyp.rule_to_word_ctx_tm1, hyp.word_to_rule_ctx_tm1], dim=1) 251 | else: 252 | inp = torch.cat([rule_h_t, hyp.rule_ctx_tm1, word_h_t, hyp.word_ctx_tm1], dim=1) 253 | mask = torch.ones(1, self.target_vocab_size).byte() 254 | if cur_nonterm.label == '*': 255 | word_index = torch.arange(self.hparams.target_word_vocab_size).long() 256 | mask.index_fill_(1, word_index, 0) 257 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 258 | word_pre_readout = self.dropout(word_pre_readout) 259 | score_t = self.readout(word_pre_readout) 260 | num_rule_index = -1 261 | rule_select_index = [] 262 | else: 263 | rule_with_lhs = target_rule_vocab.rule_index_with_lhs(cur_nonterm.label) 264 | rule_select_index = [] 265 | for i in rule_with_lhs: rule_select_index.append(i + self.hparams.target_word_vocab_size) 266 | num_rule_index = len(rule_with_lhs) 267 | rule_index = torch.arange(self.hparams.target_rule_vocab_size).long() + self.hparams.target_word_vocab_size 268 | mask.index_fill_(1, rule_index, 0) 269 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 270 | rule_pre_readout = self.dropout(rule_pre_readout) 271 | score_t = self.readout(rule_pre_readout) 272 | if self.hparams.cuda: 273 | mask = mask.cuda() 274 | score_t.data.masked_fill_(mask, -float("inf")) 275 | rule_hidden = (rule_h_t, rule_c_t) 276 | word_hidden = (word_h_t, word_c_t) 277 | 278 | hyp.rule_hidden = rule_hidden 279 | hyp.word_hidden = word_hidden 280 | 281 | return score_t, num_rule_index, rule_select_index 282 | 283 | class TrDecAttn(nn.Module): 284 | def __init__(self, hparams): 285 | super(TrDecAttn, self).__init__() 286 | self.encoder = Encoder(hparams) 287 | self.decoder = TreeDecoderAttn(hparams) 288 | # transform encoder state vectors into attention key vector 289 | self.enc_to_k = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 290 | self.hparams = hparams 291 | if self.hparams.cuda: 292 | self.enc_to_k = self.enc_to_k.cuda() 293 | 294 | def forward(self, x_train, x_mask, x_len, y_train, y_mask, y_len, score_mask, y_label=None): 295 | # [batch_size, x_len, d_model * 2] 296 | x_enc, dec_init = self.encoder(x_train, x_len) 297 | x_enc_k = self.enc_to_k(x_enc) 298 | # [batch_size, y_len-1, trg_vocab_size] 299 | logits = self.decoder(x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label) 300 | return logits 301 | 302 | def translate(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=0): 303 | hyps = [] 304 | scores = [] 305 | i = 0 306 | self.logsoftmax = nn.LogSoftmax(dim=1) 307 | for x in x_train: 308 | x = Variable(torch.LongTensor(x), volatile=True) 309 | if y_label: 310 | y = y_label[i][1:] 311 | else: 312 | y = None 313 | if self.hparams.cuda: 314 | x = x.cuda() 315 | hyp, nll_score = self.translate_sent(x, target_rule_vocab, max_len=max_len, beam_size=beam_size, y_label=y, poly_norm_m=poly_norm_m) 316 | if hasattr(self.hparams, "nbest") and self.hparams.nbest: 317 | nbest = [h.y[1:] for h in hyp] 318 | hyps.append(nbest) 319 | else: 320 | hyp = hyp[0] 321 | hyps.append(hyp.y[1:]) 322 | scores.append(sum(nll_score)) 323 | #print(hyp.y) 324 | #print("trans score:", nll_score) 325 | #print("trans label:", y) 326 | i += 1 327 | gc.collect() 328 | return hyps, scores 329 | 330 | def translate_sent(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=1.): 331 | assert len(x_train.size()) == 1 332 | x_len = [x_train.size(0)] 333 | x_train = x_train.unsqueeze(0) 334 | x_enc, dec_init = self.encoder(x_train, x_len) 335 | x_enc_k = self.enc_to_k(x_enc) 336 | length = 0 337 | completed_hyp = [] 338 | input_feed_zeros = torch.zeros(1, self.hparams.d_model * 2) 339 | to_input_feed_zeros = torch.zeros(1, self.hparams.d_model) 340 | state_zeros = torch.zeros(1, 1, self.hparams.d_model) 341 | if self.hparams.cuda: 342 | input_feed_zeros = input_feed_zeros.cuda() 343 | to_input_feed_zeros = to_input_feed_zeros.cuda() 344 | state_zeros = state_zeros.cuda() 345 | active_hyp = [TrAttnHyp(rule_hidden=dec_init, 346 | word_hidden=dec_init, 347 | y=[self.hparams.bos_id], 348 | rule_ctx_tm1= Variable(input_feed_zeros, requires_grad=False), 349 | word_ctx_tm1=Variable(input_feed_zeros, requires_grad=False), 350 | rule_to_word_ctx_tm1=Variable(to_input_feed_zeros, requires_grad=False), 351 | word_to_rule_ctx_tm1= Variable(to_input_feed_zeros, requires_grad=False), 352 | rule_states=None, 353 | word_states=None, 354 | open_nonterms=[OpenNonterm(label=self.hparams.root_label, parent_state=state_zeros)], 355 | score=0.)] 356 | nll_score = [] 357 | if y_label is not None: 358 | max_len = len(y_label) 359 | while len(completed_hyp) < beam_size and length < max_len: 360 | length += 1 361 | new_active_hyp = [] 362 | for i, hyp in enumerate(active_hyp): 363 | logits, num_rule_index, rule_index = self.decoder.step(x_enc, 364 | x_enc_k, hyp, target_rule_vocab) 365 | 366 | rule_index = set(rule_index) 367 | logits = logits.view(-1) 368 | p_t = F.log_softmax(logits, dim=0).data 369 | 370 | if hasattr(self.hparams, "ignore_rule_len") and self.hparams.ignore_rule_len: 371 | l = (np.array(hyp.y) < self.hparams.target_word_vocab_size).sum() 372 | if poly_norm_m > 0 and l > 1: 373 | new_hyp_scores = (hyp.score * pow(l-1, poly_norm_m) + p_t) / pow(l, poly_norm_m) 374 | else: 375 | new_hyp_scores = hyp.score + p_t 376 | else: 377 | if poly_norm_m > 0 and length > 1: 378 | new_hyp_scores = (hyp.score * pow(length-1, poly_norm_m) + p_t) / pow(length, poly_norm_m) 379 | else: 380 | new_hyp_scores = hyp.score + p_t 381 | 382 | if y_label is not None: 383 | top_ids = [y_label[length-1][0]] 384 | nll = -(p_t[top_ids[0]]) 385 | nll_score.append(nll) 386 | print("logit dedcode", logits[top_ids[0]]) 387 | else: 388 | num_select = beam_size 389 | if num_rule_index >= 0: num_select = min(num_select, num_rule_index) 390 | top_ids = (-new_hyp_scores).cpu().numpy().argsort()[:num_select] 391 | for word_id in top_ids: 392 | if y_label is None and len(rule_index) > 0 and word_id not in rule_index: continue 393 | open_nonterms = hyp.open_nonterms[:] 394 | if word_id >= self.hparams.target_word_vocab_size: 395 | rule = target_rule_vocab[word_id] 396 | # force the first rule to be not preterminal 397 | if hasattr(self.hparams, "force_rule") and self.hparams.force_rule: 398 | if length <= self.hparams.force_rule_step and rule.rhs[0] == "*": 399 | continue 400 | cur_nonterm = open_nonterms.pop() 401 | for c in reversed(rule.rhs): 402 | open_nonterms.append(OpenNonterm(label=c, parent_state=hyp.rule_hidden[0])) 403 | else: 404 | if open_nonterms[-1].label != '*': 405 | print(open_nonterms[-1].label, word_id, new_hyp_scores[word_id]) 406 | print(target_rule_vocab.rule_index_with_lhs(open_nonterms[-1].label)) 407 | print(top_ids) 408 | assert open_nonterms[-1].label == '*' 409 | if word_id == self.hparams.eos_id: 410 | open_nonterms.pop() 411 | new_hyp = TrAttnHyp(rule_hidden=(hyp.rule_hidden[0], hyp.rule_hidden[1]), 412 | word_hidden=(hyp.word_hidden[0], hyp.word_hidden[1]), 413 | y=hyp.y+[word_id], 414 | rule_ctx_tm1=hyp.rule_ctx_tm1, 415 | word_ctx_tm1=hyp.word_ctx_tm1, 416 | rule_to_word_ctx_tm1=hyp.rule_to_word_ctx_tm1, 417 | word_to_rule_ctx_tm1=hyp.word_to_rule_ctx_tm1, 418 | rule_states=hyp.rule_states, 419 | word_states=hyp.word_states, 420 | open_nonterms=open_nonterms, 421 | score=new_hyp_scores[word_id]) 422 | new_active_hyp.append(new_hyp) 423 | if y_label is None: 424 | live_hyp_num = beam_size - len(completed_hyp) 425 | new_active_hyp = sorted(new_active_hyp, key=lambda x:x.score, reverse=True)[:min(beam_size, live_hyp_num)] 426 | active_hyp = [] 427 | for hyp in new_active_hyp: 428 | if len(hyp.open_nonterms) == 0: 429 | #if poly_norm_m <= 0: 430 | # hyp.score = hyp.score / len(hyp.y) 431 | completed_hyp.append(hyp) 432 | else: 433 | active_hyp.append(hyp) 434 | else: 435 | active_hyp = new_active_hyp 436 | 437 | if len(completed_hyp) == 0: 438 | completed_hyp.append(active_hyp[0]) 439 | return sorted(completed_hyp, key=lambda x: x.score, reverse=True), nll_score 440 | 441 | class TrAttnHyp(object): 442 | def __init__(self, rule_hidden, word_hidden, y, 443 | rule_ctx_tm1, word_ctx_tm1, 444 | rule_to_word_ctx_tm1, word_to_rule_ctx_tm1, 445 | rule_states, word_states, score, open_nonterms): 446 | self.rule_hidden = rule_hidden 447 | self.word_hidden = word_hidden 448 | self.rule_states = rule_states 449 | self.word_states = word_states 450 | # [length_y, 2], each element (index, is_word) 451 | self.y = y 452 | self.rule_ctx_tm1 = rule_ctx_tm1 453 | self.word_ctx_tm1 = word_ctx_tm1 454 | self.rule_to_word_ctx_tm1 = rule_to_word_ctx_tm1 455 | self.word_to_rule_ctx_tm1 = word_to_rule_ctx_tm1 456 | self.score = score 457 | self.open_nonterms = open_nonterms 458 | 459 | class OpenNonterm(object): 460 | def __init__(self, label, parent_state=None): 461 | self.label = label 462 | self.parent_state = parent_state 463 | -------------------------------------------------------------------------------- /src/trdec_attn_v1.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.init as init 3 | from torch.autograd import Variable 4 | from torch import nn 5 | from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | from utils import * 10 | from models import * 11 | import gc 12 | 13 | class TreeDecoderAttn_v1(nn.Module): 14 | def __init__(self, hparams): 15 | super(TreeDecoderAttn_v1, self).__init__() 16 | self.hparams = hparams 17 | self.target_vocab_size = self.hparams.target_word_vocab_size+self.hparams.target_rule_vocab_size 18 | self.emb = nn.Embedding(self.target_vocab_size, 19 | self.hparams.d_word_vec, 20 | padding_idx=hparams.pad_id) 21 | if self.hparams.attn == "mlp": 22 | self.rule_attention = MlpAttn(hparams) 23 | self.word_attention = MlpAttn(hparams) 24 | self.rule_to_word_attn = MlpAttn(hparams) 25 | self.word_to_rule_attn = MlpAttn(hparams) 26 | else: 27 | self.rule_attention = DotProdAttn(hparams) 28 | self.word_attention = DotProdAttn(hparams) 29 | self.rule_to_word_attn = DotProdAttn(hparams) 30 | self.word_to_rule_attn = DotProdAttn(hparams) 31 | # transform [word_ctx, word_h_t, rule_ctx, rule_h_t] to readout state vectors before softmax 32 | self.rule_ctx_to_readout = nn.Linear(hparams.d_model * 6, hparams.d_model, bias=False) 33 | self.word_ctx_to_readout = nn.Linear(hparams.d_model * 6, hparams.d_model, bias=False) 34 | 35 | self.readout = nn.Linear(hparams.d_model, 36 | self.target_vocab_size, 37 | bias=False) 38 | if self.hparams.share_emb_softmax: 39 | self.emb.weight = self.readout.weight 40 | # input: [y_t-1, input_feed, rule_to_word, word_state_ctx] 41 | rule_inp = hparams.d_word_vec + hparams.d_model * 3 42 | # input: [y_t-1, input_feed, word_to_rule, rule_state_ctx] 43 | word_inp = hparams.d_word_vec + hparams.d_model * 3 44 | 45 | self.rule_lstm_cell = nn.LSTMCell(rule_inp, hparams.d_model) 46 | self.word_lstm_cell = nn.LSTMCell(word_inp, hparams.d_model) 47 | self.dropout = nn.Dropout(hparams.dropout) 48 | 49 | vocab_mask = torch.zeros(1, 1, self.target_vocab_size) 50 | self.word_vocab_mask = vocab_mask.index_fill_(2, torch.arange(self.hparams.target_word_vocab_size).long(), 1) 51 | self.rule_vocab_mask = 1 - self.word_vocab_mask 52 | if self.hparams.cuda: 53 | self.rule_vocab_mask = self.rule_vocab_mask.cuda() 54 | self.word_vocab_mask = self.word_vocab_mask.cuda() 55 | self.emb = self.emb.cuda() 56 | self.rule_attention = self.rule_attention.cuda() 57 | self.word_attention = self.word_attention.cuda() 58 | self.rule_to_word_attn = self.rule_to_word_attn.cuda() 59 | self.word_to_rule_attn = self.word_to_rule_attn.cuda() 60 | 61 | self.rule_ctx_to_readout = self.rule_ctx_to_readout.cuda() 62 | self.word_ctx_to_readout = self.word_ctx_to_readout.cuda() 63 | self.readout = self.readout.cuda() 64 | self.rule_lstm_cell = self.rule_lstm_cell.cuda() 65 | self.word_lstm_cell = self.word_lstm_cell.cuda() 66 | self.dropout = self.dropout.cuda() 67 | 68 | def forward(self, x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label=None): 69 | # get decoder init state and cell, use x_ct 70 | """ 71 | x_enc: [batch_size, max_x_len, d_model * 2] 72 | """ 73 | batch_size_x, x_max_len, d_x = x_enc.size() 74 | batch_size, y_max_len, data_len = y_train.size() 75 | assert batch_size_x == batch_size 76 | #print(y_train) 77 | input_feed_zeros = torch.zeros(batch_size, self.hparams.d_model * 2) 78 | input_feed_zeros_d = torch.zeros(batch_size, self.hparams.d_model) 79 | rule_states = Variable(torch.zeros(batch_size, y_max_len, self.hparams.d_model) ) 80 | word_states = Variable(torch.zeros(batch_size, y_max_len, self.hparams.d_model) ) 81 | rule_states_mask = torch.ones(batch_size, y_max_len).byte() 82 | word_states_mask = torch.ones(batch_size, y_max_len).byte() 83 | # avoid attn nan 84 | word_states_mask.index_fill_(1, torch.LongTensor([0]), 0) 85 | rule_states_mask.index_fill_(1, torch.LongTensor([0]), 0) 86 | if self.hparams.cuda: 87 | input_feed_zeros = input_feed_zeros.cuda() 88 | rule_states = rule_states.cuda() 89 | word_states = word_states.cuda() 90 | rule_states_mask = rule_states_mask.cuda() 91 | word_states_mask = word_states_mask.cuda() 92 | 93 | rule_hidden = dec_init 94 | word_hidden = dec_init 95 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 96 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 97 | rule_to_word_input_feed = Variable(input_feed_zeros_d, requires_grad=False) 98 | word_to_rule_input_feed = Variable(input_feed_zeros_d, requires_grad=False) 99 | if self.hparams.cuda: 100 | rule_input_feed = rule_input_feed.cuda() 101 | word_input_feed = word_input_feed.cuda() 102 | rule_to_word_input_feed = rule_to_word_input_feed.cuda() 103 | word_to_rule_input_feed = word_to_rule_input_feed.cuda() 104 | # [batch_size, y_len, d_word_vec] 105 | trg_emb = self.emb(y_train[:, :, 0]) 106 | logits = [] 107 | rule_pre_readouts = [] 108 | word_pre_readouts = [] 109 | offset = torch.arange(batch_size).long() 110 | if self.hparams.cuda: 111 | offset = offset.cuda() 112 | for t in range(y_max_len): 113 | y_emb_tm1 = trg_emb[:, t, :] 114 | word_mask = y_train[:, t, 2].unsqueeze(1).float() # (1 is word, 0 is rule) 115 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed], dim=1) 116 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 117 | 118 | word_h_t = word_h_t * word_mask + word_hidden[0] * (1-word_mask) 119 | word_c_t = word_c_t * word_mask + word_hidden[1] * (1-word_mask) 120 | 121 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, rule_to_word_input_feed], dim=1) 122 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 123 | 124 | #eos_mask = (y_train[:, t, 0] == self.hparams.eos_id).unsqueeze(1).float() 125 | #word_mask = word_mask - eos_mask 126 | #rule_h_t = rule_h_t * (1-word_mask) + rule_hidden[0] * word_mask 127 | #rule_c_t = rule_c_t * (1-word_mask) + rule_hidden[1] * word_mask 128 | 129 | rule_states.data[:, t, :] = rule_h_t.data 130 | word_states.data[:, t, :] = word_h_t.data 131 | # word_mask: 1 is word, 0 is rule 132 | if t > 0: 133 | rule_states_mask[:, t] = word_mask.byte().data 134 | word_states_mask[:, t] = (1-word_mask).byte().data 135 | #idx = torch.LongTensor([t]) 136 | #if self.hparams.cuda: 137 | # idx = idx.cuda() 138 | #rule_states_mask.index_fill_(1, idx, 0) 139 | rule_to_word_ctx = self.rule_to_word_attn(rule_h_t, word_states, word_states, attn_mask=word_states_mask) 140 | word_to_rule_ctx = self.word_to_rule_attn(word_h_t, rule_states, rule_states, attn_mask=rule_states_mask) 141 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 142 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 143 | 144 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 145 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 146 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 147 | 148 | rule_pre_readout = self.dropout(rule_pre_readout) 149 | word_pre_readout = self.dropout(word_pre_readout) 150 | 151 | rule_pre_readouts.append(rule_pre_readout) 152 | word_pre_readouts.append(word_pre_readout) 153 | 154 | rule_input_feed = rule_ctx 155 | word_input_feed = word_ctx 156 | rule_to_word_input_feed = rule_to_word_ctx 157 | word_to_rule_input_feed = word_to_rule_ctx 158 | 159 | rule_hidden = (rule_h_t, rule_c_t) 160 | word_hidden = (word_h_t, word_c_t) 161 | # [len_y, batch_size, trg_vocab_size] 162 | rule_readouts = self.readout(torch.stack(rule_pre_readouts))[:,:,-self.hparams.target_rule_vocab_size:] 163 | word_readouts = self.readout(torch.stack(word_pre_readouts))[:,:,:self.hparams.target_word_vocab_size] 164 | if self.hparams.label_smooth > 0: 165 | smooth = self.hparams.label_smooth 166 | rule_probs = (1.0 - smooth) * F.softmax(rule_readouts, dim=2) + smooth / self.hparams.target_rule_vocab_size 167 | rule_readouts = torch.log(rule_probs) 168 | # [batch_size, len_y, trg_vocab_size] 169 | logits = torch.cat([word_readouts, rule_readouts], dim=2).transpose(0, 1).contiguous() 170 | score_mask = score_mask.unsqueeze(2).float().data 171 | mask_t = self.word_vocab_mask * (1-score_mask) + self.rule_vocab_mask * score_mask 172 | logits.data.masked_fill_(mask_t.byte(), -float("inf")) 173 | return logits 174 | 175 | def step(self, x_enc, x_enc_k, hyp, target_rule_vocab): 176 | y_tm1 = torch.LongTensor([int(hyp.y[-1])]) 177 | if self.hparams.cuda: 178 | y_tm1 = y_tm1.cuda() 179 | y_tm1 = Variable(y_tm1, volatile=True) 180 | open_nonterms = hyp.open_nonterms 181 | rule_input_feed = hyp.rule_ctx_tm1 182 | word_input_feed = hyp.word_ctx_tm1 183 | rule_to_word_input_feed = hyp.rule_to_word_ctx_tm1 184 | word_to_rule_input_feed = hyp.word_to_rule_ctx_tm1 185 | rule_hidden = hyp.rule_hidden 186 | word_hidden = hyp.word_hidden 187 | word_h_t, word_c_t = word_hidden 188 | rule_h_t, rule_c_t = rule_hidden 189 | 190 | y_emb_tm1 = self.emb(y_tm1) 191 | cur_nonterm = open_nonterms[-1] 192 | 193 | if hyp.y[-1] < self.hparams.target_word_vocab_size: 194 | # word 195 | word_input = torch.cat([y_emb_tm1, word_input_feed, word_to_rule_input_feed], dim=1) 196 | word_h_t, word_c_t = self.word_lstm_cell(word_input, word_hidden) 197 | if hyp.word_states is None: 198 | hyp.word_states = word_h_t.unsqueeze(1) 199 | else: 200 | hyp.word_states = torch.cat([hyp.word_states, word_h_t.unsqueeze(1)], dim=1) 201 | rule_input = torch.cat([y_emb_tm1, rule_input_feed, rule_to_word_input_feed], dim=1) 202 | rule_h_t, rule_c_t = self.rule_lstm_cell(rule_input, rule_hidden) 203 | if hyp.rule_states is None: 204 | hyp.rule_states = rule_h_t.unsqueeze(1) 205 | else: 206 | if hyp.y[-1] >= self.hparams.target_word_vocab_size: 207 | hyp.rule_states = torch.cat([hyp.rule_states, rule_h_t.unsqueeze(1)], dim=1) 208 | hyp.rule_ctx_tm1 = self.rule_attention(rule_h_t, x_enc_k, x_enc) 209 | hyp.word_ctx_tm1 = self.word_attention(word_h_t, x_enc_k, x_enc) 210 | hyp.rule_to_word_ctx_tm1 = self.rule_to_word_attn(rule_h_t, hyp.word_states, hyp.word_states) 211 | hyp.word_to_rule_ctx_tm1 = self.word_to_rule_attn(word_h_t, hyp.rule_states, hyp.rule_states) 212 | 213 | inp = torch.cat([rule_h_t, hyp.rule_ctx_tm1, word_h_t, hyp.word_ctx_tm1], dim=1) 214 | mask = torch.ones(1, self.target_vocab_size).byte() 215 | if cur_nonterm.label == '*': 216 | word_index = torch.arange(self.hparams.target_word_vocab_size).long() 217 | mask.index_fill_(1, word_index, 0) 218 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 219 | #word_pre_readout = self.dropout(word_pre_readout) 220 | score_t = self.readout(word_pre_readout) 221 | num_rule_index = -1 222 | rule_select_index = [] 223 | else: 224 | rule_with_lhs = target_rule_vocab.rule_index_with_lhs(cur_nonterm.label) 225 | rule_select_index = [] 226 | for i in rule_with_lhs: rule_select_index.append(i + self.hparams.target_word_vocab_size) 227 | num_rule_index = len(rule_with_lhs) 228 | rule_index = torch.arange(self.hparams.target_rule_vocab_size).long() + self.hparams.target_word_vocab_size 229 | mask.index_fill_(1, rule_index, 0) 230 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 231 | #rule_pre_readout = self.dropout(rule_pre_readout) 232 | score_t = self.readout(rule_pre_readout) 233 | if self.hparams.cuda: 234 | mask = mask.cuda() 235 | score_t.data.masked_fill_(mask, -float("inf")) 236 | rule_hidden = (rule_h_t, rule_c_t) 237 | word_hidden = (word_h_t, word_c_t) 238 | 239 | hyp.rule_hidden = rule_hidden 240 | hyp.word_hidden = word_hidden 241 | 242 | return score_t, num_rule_index, rule_select_index 243 | 244 | class TrDecAttn_v1(nn.Module): 245 | def __init__(self, hparams): 246 | super(TrDecAttn_v1, self).__init__() 247 | self.encoder = Encoder(hparams) 248 | self.decoder = TreeDecoderAttn_v1(hparams) 249 | # transform encoder state vectors into attention key vector 250 | self.enc_to_k = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 251 | self.hparams = hparams 252 | if self.hparams.cuda: 253 | self.enc_to_k = self.enc_to_k.cuda() 254 | 255 | def forward(self, x_train, x_mask, x_len, y_train, y_mask, y_len, score_mask, y_label=None): 256 | # [batch_size, x_len, d_model * 2] 257 | x_enc, dec_init = self.encoder(x_train, x_len) 258 | x_enc_k = self.enc_to_k(x_enc) 259 | # [batch_size, y_len-1, trg_vocab_size] 260 | logits = self.decoder(x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label) 261 | return logits 262 | 263 | def translate(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=0): 264 | hyps = [] 265 | scores = [] 266 | i = 0 267 | self.logsoftmax = nn.LogSoftmax(dim=1) 268 | for x in x_train: 269 | x = Variable(torch.LongTensor(x), volatile=True) 270 | if y_label: 271 | y = y_label[i][1:] 272 | else: 273 | y = None 274 | if self.hparams.cuda: 275 | x = x.cuda() 276 | hyp, nll_score = self.translate_sent(x, target_rule_vocab, max_len=max_len, beam_size=beam_size, y_label=y, poly_norm_m=poly_norm_m) 277 | hyp = hyp[0] 278 | hyps.append(hyp.y[1:]) 279 | scores.append(sum(nll_score)) 280 | #print(hyp.y) 281 | #print("trans score:", nll_score) 282 | #print("trans label:", y) 283 | i += 1 284 | gc.collect() 285 | return hyps, scores 286 | 287 | def translate_sent(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=1.): 288 | assert len(x_train.size()) == 1 289 | x_len = [x_train.size(0)] 290 | x_train = x_train.unsqueeze(0) 291 | x_enc, dec_init = self.encoder(x_train, x_len) 292 | x_enc_k = self.enc_to_k(x_enc) 293 | length = 0 294 | completed_hyp = [] 295 | input_feed_zeros = torch.zeros(1, self.hparams.d_model * 2) 296 | to_input_feed_zeros = torch.zeros(1, self.hparams.d_model) 297 | #state_zeros = torch.zeros(1, 1, self.hparams.d_model) 298 | if self.hparams.cuda: 299 | input_feed_zeros = input_feed_zeros.cuda() 300 | to_input_feed_zeros = to_input_feed_zeros.cuda() 301 | #state_zeros = state_zeros.cuda() 302 | active_hyp = [TrAttnHyp(rule_hidden=dec_init, 303 | word_hidden=dec_init, 304 | y=[self.hparams.bos_id], 305 | rule_ctx_tm1= Variable(input_feed_zeros, requires_grad=False), 306 | word_ctx_tm1=Variable(input_feed_zeros, requires_grad=False), 307 | rule_to_word_ctx_tm1=Variable(to_input_feed_zeros, requires_grad=False), 308 | word_to_rule_ctx_tm1= Variable(to_input_feed_zeros, requires_grad=False), 309 | rule_states=None, 310 | word_states=None, 311 | open_nonterms=[OpenNonterm(label=self.hparams.root_label)], 312 | score=0.)] 313 | nll_score = [] 314 | if y_label is not None: 315 | max_len = len(y_label) 316 | while len(completed_hyp) < beam_size and length < max_len: 317 | length += 1 318 | new_active_hyp = [] 319 | for i, hyp in enumerate(active_hyp): 320 | logits, num_rule_index, rule_index = self.decoder.step(x_enc, 321 | x_enc_k, hyp, target_rule_vocab) 322 | 323 | rule_index = set(rule_index) 324 | logits = logits.view(-1) 325 | p_t = F.log_softmax(logits, dim=0).data 326 | if poly_norm_m > 0 and length > 1: 327 | new_hyp_scores = (hyp.score * pow(length-1, poly_norm_m) + p_t) / pow(length, poly_norm_m) 328 | else: 329 | new_hyp_scores = hyp.score + p_t 330 | if y_label is not None: 331 | top_ids = [y_label[length-1][0]] 332 | nll = -(p_t[top_ids[0]]) 333 | nll_score.append(nll) 334 | print("logit dedcode", logits[top_ids[0]]) 335 | else: 336 | num_select = beam_size 337 | if num_rule_index >= 0: num_select = min(num_select, num_rule_index) 338 | top_ids = (-new_hyp_scores).cpu().numpy().argsort()[:num_select] 339 | for word_id in top_ids: 340 | if y_label is None and len(rule_index) > 0 and word_id not in rule_index: continue 341 | open_nonterms = hyp.open_nonterms[:] 342 | if word_id >= self.hparams.target_word_vocab_size: 343 | rule = target_rule_vocab[word_id] 344 | cur_nonterm = open_nonterms.pop() 345 | for c in reversed(rule.rhs): 346 | open_nonterms.append(OpenNonterm(label=c)) 347 | else: 348 | if open_nonterms[-1].label != '*': 349 | print(open_nonterms[-1].label, word_id, new_hyp_scores[word_id]) 350 | print(target_rule_vocab.rule_index_with_lhs(open_nonterms[-1].label)) 351 | print(top_ids) 352 | assert open_nonterms[-1].label == '*' 353 | if word_id == self.hparams.eos_id: 354 | open_nonterms.pop() 355 | new_hyp = TrAttnHyp(rule_hidden=(hyp.rule_hidden[0], hyp.rule_hidden[1]), 356 | word_hidden=(hyp.word_hidden[0], hyp.word_hidden[1]), 357 | y=hyp.y+[word_id], 358 | rule_ctx_tm1=hyp.rule_ctx_tm1, 359 | word_ctx_tm1=hyp.word_ctx_tm1, 360 | rule_to_word_ctx_tm1=hyp.rule_to_word_ctx_tm1, 361 | word_to_rule_ctx_tm1=hyp.word_to_rule_ctx_tm1, 362 | rule_states=hyp.rule_states, 363 | word_states=hyp.word_states, 364 | open_nonterms=open_nonterms, 365 | score=new_hyp_scores[word_id]) 366 | new_active_hyp.append(new_hyp) 367 | if y_label is None: 368 | live_hyp_num = beam_size - len(completed_hyp) 369 | new_active_hyp = sorted(new_active_hyp, key=lambda x:x.score, reverse=True)[:min(beam_size, live_hyp_num)] 370 | active_hyp = [] 371 | for hyp in new_active_hyp: 372 | if len(hyp.open_nonterms) == 0: 373 | #if poly_norm_m <= 0: 374 | # hyp.score = hyp.score / len(hyp.y) 375 | completed_hyp.append(hyp) 376 | else: 377 | active_hyp.append(hyp) 378 | else: 379 | active_hyp = new_active_hyp 380 | 381 | if len(completed_hyp) == 0: 382 | completed_hyp.append(active_hyp[0]) 383 | return sorted(completed_hyp, key=lambda x: x.score, reverse=True), nll_score 384 | 385 | class TrAttnHyp(object): 386 | def __init__(self, rule_hidden, word_hidden, y, 387 | rule_ctx_tm1, word_ctx_tm1, 388 | rule_to_word_ctx_tm1, word_to_rule_ctx_tm1, 389 | rule_states, word_states, score, open_nonterms): 390 | self.rule_hidden = rule_hidden 391 | self.word_hidden = word_hidden 392 | self.rule_states = rule_states 393 | self.word_states = word_states 394 | # [length_y, 2], each element (index, is_word) 395 | self.y = y 396 | self.rule_ctx_tm1 = rule_ctx_tm1 397 | self.word_ctx_tm1 = word_ctx_tm1 398 | self.rule_to_word_ctx_tm1 = rule_to_word_ctx_tm1 399 | self.word_to_rule_ctx_tm1 = word_to_rule_ctx_tm1 400 | self.score = score 401 | self.open_nonterms = open_nonterms 402 | 403 | class OpenNonterm(object): 404 | def __init__(self, label, parent_state=None): 405 | self.label = label 406 | self.parent_state = parent_state 407 | -------------------------------------------------------------------------------- /src/trdec_single.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.init as init 3 | from torch.autograd import Variable 4 | from torch import nn 5 | from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence 6 | import torch.nn.functional as F 7 | 8 | import numpy as np 9 | from utils import * 10 | from models import * 11 | import gc 12 | 13 | class TreeDecoderSingle(nn.Module): 14 | def __init__(self, hparams): 15 | super(TreeDecoderSingle, self).__init__() 16 | self.hparams = hparams 17 | self.target_vocab_size = self.hparams.target_word_vocab_size+self.hparams.target_rule_vocab_size 18 | self.emb = nn.Embedding(self.target_vocab_size, 19 | self.hparams.d_word_vec, 20 | padding_idx=hparams.pad_id) 21 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 22 | if self.hparams.attn == "mlp": 23 | self.attention = MlpAttn(hparams) 24 | else: 25 | self.attention = DotProdAttn(hparams) 26 | else: 27 | if self.hparams.attn == "mlp": 28 | self.rule_attention = MlpAttn(hparams) 29 | self.word_attention = MlpAttn(hparams) 30 | else: 31 | self.rule_attention = DotProdAttn(hparams) 32 | self.word_attention = DotProdAttn(hparams) 33 | # transform [word_ctx, word_h_t, rule_ctx, rule_h_t] to readout state vectors before softmax 34 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 35 | inp_dim = hparams.d_model * 3 36 | else: 37 | inp_dim = hparams.d_model * 6 38 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 39 | self.ctx_to_readout = nn.Linear(inp_dim, hparams.d_model, bias=False) 40 | else: 41 | self.rule_ctx_to_readout = nn.Linear(inp_dim, hparams.d_model, bias=False) 42 | self.word_ctx_to_readout = nn.Linear(hparams.d_model * 6, hparams.d_model, bias=False) 43 | 44 | self.readout = nn.Linear(hparams.d_model, 45 | self.target_vocab_size, 46 | bias=False) 47 | if self.hparams.share_emb_softmax: 48 | self.emb.weight = self.readout.weight 49 | # input: [y_t-1, parent_state, input_feed] 50 | inp = hparams.d_word_vec + hparams.d_model * 3 51 | # input: [y_t-1, parent_state, input_feed] 52 | self.lstm_cell = nn.LSTMCell(inp, hparams.d_model) 53 | self.dropout = nn.Dropout(hparams.dropout) 54 | 55 | vocab_mask = torch.zeros(1, 1, self.target_vocab_size) 56 | self.word_vocab_mask = vocab_mask.index_fill_(2, torch.arange(self.hparams.target_word_vocab_size).long(), 1) 57 | self.rule_vocab_mask = 1 - self.word_vocab_mask 58 | if self.hparams.cuda: 59 | self.rule_vocab_mask = self.rule_vocab_mask.cuda() 60 | self.word_vocab_mask = self.word_vocab_mask.cuda() 61 | self.emb = self.emb.cuda() 62 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 63 | self.attention = self.attention.cuda() 64 | else: 65 | self.rule_attention = self.rule_attention.cuda() 66 | self.word_attention = self.word_attention.cuda() 67 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 68 | self.ctx_to_readout = self.ctx_to_readout.cuda() 69 | else: 70 | self.rule_ctx_to_readout = self.rule_ctx_to_readout.cuda() 71 | self.word_ctx_to_readout = self.word_ctx_to_readout.cuda() 72 | self.readout = self.readout.cuda() 73 | self.lstm_cell = self.lstm_cell.cuda() 74 | self.dropout = self.dropout.cuda() 75 | 76 | def forward(self, x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label=None): 77 | # get decoder init state and cell, use x_ct 78 | """ 79 | x_enc: [batch_size, max_x_len, d_model * 2] 80 | """ 81 | batch_size_x = x_enc.size()[0] 82 | batch_size, y_max_len, data_len = y_train.size() 83 | assert batch_size_x == batch_size 84 | #print(y_train) 85 | input_feed_zeros = torch.zeros(batch_size, self.hparams.d_model * 2) 86 | if self.hparams.cuda: 87 | input_feed_zeros = input_feed_zeros.cuda() 88 | rule_hidden = dec_init 89 | word_hidden = dec_init 90 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 91 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 92 | if self.hparams.cuda: 93 | rule_input_feed = rule_input_feed.cuda() 94 | word_input_feed = word_input_feed.cuda() 95 | # [batch_size, y_len, d_word_vec] 96 | trg_emb = self.emb(y_train[:, :, 0]) 97 | logits = [] 98 | rule_pre_readouts = [] 99 | word_pre_readouts = [] 100 | if self.hparams.parent_feed: 101 | all_state = Variable(torch.zeros(batch_size, self.hparams.d_model), requires_grad=False) 102 | offset = torch.arange(batch_size).long() 103 | if self.hparams.cuda: 104 | offset = offset.cuda() 105 | if self.hparams.parent_feed: 106 | all_state = all_state.cuda() 107 | for t in range(y_max_len): 108 | y_emb_tm1 = trg_emb[:, t, :] 109 | 110 | if self.hparams.parent_feed: 111 | state_idx_t = t 112 | state_idx_t += 1 113 | parent_t = y_train.data[:, t, 1] + state_idx_t * offset # [batch_size,] 114 | parent_t = Variable(parent_t, requires_grad=False) 115 | parent_state = torch.index_select(all_state.view(state_idx_t*batch_size, self.hparams.d_model), dim=0, index=parent_t) # [batch_size, d_model] 116 | 117 | word_mask = y_train[:, t, 2].unsqueeze(1).float() # (1 is word, 0 is rule) 118 | 119 | if self.hparams.parent_feed: 120 | word_input = torch.cat([y_emb_tm1, parent_state, word_input_feed], dim=1) 121 | else: 122 | word_input = torch.cat([y_emb_tm1, rule_hidden[0], word_input_feed], dim=1) 123 | word_h_t, word_c_t = self.lstm_cell(word_input, word_hidden) 124 | 125 | word_h_t = word_h_t * word_mask + word_hidden[0] * (1-word_mask) 126 | word_c_t = word_c_t * word_mask + word_hidden[1] * (1-word_mask) 127 | 128 | if self.hparams.rule_parent_feed: 129 | rule_input = torch.cat([y_emb_tm1, parent_state, rule_input_feed], dim=1) 130 | else: 131 | rule_input = torch.cat([y_emb_tm1, rule_input_feed], dim=1) 132 | rule_h_t, rule_c_t = self.lstm_cell(rule_input, rule_hidden) 133 | 134 | if hasattr(self.hparams, "no_word_to_rule") and self.hparams.no_word_to_rule: 135 | eos_mask = (y_train[:, t, 0] == self.hparams.eos_id).unsqueeze(1).float() 136 | word_mask = word_mask - eos_mask 137 | rule_h_t = rule_h_t * (1-word_mask) + rule_hidden[0] * word_mask 138 | rule_c_t = rule_c_t * (1-word_mask) + rule_hidden[1] * word_mask 139 | 140 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 141 | rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 142 | word_ctx = self.attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 143 | else: 144 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc, attn_mask=x_mask) 145 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc, attn_mask=x_mask) 146 | 147 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 148 | rule_pre_readout = F.tanh(self.ctx_to_readout(torch.cat([rule_h_t, rule_ctx], dim=1))) 149 | word_pre_readout = F.tanh(self.ctx_to_readout(torch.cat([word_h_t, word_ctx], dim=1))) 150 | else: 151 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 152 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 153 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 154 | 155 | rule_pre_readout = self.dropout(rule_pre_readout) 156 | word_pre_readout = self.dropout(word_pre_readout) 157 | 158 | rule_pre_readouts.append(rule_pre_readout) 159 | word_pre_readouts.append(word_pre_readout) 160 | 161 | #inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 162 | #if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 163 | # pre_readout = F.tanh(self.ctx_to_readout(r_inp)) 164 | # pre_readout = self.dropout(pre_readout) 165 | # pre_readouts.append(pre_readout) 166 | #else: 167 | # rule_pre_readout = F.tanh(self.rule_ctx_to_readout(r_inp)) 168 | # word_pre_readout = F.tanh(self.word_ctx_to_readout(w_inp)) 169 | # 170 | # rule_pre_readout = self.dropout(rule_pre_readout) 171 | # word_pre_readout = self.dropout(word_pre_readout) 172 | # 173 | # rule_pre_readouts.append(rule_pre_readout) 174 | # word_pre_readouts.append(word_pre_readout) 175 | 176 | rule_input_feed = rule_ctx 177 | word_input_feed = word_ctx 178 | 179 | rule_hidden = (rule_h_t, rule_c_t) 180 | word_hidden = (word_h_t, word_c_t) 181 | if self.hparams.parent_feed: 182 | all_state = torch.cat([all_state, rule_h_t], dim=1) 183 | # [len_y, batch_size, trg_vocab_size] 184 | rule_readouts = self.readout(torch.stack(rule_pre_readouts))[:,:,-self.hparams.target_rule_vocab_size:] 185 | if hasattr(self.hparams, "rule_tanh") and self.hparams.rule_tanh > 0: 186 | rule_readouts = self.hparams.rule_tanh * torch.tanh(rule_readouts) 187 | word_readouts = self.readout(torch.stack(word_pre_readouts))[:,:,:self.hparams.target_word_vocab_size] 188 | if self.hparams.label_smooth > 0: 189 | smooth = self.hparams.label_smooth 190 | rule_probs = (1.0 - smooth) * F.softmax(rule_readouts, dim=2) + smooth / self.hparams.target_rule_vocab_size 191 | rule_readouts = torch.log(rule_probs) 192 | # [batch_size, len_y, trg_vocab_size] 193 | logits = torch.cat([word_readouts, rule_readouts], dim=2).transpose(0, 1).contiguous() 194 | score_mask = score_mask.unsqueeze(2).float().data 195 | mask_t = self.word_vocab_mask * (1-score_mask) + self.rule_vocab_mask * score_mask 196 | logits.data.masked_fill_(mask_t.byte(), -float("inf")) 197 | return logits 198 | 199 | def step(self, x_enc, x_enc_k, hyp, target_rule_vocab): 200 | y_tm1 = torch.LongTensor([int(hyp.y[-1])]) 201 | if self.hparams.cuda: 202 | y_tm1 = y_tm1.cuda() 203 | y_tm1 = Variable(y_tm1, volatile=True) 204 | open_nonterms = hyp.open_nonterms 205 | rule_input_feed = hyp.rule_ctx_tm1 206 | word_input_feed = hyp.word_ctx_tm1 207 | rule_hidden = hyp.rule_hidden 208 | word_hidden = hyp.word_hidden 209 | word_h_t, word_c_t = word_hidden 210 | rule_h_t, rule_c_t = rule_hidden 211 | 212 | y_emb_tm1 = self.emb(y_tm1) 213 | cur_nonterm = open_nonterms[-1] 214 | parent_state = cur_nonterm.parent_state 215 | 216 | if hyp.y[-1] < self.hparams.target_word_vocab_size: 217 | # word 218 | if self.hparams.parent_feed: 219 | word_input = torch.cat([y_emb_tm1, parent_state, word_input_feed], dim=1) 220 | else: 221 | word_input = torch.cat([y_emb_tm1, rule_h_t, word_input_feed], dim=1) 222 | word_h_t, word_c_t = self.lstm_cell(word_input, word_hidden) 223 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 224 | word_ctx = self.attention(word_h_t, x_enc_k, x_enc) 225 | else: 226 | word_ctx = self.word_attention(word_h_t, x_enc_k, x_enc) 227 | else: 228 | word_ctx = hyp.word_ctx_tm1 229 | 230 | update_rule_rnn = True 231 | if hasattr(self.hparams, "no_word_to_rule") and self.hparams.no_word_to_rule and hyp.y[-1] < self.hparams.target_word_vocab_size: 232 | if hyp.y[-1] != self.hparams.eos_id: 233 | update_rule_rnn = False 234 | 235 | if update_rule_rnn: 236 | if self.hparams.rule_parent_feed: 237 | rule_input = torch.cat([y_emb_tm1, parent_state, rule_input_feed], dim=1) 238 | else: 239 | rule_input = torch.cat([y_emb_tm1, rule_input_feed], dim=1) 240 | rule_h_t, rule_c_t = self.lstm_cell(rule_input, rule_hidden) 241 | # if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 242 | # rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc) 243 | # else: 244 | # rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc) 245 | #else: 246 | # rule_ctx = hyp.rule_ctx_tm1 247 | if hasattr(self.hparams, "single_attn") and self.hparams.single_attn: 248 | rule_ctx = self.attention(rule_h_t, x_enc_k, x_enc) 249 | else: 250 | rule_ctx = self.rule_attention(rule_h_t, x_enc_k, x_enc) 251 | 252 | mask = torch.ones(1, self.target_vocab_size).byte() 253 | if cur_nonterm.label == '*': 254 | word_index = torch.arange(self.hparams.target_word_vocab_size).long() 255 | mask.index_fill_(1, word_index, 0) 256 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 257 | inp = torch.cat([word_h_t, word_ctx], dim=1) 258 | else: 259 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 260 | #inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 261 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 262 | word_pre_readout = F.tanh(self.ctx_to_readout(inp)) 263 | else: 264 | word_pre_readout = F.tanh(self.word_ctx_to_readout(inp)) 265 | word_pre_readout = self.dropout(word_pre_readout) 266 | score_t = self.readout(word_pre_readout) 267 | num_rule_index = -1 268 | rule_select_index = [] 269 | else: 270 | rule_with_lhs = target_rule_vocab.rule_index_with_lhs(cur_nonterm.label) 271 | rule_select_index = [] 272 | for i in rule_with_lhs: rule_select_index.append(i + self.hparams.target_word_vocab_size) 273 | num_rule_index = len(rule_with_lhs) 274 | rule_index = torch.arange(self.hparams.target_rule_vocab_size).long() + self.hparams.target_word_vocab_size 275 | mask.index_fill_(1, rule_index, 0) 276 | if hasattr(self.hparams, "single_inp_readout") and self.hparams.single_inp_readout: 277 | inp = torch.cat([rule_h_t, rule_ctx], dim=1) 278 | else: 279 | inp = torch.cat([rule_h_t, rule_ctx, word_h_t, word_ctx], dim=1) 280 | if hasattr(self.hparams, "single_readout") and self.hparams.single_readout: 281 | rule_pre_readout = F.tanh(self.ctx_to_readout(inp)) 282 | else: 283 | rule_pre_readout = F.tanh(self.rule_ctx_to_readout(inp)) 284 | #rule_pre_readout = self.dropout(rule_pre_readout) 285 | score_t = self.readout(rule_pre_readout) 286 | if self.hparams.cuda: 287 | mask = mask.cuda() 288 | score_t.data.masked_fill_(mask, -float("inf")) 289 | rule_hidden = (rule_h_t, rule_c_t) 290 | word_hidden = (word_h_t, word_c_t) 291 | return score_t, rule_hidden, word_hidden, rule_ctx, word_ctx, num_rule_index, rule_select_index 292 | 293 | class TrDecSingle(nn.Module): 294 | 295 | def __init__(self, hparams): 296 | super(TrDecSingle, self).__init__() 297 | self.encoder = Encoder(hparams) 298 | self.decoder = TreeDecoderSingle(hparams) 299 | # transform encoder state vectors into attention key vector 300 | self.enc_to_k = nn.Linear(hparams.d_model * 2, hparams.d_model, bias=False) 301 | self.hparams = hparams 302 | if self.hparams.cuda: 303 | self.enc_to_k = self.enc_to_k.cuda() 304 | 305 | def forward(self, x_train, x_mask, x_len, y_train, y_mask, y_len, score_mask, y_label=None): 306 | # [batch_size, x_len, d_model * 2] 307 | x_enc, dec_init = self.encoder(x_train, x_len) 308 | x_enc_k = self.enc_to_k(x_enc) 309 | # [batch_size, y_len-1, trg_vocab_size] 310 | logits = self.decoder(x_enc, x_enc_k, dec_init, x_mask, y_train, y_mask, score_mask, y_label) 311 | return logits 312 | 313 | def translate(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=0): 314 | hyps = [] 315 | scores = [] 316 | i = 0 317 | self.logsoftmax = nn.LogSoftmax(dim=1) 318 | for x in x_train: 319 | x = Variable(torch.LongTensor(x), volatile=True) 320 | if y_label: 321 | y = y_label[i][1:] 322 | else: 323 | y = None 324 | if self.hparams.cuda: 325 | x = x.cuda() 326 | hyp, nll_score = self.translate_sent(x, target_rule_vocab, max_len=max_len, beam_size=beam_size, y_label=y, poly_norm_m=poly_norm_m) 327 | hyp = hyp[0] 328 | hyps.append(hyp.y[1:]) 329 | scores.append(sum(nll_score)) 330 | #print(hyp.y) 331 | #print("trans score:", nll_score) 332 | #print("trans label:", y) 333 | i += 1 334 | gc.collect() 335 | return hyps, scores 336 | 337 | def translate_sent(self, x_train, target_rule_vocab, max_len=100, beam_size=5, y_label=None, poly_norm_m=1.): 338 | assert len(x_train.size()) == 1 339 | x_len = [x_train.size(0)] 340 | x_train = x_train.unsqueeze(0) 341 | x_enc, dec_init = self.encoder(x_train, x_len) 342 | x_enc_k = self.enc_to_k(x_enc) 343 | length = 0 344 | completed_hyp = [] 345 | input_feed_zeros = torch.zeros(1, self.hparams.d_model * 2) 346 | state_zeros = torch.zeros(1, self.hparams.d_model) 347 | if self.hparams.cuda: 348 | input_feed_zeros = input_feed_zeros.cuda() 349 | state_zeros = state_zeros.cuda() 350 | rule_input_feed = Variable(input_feed_zeros, requires_grad=False) 351 | word_input_feed = Variable(input_feed_zeros, requires_grad=False) 352 | active_hyp = [TrHyp(rule_hidden=dec_init, 353 | word_hidden=dec_init, 354 | y=[self.hparams.bos_id], 355 | rule_ctx_tm1=rule_input_feed, 356 | word_ctx_tm1=word_input_feed, 357 | open_nonterms=[OpenNonterm(label=self.hparams.root_label, 358 | parent_state=Variable(state_zeros, requires_grad=False))], 359 | score=0.)] 360 | nll_score = [] 361 | if y_label is not None: 362 | max_len = len(y_label) 363 | while len(completed_hyp) < beam_size and length < max_len: 364 | length += 1 365 | new_active_hyp = [] 366 | for i, hyp in enumerate(active_hyp): 367 | logits, rule_hidden, word_hidden, rule_ctx, word_ctx, num_rule_index, rule_index = self.decoder.step(x_enc, 368 | x_enc_k, hyp, target_rule_vocab) 369 | hyp.rule_hidden = rule_hidden 370 | hyp.word_hidden = word_hidden 371 | hyp.rule_ctx_tm1 = rule_ctx 372 | hyp.word_ctx_tm1 = word_ctx 373 | 374 | rule_index = set(rule_index) 375 | logits = logits.view(-1) 376 | p_t = F.log_softmax(logits, dim=0).data 377 | if hasattr(self.hparams, "ignore_rule_len") and self.hparams.ignore_rule_len: 378 | l = (np.array(hyp.y) < self.hparams.target_word_vocab_size).sum() 379 | if poly_norm_m > 0 and l > 1: 380 | new_hyp_scores = (hyp.score * pow(l-1, poly_norm_m) + p_t) / pow(l, poly_norm_m) 381 | else: 382 | new_hyp_scores = hyp.score + p_t 383 | else: 384 | if poly_norm_m > 0 and length > 1: 385 | new_hyp_scores = (hyp.score * pow(length-1, poly_norm_m) + p_t) / pow(length, poly_norm_m) 386 | else: 387 | new_hyp_scores = hyp.score + p_t 388 | if y_label is not None: 389 | top_ids = [y_label[length-1][0]] 390 | nll = -(p_t[top_ids[0]]) 391 | nll_score.append(nll) 392 | print(logits[top_ids[0]]) 393 | else: 394 | num_select = beam_size 395 | if num_rule_index >= 0: num_select = min(num_select, num_rule_index) 396 | top_ids = (-new_hyp_scores).cpu().numpy().argsort()[:num_select] 397 | for word_id in top_ids: 398 | if y_label is None and len(rule_index) > 0 and word_id not in rule_index: continue 399 | open_nonterms = hyp.open_nonterms[:] 400 | if word_id >= self.hparams.target_word_vocab_size: 401 | rule = target_rule_vocab[word_id] 402 | # force the first rule to be not preterminal 403 | if hasattr(self.hparams, "force_rule") and self.hparams.force_rule: 404 | if length <= self.hparams.force_rule_step and rule.rhs[0] == "*": 405 | continue 406 | cur_nonterm = open_nonterms.pop() 407 | parent_state = hyp.rule_hidden[0] 408 | for c in reversed(rule.rhs): 409 | open_nonterms.append(OpenNonterm(label=c, parent_state=parent_state)) 410 | else: 411 | if open_nonterms[-1].label != '*': 412 | print(open_nonterms[-1].label, word_id, new_hyp_scores[word_id]) 413 | print(target_rule_vocab.rule_index_with_lhs(open_nonterms[-1].label)) 414 | print(top_ids) 415 | assert open_nonterms[-1].label == '*' 416 | if word_id == self.hparams.eos_id: 417 | open_nonterms.pop() 418 | new_hyp = TrHyp(rule_hidden=(hyp.rule_hidden[0], hyp.rule_hidden[1]), 419 | word_hidden=(hyp.word_hidden[0], hyp.word_hidden[1]), 420 | y=hyp.y+[word_id], 421 | rule_ctx_tm1=hyp.rule_ctx_tm1, 422 | word_ctx_tm1=hyp.word_ctx_tm1, 423 | open_nonterms=open_nonterms, 424 | score=new_hyp_scores[word_id], 425 | c_p=p_t[word_id]) 426 | new_active_hyp.append(new_hyp) 427 | if y_label is None: 428 | live_hyp_num = beam_size - len(completed_hyp) 429 | new_active_hyp = sorted(new_active_hyp, key=lambda x:x.score, reverse=True)[:min(beam_size, live_hyp_num)] 430 | active_hyp = [] 431 | for hyp in new_active_hyp: 432 | if len(hyp.open_nonterms) == 0: 433 | #if poly_norm_m <= 0: 434 | # hyp.score = hyp.score / len(hyp.y) 435 | completed_hyp.append(hyp) 436 | else: 437 | active_hyp.append(hyp) 438 | else: 439 | active_hyp = new_active_hyp 440 | 441 | if len(completed_hyp) == 0: 442 | completed_hyp.append(active_hyp[0]) 443 | return sorted(completed_hyp, key=lambda x: x.score, reverse=True), nll_score 444 | 445 | class TrHyp(object): 446 | def __init__(self, rule_hidden, word_hidden, y, rule_ctx_tm1, word_ctx_tm1, score, open_nonterms, c_p=0): 447 | self.rule_hidden = rule_hidden 448 | self.word_hidden = word_hidden 449 | # [length_y, 2], each element (index, is_word) 450 | self.y = y 451 | self.rule_ctx_tm1 = rule_ctx_tm1 452 | self.word_ctx_tm1 = word_ctx_tm1 453 | self.score = score 454 | self.open_nonterms = open_nonterms 455 | self.c_p = c_p 456 | 457 | class OpenNonterm(object): 458 | def __init__(self, label, parent_state): 459 | self.label = label 460 | self.parent_state = parent_state 461 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import gc 5 | 6 | from datetime import datetime 7 | 8 | import numpy as np 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.init as init 12 | 13 | def memReport(): 14 | for obj in gc.get_objects(): 15 | if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)): 16 | print(type(obj), obj.size()) 17 | 18 | def get_criterion(hparams): 19 | loss_reduce = True 20 | if hparams.trdec: 21 | loss_reduce = False 22 | crit = nn.CrossEntropyLoss(ignore_index=hparams.pad_id, size_average=False, reduce=loss_reduce) 23 | if hparams.cuda: 24 | crit = crit.cuda() 25 | return crit 26 | 27 | def get_performance(crit, logits, labels, hparams, sum_loss=True): 28 | mask = (labels == hparams.pad_id) 29 | if hparams.trdec: 30 | rule_mask = (labels >= hparams.target_word_vocab_size) 31 | eos_mask = (labels == hparams.eos_id) 32 | word_mask = (labels < hparams.target_word_vocab_size) ^ eos_mask ^ mask 33 | 34 | #labels = labels - rule_mask.long() * hparams.target_word_vocab_size 35 | loss = crit(logits, labels) 36 | _, preds = torch.max(logits, dim=1) 37 | acc = torch.eq(preds, labels).int().masked_fill_(mask, 0).sum() 38 | 39 | rule_loss = loss[rule_mask].sum() 40 | eos_loss = loss[eos_mask].sum() 41 | word_loss = loss[word_mask].sum() 42 | if sum_loss: loss = loss.sum() 43 | return loss, acc, rule_loss, word_loss, eos_loss, rule_mask.long().sum(), word_mask.long().sum(), eos_mask.long().sum() 44 | else: 45 | loss = crit(logits, labels) 46 | _, preds = torch.max(logits, dim=1) 47 | acc = torch.eq(preds, labels).int().masked_fill_(mask, 0).sum() 48 | if sum_loss: loss = loss.sum() 49 | return loss, acc 50 | 51 | def count_params(params): 52 | num_params = sum(p.data.nelement() for p in params) 53 | return num_params 54 | 55 | def save_checkpoint(extra, model, optimizer, hparams, path): 56 | print("Saving model to '{0}'".format(path)) 57 | torch.save(extra, os.path.join(path, "extra.pt")) 58 | torch.save(model, os.path.join(path, "model.pt")) 59 | torch.save(optimizer.state_dict(), os.path.join(path, "optimizer.pt")) 60 | torch.save(hparams, os.path.join(path, "hparams.pt")) 61 | 62 | class Logger(object): 63 | def __init__(self, output_file): 64 | self.terminal = sys.stdout 65 | self.log = open(output_file, "a") 66 | 67 | def write(self, message): 68 | print(message, end="", file=self.terminal, flush=True) 69 | print(message, end="", file=self.log, flush=True) 70 | 71 | def flush(self): 72 | self.terminal.flush() 73 | self.log.flush() 74 | 75 | def set_lr(optim, lr): 76 | for param_group in optim.param_groups: 77 | param_group["lr"] = lr 78 | 79 | def init_param(p, init_type="uniform", init_range=None): 80 | if init_type == "xavier_normal": 81 | init.xavier_normal(p) 82 | elif init_type == "xavier_uniform": 83 | init.xavier_uniform(p) 84 | elif init_type == "kaiming_normal": 85 | init.kaiming_normal(p) 86 | elif init_type == "kaiming_uniform": 87 | init.kaiming_uniform(p) 88 | elif init_type == "uniform": 89 | assert init_range is not None and init_range > 0 90 | init.uniform(p, -init_range, init_range) 91 | else: 92 | raise ValueError("Unknown init_type '{0}'".format(init_type)) 93 | 94 | def add_argument(parser, name, type, default, help): 95 | """Add an argument. 96 | 97 | Args: 98 | name: arg's name. 99 | type: must be ["bool", "int", "float", "str"]. 100 | default: corresponding type of value. 101 | help: help message. 102 | """ 103 | 104 | if type == "bool": 105 | feature_parser = parser.add_mutually_exclusive_group(required=False) 106 | feature_parser.add_argument("--{0}".format(name), dest=name, 107 | action="store_true", help=help) 108 | feature_parser.add_argument("--no_{0}".format(name), dest=name, 109 | action="store_false", help=help) 110 | parser.set_defaults(name=default) 111 | elif type == "int": 112 | parser.add_argument("--{0}".format(name), 113 | type=int, default=default, help=help) 114 | elif type == "float": 115 | parser.add_argument("--{0}".format(name), 116 | type=float, default=default, help=help) 117 | elif type == "str": 118 | parser.add_argument("--{0}".format(name), 119 | type=str, default=default, help=help) 120 | else: 121 | raise ValueError("Unknown type '{0}'".format(type)) 122 | 123 | --------------------------------------------------------------------------------