├── .gitignore ├── .travis.yml ├── Dockerfile ├── README.md ├── api ├── __init__.py └── moses_api.py ├── config.yaml ├── run_moses.py └── translate_me.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # Ipython Notebook 62 | .ipynb_checkpoints 63 | 64 | # If you want to ignore the config... 65 | # config.yaml 66 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | services: docker 4 | 5 | before_install: 6 | - sudo apt-get -y install curl 7 | - docker pull vlall/moses-api 8 | 9 | install: 10 | - docker run -d -p 5000:5000 vlall/moses-api /bin/sh -c "cd /home/moses/mosesdecoder; ./bjam --with-boost=/home/moses/Downloads/boost_1_60_0 --with-cmph=/home/moses/cmph-2.0 --with-irstlm=/home/moses/irstlm -j12" 11 | 12 | script: 13 | - id=$(docker ps -q --filter "ancestor=vlall/moses-api:latest") 14 | - docker exec -i $id /bin/sh -c "sudo apt-get -y install screen && cd /home/moses/Downloads/moses-api && screen -dm python run_moses.py" 15 | 16 | after_success: docker exec -i $id /bin/sh -c "curl -XGET localhost:5000/haus | python -m json.tool" 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Vishal Lall "vishal.lall@caci.com" 3 | LABEL Description="Install Moses SMT API" Version="1.2" 4 | EXPOSE 5000 5 | 6 | RUN apt-get update && apt-get install -y \ 7 | automake \ 8 | build-essential \ 9 | curl \ 10 | g++ \ 11 | git \ 12 | graphviz \ 13 | imagemagick \ 14 | libboost-all-dev \ 15 | libbz2-dev \ 16 | libgoogle-perftools-dev \ 17 | liblzma-dev \ 18 | libtool \ 19 | make \ 20 | python-dev \ 21 | python-pip \ 22 | python-yaml \ 23 | subversion \ 24 | unzip \ 25 | wget \ 26 | zlib1g-dev 27 | 28 | RUN pip install \ 29 | flask \ 30 | flask-api 31 | 32 | RUN mkdir -p /home/moses 33 | WORKDIR /home/moses 34 | RUN git clone https://github.com/moses-smt/mosesdecoder 35 | RUN mkdir moses-models 36 | 37 | # Giza 38 | RUN wget -O giza-pp.zip "http://github.com/moses-smt/giza-pp/archive/master.zip" 39 | RUN unzip giza-pp.zip 40 | RUN rm giza-pp.zip 41 | RUN mv giza-pp-master giza-pp 42 | WORKDIR /home/moses/giza-pp 43 | RUN make 44 | WORKDIR /home/moses 45 | RUN mkdir external-bin-dir 46 | RUN cp giza-pp/GIZA++-v2/GIZA++ external-bin-dir 47 | RUN cp giza-pp/GIZA++-v2/snt2cooc.out external-bin-dir 48 | RUN cp giza-pp/mkcls-v2/mkcls external-bin-dir 49 | 50 | # CMPH 51 | RUN wget -O cmph-2.0.tar.gz "http://downloads.sourceforge.net/project/cmph/cmph/cmph-2.0.tar.gz?r=&ts=1426574097&use_mirror=cznic" 52 | RUN tar zxvf cmph-2.0.tar.gz 53 | WORKDIR /home/moses/cmph-2.0 54 | RUN ./configure 55 | RUN make 56 | RUN make install 57 | 58 | # IRSTLM 59 | WORKDIR /home/moses 60 | RUN wget -O irstlm-5.80.08.tgz "http://downloads.sourceforge.net/project/irstlm/irstlm/irstlm-5.80/irstlm-5.80.08.tgz?r=&ts=1342430877&use_mirror=kent" 61 | RUN tar zxvf irstlm-5.80.08.tgz 62 | WORKDIR /home/moses/irstlm-5.80.08/trunk 63 | RUN /bin/bash -c "source regenerate-makefiles.sh" 64 | RUN ./configure -prefix=/home/moses/irstlm 65 | RUN make 66 | RUN make install 67 | WORKDIR /home/moses 68 | ENV IRSTLM /home/moses/irstlm 69 | 70 | # Get Newest Boost 71 | RUN mkdir /home/moses/Downloads 72 | WORKDIR /home/moses/Downloads 73 | RUN wget https://sourceforge.net/projects/boost/files/boost/1.60.0/boost_1_60_0.tar.gz 74 | RUN tar xf boost_1_60_0.tar.gz 75 | RUN rm boost_1_60_0.tar.gz 76 | WORKDIR boost_1_60_0/ 77 | RUN ./bootstrap.sh 78 | RUN ./b2 -j4 --prefix=$PWD --libdir=$PWD/lib64 --layout=system link=static install || echo FAILURE 79 | 80 | # Buckwalter for Arabic 81 | WORKDIR /home/moses/Downloads 82 | RUN wget http://search.cpan.org/CPAN/authors/id/G/GR/GRAFF/Encode-Buckwalter-1.1.tar.gz 83 | RUN tar xf Encode-Buckwalter-1.1.tar.gz 84 | RUN rm Encode-Buckwalter-1.1.tar.gz 85 | WORKDIR /home/moses/Downloads/Encode-Buckwalter-1.1 86 | RUN perl Makefile.PL 87 | RUN make 88 | RUN make test 89 | RUN make install 90 | 91 | # Get Flask API 92 | WORKDIR /home/moses/Downloads 93 | RUN git clone https://github.com/vlall/moses-api 94 | 95 | # Download sample model 96 | WORKDIR /home/moses/moses-models 97 | RUN wget http://www.statmt.org/moses/download/sample-models.tgz 98 | RUN tar xf sample-models.tgz 99 | RUN rm sample-models.tgz 100 | 101 | WORKDIR /home/moses/mosesdecoder 102 | # COMPILE MOSES (Takes awhile...) 103 | #RUN ./bjam --with-boost=/home/moses/Downloads/boost_1_60_0 --with-cmph=/home/moses/cmph-2.0 --with-irstlm=/home/moses/irstlm -j12 104 | #WORKDIR /home/moses/ 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/vlall/Moses-API.svg?branch=master)](https://travis-ci.org/vlall/Moses-API) 2 | # Moses-API 3 | Build a RESTful API server running your own machine translation model 4 | 5 | ### Instructions 6 | - First pull the Docker image 7 | 8 | ```$ docker pull vlall/moses-api``` 9 | 10 | - Next, run the image and forward from 5000 to the local port you want the API to be running on 11 | ``` 12 | $ docker run -it -p 5000:5000 vlall/moses-api 13 | ``` 14 | 15 | - You will be directed to the /mosesdecoder directory. Here is where you will compile Moses (takes a few minutes to run). 16 | ``` 17 | $ ./bjam --with-boost=/home/moses/Downloads/boost_1_60_0 --with-cmph=/home/moses/cmph-2.0 --with-irstlm=/home/moses/irstlm -j12 18 | ``` 19 | 20 | - Your Docker image is already modified to work with the sample model. Use /home/moses/Downloads/moses-api/config.yaml as a reference for integration with your own models. 21 | 22 | - Start your RESTful API server in the background 23 | 24 | ```$ python run_moses.py &``` 25 | 26 | - For a quick translation, run: 27 | 28 | ``` 29 | $ curl -XGET localhost:5000/haus | python -m json.tool 30 | { 31 | "CMD": "/home/moses/mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out", 32 | "DURATION": "0.603 seconds", 33 | "INPUT": "haus", 34 | "INPUT_PATH": "/home/moses/moses-models/sample-models/phrase-model/in", 35 | "INPUT_SIZE": 4, 36 | "LAN": "N/A", 37 | "MODEL": "/home/moses/moses-models/sample-models", 38 | "OUTPUT": "house", 39 | "OUTPUT_PATH": "/home/moses/moses-models/sample-models/out", 40 | "OUTPUT_SIZE": 7, 41 | "STATUS": "Files successfully read", 42 | "URL": "http://localhost:5000" 43 | } 44 | 45 | ``` 46 | 47 | - To translate a whole file, navigate to your /moses-api folder, then run: 48 | 49 | ``` 50 | $ curl -XPUT -F name=@translate_me.txt localhost:5000/upload | python -m json.tool 51 | { 52 | "CMD": "/home/moses/mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out", 53 | "DURATION": "0.624 seconds", 54 | "INPUT": "das ist ein klein haus\n", 55 | "INPUT_PATH": "/home/moses/moses-models/sample-models/phrase-model/in", 56 | "INPUT_SIZE": 23, 57 | "LAN": "N/A", 58 | "MODEL": "/home/moses/moses-models/sample-models", 59 | "OUTPUT": "this is a small house", 60 | "OUTPUT_PATH": "/home/moses/moses-models/sample-models/out", 61 | "OUTPUT_SIZE": 24, 62 | "STATUS": "Files successfully read", 63 | "URL": "http://localhost:5000" 64 | } 65 | ``` 66 | 67 | Since this the Moses sample-model, it is very limited. Visit the Moses SMT website for more information on creating your own models: http://www.statmt.org/moses/?n=Development.GetStarted 68 | -------------------------------------------------------------------------------- /api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlall/Moses-API/e41e1c29ec89dca94a99e58c0374999cf1900a34/api/__init__.py -------------------------------------------------------------------------------- /api/moses_api.py: -------------------------------------------------------------------------------- 1 | from flask import request, url_for 2 | from flask.ext.api import FlaskAPI, status, exceptions 3 | import subprocess 4 | import yaml 5 | from werkzeug import secure_filename 6 | import codecs 7 | import json 8 | import time 9 | 10 | app = FlaskAPI(__name__) 11 | ALLOWED_EXTENSIONS = set(['txt', 'pdf']) 12 | 13 | 14 | def allowed_file(filename): 15 | return '.' in filename and \ 16 | filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS 17 | 18 | 19 | def translate(text): 20 | start_time = time.time() 21 | """ 22 | Run translation model using config 23 | """ 24 | with open('/home/moses/Downloads/moses-api/config.yaml', 'r') as f: 25 | doc = yaml.load(f) 26 | fileIn = doc['sample-models']['in'] 27 | fileOut = doc['sample-models']['out'] 28 | homeDir = doc['sample-models']['homeDir'] 29 | runCommand = doc['sample-models']['command'] 30 | status = 'Files successfully read' 31 | subprocess.call(['rm %s && rm %s' % (fileIn, fileOut)], shell=True) 32 | text8 = text.encode('utf8') 33 | inputFile = open(fileIn, 'w') 34 | inputFile.write(text8 + '\n') 35 | inputFile.close() 36 | subprocess.call([runCommand], cwd=homeDir, shell=True) 37 | readTranslate = open(fileOut, 'r') 38 | translatedText = readTranslate.read().decode('utf8') 39 | readTranslate.close() 40 | return { 41 | "STATUS": status, 42 | "LAN": 'N/A', 43 | "MODEL": str(homeDir), 44 | "CMD": str(runCommand), 45 | "URL": request.host_url.rstrip('/').decode().encode('utf8'), 46 | "INPUT": text.encode('utf8'), 47 | "INPUT_SIZE": len(text.encode('utf8')), 48 | "INPUT_PATH": str(fileIn), 49 | "OUTPUT": translatedText.encode('utf8').rstrip(), 50 | "OUTPUT_SIZE": len(translatedText.encode('utf8')), 51 | "OUTPUT_PATH": str(fileOut), 52 | "DURATION": '%.3f seconds' % (time.time() - start_time) 53 | } 54 | 55 | 56 | @app.route("/", methods=['GET']) 57 | def instructions(): 58 | return 'The Moses API is working! Try a GET request with text.\n' 59 | 60 | 61 | @app.route("/", methods=['GET']) 62 | def user_get(text): 63 | """ 64 | Translate text 65 | """ 66 | text = translate(text.decode('utf8')) 67 | return text 68 | 69 | 70 | @app.route("/upload", methods=['POST', 'PUT']) 71 | def upload(): 72 | """ 73 | Tranlsate file 74 | """ 75 | file = request.files['name'] 76 | if file and allowed_file(file.filename): 77 | filename = secure_filename(file.filename) 78 | text = file.read().decode('utf8') 79 | text = translate(text) 80 | return text 81 | else: 82 | return ('Error reading file...\n') 83 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | sample-models: 2 | in: '/home/moses/moses-models/sample-models/phrase-model/in' 3 | out: '/home/moses/moses-models/sample-models/out' 4 | homeDir: '/home/moses/moses-models/sample-models' 5 | command: '/home/moses/mosesdecoder/bin/moses -f phrase-model/moses.ini < phrase-model/in > out' 6 | -------------------------------------------------------------------------------- /run_moses.py: -------------------------------------------------------------------------------- 1 | from api.moses_api import app 2 | 3 | 4 | def main(): 5 | app.run( 6 | host='0.0.0.0', 7 | port=5000, 8 | threaded=True, 9 | debug=True 10 | ) 11 | 12 | 13 | if __name__ == '__main__': 14 | main() 15 | -------------------------------------------------------------------------------- /translate_me.txt: -------------------------------------------------------------------------------- 1 | das ist ein klein haus 2 | --------------------------------------------------------------------------------