├── .coveragerc ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── cwr_validator ├── __init__.py ├── app.py ├── config.py ├── resources │ ├── __init__.py │ └── upload.py ├── service │ ├── __init__.py │ └── cwr_parser.py ├── uploads │ ├── __init__.py │ └── __uploads__.py └── util │ ├── __init__.py │ └── parallel.py ├── data_validator ├── __init__.py ├── accessor.py └── config.properties ├── make.bat ├── requirements.txt ├── run_ws.py ├── run_ws.wsgi ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── data │ ├── __data_test__.py │ └── __init__.py ├── endpoints │ ├── __init__.py │ └── test_upload.py └── service │ ├── __init__.py │ └── test_parser.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | */python?.?/* 4 | */site-packages/nose/* 5 | *__init__* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *.pyc 5 | *.pyo 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | *.pot 48 | 49 | # Django stuff: 50 | *.log 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | # PyBuilder 56 | target/ 57 | 58 | # Idea PyCharm 59 | .idea/ 60 | .idea_modules/ 61 | out/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | - "pypy" 6 | matrix: 7 | include: 8 | - python: "2.7" 9 | env: COVERAGE=true 10 | 11 | install: 12 | - pip install tox 13 | script: 14 | - if [ -z "$COVERAGE" ]; then tox -e $(echo py$TRAVIS_PYTHON_VERSION | tr -d . | sed -e 's/pypypy/pypy/'); fi 15 | - if [ ! -z "$COVERAGE" ]; then tox -e coverage; fi -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 WESO 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Miscellany files 2 | include LICENSE 3 | include README.rst 4 | prune tests -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for the Python project 2 | # 3 | # It supports creating distribution files, and deploying them to Pypi and Pypitest or installing them locally 4 | # 5 | # A Python interpreter is required, and it should be accessible from the command line. 6 | 7 | # Sets the variables. 8 | 9 | # Sets the Python executable. 10 | # It will be the executable for the interpreter set up for the command line. 11 | PYTHON = python 12 | 13 | # Sets the distribution folder. 14 | # It will be the 'dist' folder. 15 | DISTDIR = dist 16 | 17 | # Sets the .egg file path. 18 | # The file will be located at the project's root. 19 | EGGDIR = CWR_API.egg-info 20 | 21 | # Sets the tox folder path. 22 | # It will be the '.tox' folder. 23 | TOXDIR = .tox 24 | 25 | # User-friendly check for sphinx-build 26 | ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) 27 | $(error The '$(PYTHON)' command was not found. Make sure you have a version of the python interpreter installed, then add the directory where it was installed to the PATH.) 28 | endif 29 | 30 | .PHONY: help clean 31 | 32 | # Help option 33 | # Shows the allowed commands to be received as parameters 34 | help: 35 | @echo "Please use 'make ' where is one of" 36 | @echo " dist_source to make the standard distribution" 37 | @echo " dist_binary to make the binary distribution" 38 | @echo " install to install the project" 39 | @echo " requirements to install the project requirements" 40 | @echo " pypi_reg to register on pypi" 41 | @echo " pypitest_reg to register on testpypi" 42 | @echo " pypi to upload to pypi" 43 | @echo " pypitest to upload to testpypi" 44 | @echo " test to run tests" 45 | 46 | # Clean option 47 | # Removes the distribution folder and the .egg file 48 | clean: 49 | rm -r -f $(DISTDIR) 50 | rm -r -f $(EGGDIR) 51 | rm -r -f $(TOXDIR) 52 | 53 | # Source distribution. 54 | dist_source: 55 | $(PYTHON) setup.py sdist 56 | 57 | # Binary distribution. 58 | dist_binary: 59 | $(PYTHON) setup.py bdist 60 | 61 | # Install in local libraries repository 62 | install: 63 | $(PYTHON) setup.py install 64 | 65 | # Install the project requirements 66 | requirements: 67 | pip install --upgrade -r requirements.txt 68 | 69 | # Pypi registration. 70 | pypi_reg: 71 | $(PYTHON) setup.py register -r pypi 72 | 73 | # Pypitest registration. 74 | pypitest_reg: 75 | $(PYTHON) setup.py register -r testpypi 76 | 77 | # Pypi deployment. 78 | pypi: 79 | $(PYTHON) setup.py sdist upload -r pypi 80 | 81 | # Pypitest deployment. 82 | pypitest: 83 | $(PYTHON) setup.py sdist upload -r testpypi 84 | 85 | # Tests suite. 86 | test: 87 | tox 88 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | CWR Data Model Validator 2 | ======================== 3 | 4 | Validator service for CWR files. It receives a file following the CISAC CWR 5 | standard v2.1 and returns a JSON containing the data from that same file. 6 | 7 | While right now it only handles parsing CWR files, in the future it will allow 8 | generating acknowledgement files, create validation reports, and will also 9 | support receiving the CWR files as a JSON. 10 | 11 | It uses the `CWR Data Model`_ API, and it is recommended using that same 12 | library to read the JSON created by the service. 13 | 14 | Documentation 15 | ------------- 16 | 17 | The current version is under development. No public documentation is still offered. 18 | 19 | Status 20 | ------ 21 | 22 | The project is still in the development phase. 23 | 24 | Issues management 25 | ~~~~~~~~~~~~~~~~~ 26 | 27 | Issues are managed at the GitHub `project issues page`_. 28 | 29 | Building the code 30 | ----------------- 31 | 32 | The application has been coded in Python, without using any particular framework. 33 | 34 | Prerequisites 35 | ~~~~~~~~~~~~~ 36 | 37 | The project has been tested in the following versions of the interpreter: 38 | 39 | - Python 2.6 40 | - Python 2.7 41 | - Python 3.3 42 | - Python 3.4 43 | - Pypy 44 | 45 | All other dependencies are indicated on requirements.txt. The included makefile can install them with the command: 46 | 47 | ``make requirements`` 48 | 49 | Getting the code 50 | ~~~~~~~~~~~~~~~~ 51 | 52 | The code can be found at the `GitHub project page`_. 53 | 54 | License 55 | ------- 56 | 57 | The project has been released under the `MIT License`_. 58 | 59 | .. _CWR Data Model: https://github.com/weso/CWR-DataApi 60 | .. _project issues page: https://github.com/weso/CWR-Validator/issues 61 | .. _GitHub project page: https://github.com/weso/CWR-Validator 62 | .. _MIT License: http://www.opensource.org/licenses/mit-license.php -------------------------------------------------------------------------------- /cwr_validator/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cwr_validator.app import create_app 4 | 5 | """ 6 | CWR Data API Validator WS 7 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 8 | Validator Web Service for Common Works Registrations. 9 | :copyright: (c) 2015 by WESO 10 | :license: MIT, see LICENSE for more details. 11 | """ 12 | 13 | __version__ = '0.0.1' 14 | __license__ = 'MIT' 15 | -------------------------------------------------------------------------------- /cwr_validator/app.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | """ 4 | Web app module. 5 | """ 6 | 7 | import logging 8 | from logging.handlers import RotatingFileHandler 9 | from logging import Formatter 10 | import os 11 | 12 | from flask import Flask 13 | from werkzeug.contrib.fixers import ProxyFix 14 | from flask.ext.restful import Api 15 | 16 | from cwr_validator.config import DevConfig 17 | from data_validator.accessor import CWRValidatorConfiguration 18 | from cwr_validator.resources import UploadFileResource 19 | from cwr_validator.service import ThreadingCWRParserService 20 | from cwr_validator.uploads.__uploads__ import path 21 | 22 | __author__ = 'Bernardo Martínez Garrido' 23 | __license__ = 'MIT' 24 | __status__ = 'Development' 25 | 26 | 27 | def _register_resources(api): 28 | api.add_resource(UploadFileResource, '/upload/', endpoint='cwr_upload') 29 | 30 | 31 | def _load_services(app, config): 32 | file_ws = os.environ.get('CWR_ADMIN_WS', 33 | 'http://127.0.0.1:33508/cwr/') 34 | 35 | path_upload = config['upload.folder'] 36 | if len(path_upload) == 0: 37 | path_upload = path() 38 | 39 | app.config['FILE_SERVICE'] = ThreadingCWRParserService( 40 | path_upload, file_ws + 'files/') 41 | 42 | 43 | def create_app(config_object=DevConfig): 44 | config = CWRValidatorConfiguration().get_config() 45 | 46 | app = Flask(__name__) 47 | api = Api(app) 48 | 49 | app.config.from_object(config_object) 50 | 51 | _register_resources(api) 52 | _load_services(app, config) 53 | 54 | app.wsgi_app = ProxyFix(app.wsgi_app) 55 | 56 | if app.config['DEBUG']: 57 | log = config['log.folder'] 58 | if len(log) == 0: 59 | log = 'mera_ws.log' 60 | 61 | handler = RotatingFileHandler(log, maxBytes=10000, backupCount=1) 62 | handler.setLevel(logging.DEBUG) 63 | handler.setFormatter( 64 | Formatter('[%(levelname)s][%(asctime)s] %(message)s')) 65 | 66 | logging.basicConfig(level=logging.DEBUG) 67 | logging.getLogger('').addHandler(handler) 68 | 69 | app.logger.addHandler(handler) 70 | 71 | return app 72 | -------------------------------------------------------------------------------- /cwr_validator/config.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import os 3 | 4 | __author__ = 'Bernardo Martínez Garrido' 5 | __license__ = 'MIT' 6 | __status__ = 'Development' 7 | 8 | 9 | class Config(object): 10 | os_env = os.environ 11 | 12 | SECRET_KEY = os_env.get('CWR_VALIDATOR_SECRET', os.urandom(24)) 13 | APP_DIR = os.path.abspath(os.path.dirname(__file__)) # This directory 14 | PROJECT_ROOT = os.path.abspath(os.path.join(APP_DIR, os.pardir)) 15 | 16 | 17 | class DevConfig(Config): 18 | """ 19 | Development configuration. 20 | """ 21 | ENV = 'dev' 22 | DEBUG = True 23 | 24 | 25 | class ProdConfig(Config): 26 | """ 27 | Development configuration. 28 | """ 29 | ENV = 'prod' 30 | DEBUG = False 31 | 32 | 33 | class TestConfig(Config): 34 | TESTING = True 35 | DEBUG = True 36 | -------------------------------------------------------------------------------- /cwr_validator/resources/__init__.py: -------------------------------------------------------------------------------- 1 | from cwr_validator.resources.upload import UploadFileResource 2 | -------------------------------------------------------------------------------- /cwr_validator/resources/upload.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import logging 3 | 4 | from flask import current_app 5 | from flask.ext.restful import Resource, reqparse 6 | 7 | """ 8 | Flask RESTful resources for the file uploading endpoints. 9 | 10 | These take handle of receiving and processing files. 11 | """ 12 | 13 | __author__ = 'Bernardo Martínez Garrido' 14 | __license__ = 'MIT' 15 | __status__ = 'Development' 16 | 17 | _logger = logging.getLogger(__name__) 18 | 19 | 20 | class UploadFileResource(Resource): 21 | """ 22 | Resource for building an endpoint where files are received. 23 | 24 | It receives a file and sends it to the correct service to be processed. 25 | """ 26 | 27 | def __init__(self): 28 | super(UploadFileResource, self).__init__() 29 | 30 | self._reqparse = reqparse.RequestParser() 31 | self._reqparse.add_argument('file_id', type=str, required=True, 32 | help='No file ID provided', 33 | location='json') 34 | self._reqparse.add_argument('filename', type=unicode, required=True, 35 | help='No file name provided', 36 | location='json') 37 | self._reqparse.add_argument('contents', type=unicode, required=True, 38 | help='No file contents provided', 39 | location='json') 40 | 41 | def get(self): 42 | """ 43 | Getting from the uploads endpoint is disallowed. 44 | 45 | A message is returned to indicate this. 46 | 47 | :return: a message warning that the get command is disallowed 48 | """ 49 | return 'Please, send files to the web service through a POST method.' 50 | 51 | def post(self): 52 | """ 53 | Posts a file to the endpoint. 54 | 55 | It should receive a file, which can have any name, as it will just take the first file on the request. 56 | 57 | :return: 58 | """ 59 | _logger.info('Received CWR file') 60 | 61 | file_data = self._reqparse.parse_args() 62 | 63 | _logger.info('Accepted CWR file') 64 | 65 | file_service = current_app.config['FILE_SERVICE'] 66 | 67 | file_id = file_data['file_id'] 68 | 69 | _logger.info('Beginning to process file with id %s' % file_id) 70 | 71 | file_service.process_cwr(file_data) 72 | 73 | _logger.info('Finished processing file with id %s' % file_id) 74 | 75 | return '', 200 76 | -------------------------------------------------------------------------------- /cwr_validator/service/__init__.py: -------------------------------------------------------------------------------- 1 | from cwr_validator.service.cwr_parser import ThreadingCWRParserService 2 | -------------------------------------------------------------------------------- /cwr_validator/service/cwr_parser.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | from abc import ABCMeta, abstractmethod 4 | import os 5 | import logging 6 | import json 7 | 8 | import requests 9 | from requests import ConnectionError 10 | from cwr.parser.decoder.file import default_file_decoder 11 | from cwr.parser.encoder.cwrjson import JSONEncoder 12 | 13 | """ 14 | Services for parsing CWR files. 15 | 16 | These allow creating the model graph from a CWR file, but also transforming it to and from JSON messages. 17 | """ 18 | 19 | __author__ = 'Bernardo Martínez Garrido' 20 | __license__ = 'MIT' 21 | __status__ = 'Development' 22 | 23 | _logger = logging.getLogger(__name__) 24 | 25 | 26 | class CWRParserService(object): 27 | """ 28 | Service for parsing CWR files and model instances. 29 | 30 | It can transform a CWR file into a graph of model classes, and can generate a JSON from such a graph. 31 | """ 32 | __metaclass__ = ABCMeta 33 | 34 | def __init__(self): 35 | pass 36 | 37 | @abstractmethod 38 | def process_cwr(self, file): 39 | """ 40 | Transforms a CWR file into a graph of model classes. 41 | 42 | As processing the file can take a long time, this method will return an unique identifier for the file being 43 | parsed. 44 | 45 | This ID can be used to check the parsing status of the file. 46 | 47 | :param file: the CWR file to parse 48 | :return: an unique ID for the file 49 | """ 50 | raise NotImplementedError('The save_file method must be implemented') 51 | 52 | 53 | class ThreadingCWRParserService(CWRParserService): 54 | """ 55 | Thread-based implementation of CWRParserService. 56 | 57 | This will generate a thread for each CWR parsing procedure, so these don't block the web service. 58 | """ 59 | 60 | _logger = logging.getLogger(__name__) 61 | 62 | def __init__(self, path, store_url): 63 | super(CWRParserService, self).__init__() 64 | self._path = path 65 | self._decoder = default_file_decoder() 66 | self._encoder_json = JSONEncoder() 67 | 68 | self._store_url = store_url 69 | 70 | def process_cwr(self, file): 71 | cwr_id = file['file_id'] 72 | 73 | file_path = os.path.join(self._path, cwr_id) 74 | 75 | # The file is temporarily saved 76 | # with open(file_path, 'w') as f: 77 | # contents = file['contents'] 78 | # 79 | # if sys.version_info[0] > 2: 80 | # # For Python 3 81 | # contents = str(contents) 82 | # 83 | # f.write(contents.encode('latin-1')) 84 | 85 | self._parse_cwr_threaded(cwr_id, file) 86 | 87 | def _parse_cwr_threaded(self, cwr_id, file_data): 88 | _logger.info('Begins processing CWR file with id %s' % cwr_id) 89 | self.parse_cwr(cwr_id, file_data) 90 | _logger.info('Finished processing CWR file with id %s' % cwr_id) 91 | 92 | def parse_cwr(self, cwr_id, file_data): 93 | try: 94 | result = self._decoder.decode(file_data) 95 | except: 96 | _logger.error('Error processing CWR file with id %s' % cwr_id) 97 | result = None 98 | 99 | if result: 100 | _logger.error( 101 | 'Sending processed results for file with id %s' % cwr_id) 102 | self._send_results(cwr_id, self._encoder_json.encode(result)) 103 | else: 104 | self._send_results(cwr_id, None) 105 | 106 | # os.remove(file_path) 107 | 108 | def _send_results(self, cwr_id, result): 109 | # TODO: Do this in a cleaner way 110 | 111 | headers = {'Content-Type': 'application/json'} 112 | 113 | data = { 114 | 'id': cwr_id 115 | } 116 | 117 | if result: 118 | data['data'] = result 119 | 120 | try: 121 | requests.post(self._store_url, 122 | data=json.dumps(data), headers=headers) 123 | self._logger.info('Sent parse results') 124 | except ConnectionError: 125 | self._logger.error('Failure when sending parse results') 126 | -------------------------------------------------------------------------------- /cwr_validator/uploads/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = 'Bernardo Martínez Garrido' 3 | -------------------------------------------------------------------------------- /cwr_validator/uploads/__uploads__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | __author__ = 'Bernardo Martínez Garrido' 5 | 6 | 7 | def path(): 8 | return os.path.dirname(__file__) 9 | -------------------------------------------------------------------------------- /cwr_validator/util/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Bernardo' 2 | -------------------------------------------------------------------------------- /cwr_validator/util/parallel.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | try: 4 | import thread 5 | 6 | _python2 = True 7 | except ImportError: 8 | import threading as thread 9 | from threading import Thread 10 | 11 | _python2 = False 12 | 13 | __author__ = 'Bernardo Martínez Garrido' 14 | __license__ = 'MIT' 15 | __status__ = 'Development' 16 | 17 | 18 | def threaded(function): 19 | def _decorator(*args): 20 | if _python2: 21 | thread.start_new_thread(function, args) 22 | else: 23 | Thread(target=function, args=args).start() 24 | 25 | return _decorator 26 | -------------------------------------------------------------------------------- /data_validator/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Bernardo' 2 | -------------------------------------------------------------------------------- /data_validator/accessor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | 5 | """ 6 | Facades for accessing the configuration data. 7 | """ 8 | 9 | __author__ = 'Bernardo Martínez Garrido' 10 | __license__ = 'MIT' 11 | __status__ = 'Development' 12 | 13 | 14 | class _FileReader(object): 15 | """ 16 | Offers methods to read the data files. 17 | """ 18 | 19 | # Singleton control object 20 | __shared_state = {} 21 | 22 | def __init__(self): 23 | self.__dict__ = self.__shared_state 24 | 25 | def __path(self): 26 | """ 27 | Returns the path to the folder in which this class is contained. 28 | 29 | As this class is to be on the same folder as the data files, this will be the data files folder. 30 | 31 | :return: path to the data files folder. 32 | """ 33 | return os.path.dirname(__file__) 34 | 35 | def read_properties(self, file_name): 36 | config = {} 37 | with open(os.path.join(self.__path(), os.path.basename(file_name)), 38 | 'rt') as f: 39 | for line in f: 40 | line = line.rstrip() # removes trailing whitespace and '\n' chars 41 | 42 | if "=" not in line: continue # skips blanks and comments w/o = 43 | if line.startswith( 44 | "#"): continue # skips comments which contain = 45 | 46 | k, v = line.split("=", 1) 47 | config[k] = v 48 | 49 | return config 50 | 51 | 52 | class CWRValidatorConfiguration(object): 53 | """ 54 | Offers methods to access the CWR web application configuration data. 55 | """ 56 | 57 | def __init__(self): 58 | # Reader for the files 59 | self._reader = _FileReader() 60 | 61 | # Files containing the CWR info 62 | self._file_config = 'config.properties' 63 | 64 | # Configuration 65 | self._config = None 66 | 67 | def get_config(self): 68 | """ 69 | Loads the configuration file. 70 | 71 | :return: the webapp configuration 72 | """ 73 | if not self._config: 74 | self._config = self._reader.read_properties(self._file_config) 75 | 76 | return self._config 77 | -------------------------------------------------------------------------------- /data_validator/config.properties: -------------------------------------------------------------------------------- 1 | debug=true 2 | secretKey= 3 | upload.folder= 4 | log.folder= -------------------------------------------------------------------------------- /make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Makefile-like batch file for the Python project. 4 | REM It supports creating distribution files, and deploying them to Pypi and Pypitest or installing them locally 5 | REM 6 | REM A Python interpreter is required, and it should be accessible from the command line. 7 | REM This file should be run from the project's root folder. 8 | REM 9 | REM To deploy or register to Pypi or Pypitest a valid .pypirc file should be accessible on the default location. 10 | 11 | REM Sets the variables 12 | REM Sets the Python executable. 13 | REM It will be the executable for the interpreter set up for the command line. 14 | if "%PYTHON%" == "" ( 15 | set PYTHON=python 16 | ) 17 | 18 | REM Sets the distribution folder. 19 | REM It will be the 'dist' folder. 20 | if "%DISTDIR%" == "" ( 21 | set DISTDIR=dist 22 | ) 23 | 24 | REM Sets the .egg file path. 25 | REM The file will be located at the project's root. 26 | if "%EGGDIR%" == "" ( 27 | set EGGDIR=CWR_API.egg-info 28 | ) 29 | 30 | REM Sets the tox folder path. 31 | REM It will be the '.tox' folder. 32 | if "%TOXDIR%" == "" ( 33 | set TOXDIR=.tox 34 | ) 35 | 36 | REM If no parameters are received, the help is shown 37 | if "%1" == "" goto help 38 | 39 | REM Help option 40 | REM Shows the allowed commands to be received as parameters 41 | if "%1" == "help" ( 42 | :help 43 | echo.Please use `make ^` where ^ is one of 44 | echo. dist_source to make the source distribution 45 | echo. dist_binary to make the binary distribution 46 | echo. install to install the project 47 | echo. requirements to install the project requirements 48 | echo. pypi_reg to register on pypi 49 | echo. pypitest_reg to register on pypi-test 50 | echo. pypi to upload to pypi 51 | echo. pypitest to upload to pypi-test 52 | echo. test to run tests 53 | goto end 54 | ) 55 | 56 | REM Clean option 57 | REM Removes the distribution folder and the .egg file 58 | if "%1" == "clean" ( 59 | if exist %DISTDIR% ( 60 | rd /S /Q %DISTDIR% 61 | ) 62 | if exist %EGGDIR% ( 63 | rd /S /Q %EGGDIR% 64 | ) 65 | if exist %TOXDIR% ( 66 | rd /S /Q %TOXDIR% 67 | ) 68 | goto end 69 | ) 70 | 71 | 72 | REM Checks if the interpreter is available. 73 | %PYTHON% -V> nul 74 | if errorlevel 9009 goto missing_interpreter 75 | goto interpreter_ok 76 | 77 | REM Missing interpreter. 78 | REM The process will end and a warning will be shown. 79 | :missing_interpreter 80 | 81 | echo. 82 | echo.The '%PYTHON%' command was not found. Make sure you have a 83 | echo.version of the python interpreter installed, then add the 84 | echo.directory where it was installed to the PATH. 85 | echo. 86 | exit /b 1 87 | 88 | :interpreter_ok 89 | 90 | 91 | REM Source distribution. 92 | if "%1" == "dist_source" ( 93 | %PYTHON% setup.py sdist 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Generated source distribution. It can be found in the 97 | echo.%DISTDIR% folder. 98 | goto end 99 | ) 100 | 101 | REM Binary distribution. 102 | if "%1" == "dist_binary" ( 103 | %PYTHON% setup.py bdist 104 | if errorlevel 1 exit /b 1 105 | echo. 106 | echo.Generated binary distribution. It can be found in the 107 | echo.%DISTDIR% folder. 108 | goto end 109 | ) 110 | 111 | REM Install in local libraries repository. 112 | if "%1" == "install" ( 113 | %PYTHON% setup.py install 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Installed the project into the local repository. 117 | goto end 118 | ) 119 | 120 | REM Install the project requirements 121 | if "%1" == "requirements" ( 122 | pip install --upgrade -r requirements.txt 123 | ) 124 | 125 | REM Pypi registration. 126 | if "%1" == "pypi_reg" ( 127 | %PYTHON% setup.py register -r pypi 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Registered project on pypi. 131 | goto end 132 | ) 133 | 134 | REM Pypitest registration. 135 | if "%1" == "pypitest_reg" ( 136 | %PYTHON% setup.py register -r pypitest 137 | if errorlevel 1 exit /b 1 138 | echo. 139 | echo.Registered project on pypitest. 140 | goto end 141 | ) 142 | 143 | REM Pypi deployment. 144 | if "%1" == "pypi" ( 145 | %PYTHON% setup.py sdist upload -r pypi 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Uploaded project to pypi. 149 | goto end 150 | ) 151 | 152 | REM Pypitest deployment. 153 | if "%1" == "pypitest" ( 154 | %PYTHON% setup.py sdist upload -r pypitest 155 | if errorlevel 1 exit /b 1 156 | echo. 157 | echo.Uploaded project to pypitest. 158 | goto end 159 | ) 160 | 161 | REM Tests suite. 162 | if "%1" == "test" ( 163 | tox 164 | ) 165 | 166 | :end 167 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Flask 2 | Flask==0.10.1 3 | Flask-RESTful==0.3.2 4 | 5 | # CWR API 6 | CWR-API>=0.0.26 7 | 8 | # Requests 9 | requests==2.7.0 10 | 11 | # Compatibility libraries 12 | ordereddict==1.1 13 | 14 | # Deployment 15 | twine>=1.5.0 -------------------------------------------------------------------------------- /run_ws.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import os 3 | 4 | from cwr_validator import create_app 5 | 6 | """ 7 | Runs the CWR Validator web API. 8 | """ 9 | 10 | __author__ = 'Bernardo Martínez Garrido' 11 | __license__ = 'MIT' 12 | __version__ = '0.0.0' 13 | 14 | if __name__ == '__main__': 15 | port = int(os.environ.get('PORT', 33568)) 16 | host = os.environ.get('HOST', '127.0.0.1') 17 | 18 | app = create_app() 19 | 20 | app.run(host=host, port=port) 21 | -------------------------------------------------------------------------------- /run_ws.wsgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | import logging 4 | from cwr_validator import create_app 5 | 6 | sys.path.insert(0,"/var/www/cwr_validator/") 7 | #os.chdir("/var/www/cwr_validator") 8 | 9 | application = create_app() -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | # This flag says that the code is written to work on both Python 2 and Python 3 | # 3. If at all possible, it is good practice to do this. If you cannot, you 4 | # will need to generate wheels for each Python version that you support. 5 | universal = 1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import ast 3 | import re 4 | from codecs import open 5 | from os import path 6 | 7 | from setuptools import setup, find_packages 8 | 9 | """ 10 | PyPI configuration module. 11 | """ 12 | 13 | __author__ = 'Bernardo Martínez Garrido' 14 | __license__ = 'MIT' 15 | __version__ = '0.0.0' 16 | 17 | _version_re = re.compile(r'__version__\s+=\s+(.*)') 18 | _tests_require = ['pytest', 'nose', 'Flask-Testing'] 19 | 20 | here = path.abspath(path.dirname(__file__)) 21 | 22 | # Get the long description from the relevant file 23 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 24 | long_description = f.read() 25 | 26 | with open('cwr_validator/__init__.py', 'rb', encoding='utf-8') as f: 27 | version = f.read() 28 | version = _version_re.search(version).group(1) 29 | version = str(ast.literal_eval(version.rstrip())) 30 | 31 | setup( 32 | name='CWR-Validator-WS', 33 | packages=find_packages(), 34 | include_package_data=True, 35 | package_data={ 36 | }, 37 | version=version, 38 | description='Validator web service for CWR files', 39 | author='WESO', 40 | author_email='weso@weso.es', 41 | license='MIT', 42 | url='https://github.com/weso/CWR-Validator', 43 | download_url='https://github.com/weso/CWR-Validator', 44 | keywords=['CWR', 'commonworks', 'api', 'CISAC', 'validator'], 45 | platforms='any', 46 | classifiers=['License :: OSI Approved :: MIT License', 47 | 'Development Status :: 3 - Alpha', 48 | 'Environment :: Web Environment', 49 | 'Intended Audience :: Developers', 50 | 'Operating System :: OS Independent', 51 | 'Programming Language :: Python', 52 | 'Programming Language :: Python :: 2', 53 | 'Programming Language :: Python :: 2.6', 54 | 'Programming Language :: Python :: 2.7', 55 | 'Programming Language :: Python :: 3', 56 | 'Programming Language :: Python :: 3.2', 57 | 'Programming Language :: Python :: 3.3', 58 | 'Programming Language :: Python :: 3.4', 59 | 'Programming Language :: Python :: Implementation :: PyPy'], 60 | long_description=long_description, 61 | install_requires=[ 62 | 'Flask', 63 | 'Flask-RESTful', 64 | 'CWR-API', 65 | 'requests', 66 | ], 67 | tests_require=_tests_require, 68 | extras_require={'test': _tests_require}, 69 | ) 70 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weso/CWR-Validator/18b83136f44f5bdd2f66c9af866b0e37acf682cb/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/__data_test__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | __author__ = 'Bernardo Martínez Garrido' 5 | 6 | 7 | def path(): 8 | return os.path.dirname(__file__) 9 | -------------------------------------------------------------------------------- /tests/data/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Bernardo' 2 | -------------------------------------------------------------------------------- /tests/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Bernardo' 2 | -------------------------------------------------------------------------------- /tests/endpoints/test_upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import unittest 4 | from json import JSONEncoder 5 | import json 6 | 7 | from cwr_validator import create_app 8 | 9 | __author__ = 'Bernardo Martínez Garrido' 10 | __license__ = 'MIT' 11 | __status__ = 'Development' 12 | 13 | 14 | class TestUpload(unittest.TestCase): 15 | def setUp(self): 16 | self._app = create_app() 17 | 18 | self._app.config['DEBUG'] = False 19 | self._app.config['TESTING'] = True 20 | 21 | self._client = self._app.test_client() 22 | 23 | def test_get(self): 24 | response = self._client.get('/upload/') 25 | self.assertEqual(response.status_code, 200) 26 | 27 | def test_post_no_file(self): 28 | response = self._client.post('/upload/', 29 | headers={ 30 | 'content-type': 'application/json'}) 31 | self.assertEqual(response.status_code, 400) 32 | 33 | def test_post_file_with_invalid_data(self): 34 | json_data = JSONEncoder().encode( 35 | { 36 | 'file_id': '123', 37 | 'filename': 'hello_world.txt', 38 | 'contents': 'my file contents' 39 | } 40 | ) 41 | 42 | response = self._client.post('/upload/', data=json_data, 43 | headers={ 44 | 'content-type': 'application/json'}) 45 | 46 | self.assertEqual(response.status_code, 200) 47 | 48 | 49 | data = json.loads(str(response.data)) 50 | 51 | def test_post_file_with_valid_data(self): 52 | json_data = JSONEncoder().encode( 53 | { 54 | 'file_id': '123', 55 | 'filename': 'hello_world.txt', 56 | 'contents': _file_contents_cwr() 57 | } 58 | ) 59 | 60 | response = self._client.post('/upload/', data=json_data, 61 | headers={ 62 | 'content-type': 'application/json'}) 63 | 64 | self.assertEqual(response.status_code, 200) 65 | 66 | 67 | def _file_contents_cwr(): 68 | header_file = 'HDRPB226144593AGENCIA GRUPO MUSICAL 01.102013080902591120130809 ' 69 | header_group1 = 'GRHAGR0000102.100130400001 ' 70 | agr = 'AGR000000000000000000023683606100 OS200311182013111820131118N D20131118 00009SYY ' 71 | territory = 'TER0000000000000000I2136' 72 | ipa_1 = 'IPA0000000000000001AS0026166137500000000000001183606 ITALIAN GILBERTI DUANTE 61 0500061 0000061 00000' 73 | ipa_2 = 'IPA0000000000000002AC00250165006000000000000066 SOCIETY MUSIC 61 0500061 1000061 10000' 74 | trailer_group1 = 'GRT000010000017900000719 0000000000' 75 | trailer_file = 'TRL000020000053200005703' 76 | 77 | transaction1 = agr + '\n' + territory + '\n' + ipa_1 + '\n' + ipa_2 78 | 79 | transaction2 = 'NWR0000019900000000WORK NAME 1450455 00000000 UNC000000YMTX ORI ORIORI N00000000000U Y' + '\n' + \ 80 | 'SPU0000019900000702014271370 MUSIC SOCIETY E 005101734040102328568410061 0500061 1000061 10000 0000000000000 OS ' + '\n' + \ 81 | 'SPU00000199000007030166 ANOTHER SOCIETY AM 002501650060477617137010061 0000061 0000061 00000 0000000000000 PS ' + '\n' + \ 82 | 'SPU00000199000007040170 YET ANOTHER SOCIETY SE 002261445930035870006610059 00000 00000 00000 0000000000000 PG ' + '\n' + \ 83 | 'SPT000001990000070570 050000500005000I0484Y001' + '\n' + \ 84 | 'SWR00000199000007061185684 A NAME YET ANOTHER NAME C 0026058307861 0500061 0000061 00000 0000260582865 ' + '\n' + \ 85 | 'SWT00000199000007071185684 050000500005000I0484Y001' + '\n' + \ 86 | 'PWR00000199000007084271370 MUSIC SOCIETY 01023285684100 1185684 ' + '\n' + \ 87 | 'PER0000019900000709A NAME 000000000000000000000000' + '\n' + \ 88 | 'REC000001990000071019980101 000300 A COMPILATION P A I _AR_ 33002 U ' 89 | 90 | header_group2 = 'GRHNWR0000102.100130400001 ' 91 | trailer_group2 = 'GRT000010000017900000719 0000000000' 92 | 93 | record = header_file + '\n' + \ 94 | header_group1 + '\n' + \ 95 | transaction1 + '\n' + \ 96 | transaction1 + '\n' + \ 97 | trailer_group1 + '\n' + \ 98 | header_group2 + '\n' + \ 99 | transaction2 + '\n' + \ 100 | transaction2 + '\n' + \ 101 | trailer_group2 + '\n' + \ 102 | trailer_file 103 | 104 | return record 105 | -------------------------------------------------------------------------------- /tests/service/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Bernardo' 2 | -------------------------------------------------------------------------------- /tests/service/test_parser.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | import unittest 4 | import codecs 5 | 6 | from cwr_validator.service import ThreadingCWRParserService 7 | from cwr_validator.uploads import __uploads__ 8 | from tests.data import __data_test__ 9 | 10 | __author__ = 'Bernardo Martínez Garrido' 11 | __license__ = 'MIT' 12 | __status__ = 'Development' 13 | 14 | 15 | class TestUpload(unittest.TestCase): 16 | def setUp(self): 17 | self._path_test = __data_test__.path() 18 | path = __uploads__.path() 19 | self._parser = ThreadingCWRParserService(path, 'http://127.0.0.1/') 20 | 21 | def test_parse_invalid(self): 22 | file_path = '%s/test_parse_invalid' % self._path_test 23 | 24 | file = codecs.open(file_path, 'w') 25 | file.write('') 26 | file.close() 27 | 28 | result = self._parser.parse_cwr(0, 'empty.txt') 29 | 30 | self.assertEqual(None, result) 31 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py26,py27}, 4 | {pypy}, 5 | {coverage} 6 | 7 | [testenv] 8 | deps = 9 | -r{toxinidir}/requirements.txt 10 | nose 11 | commands = 12 | nosetests 13 | 14 | [testenv:coverage] 15 | passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH 16 | deps = 17 | -r{toxinidir}/requirements.txt 18 | nose 19 | coverage 20 | coveralls 21 | commands = 22 | nosetests --with-coverage --cover-package=cwr 23 | coverage combine 24 | coverage report --omit=*test* 25 | coveralls 26 | 27 | [testenv:check] 28 | deps = 29 | -r{toxinidir}/requirements.txt 30 | docutils 31 | check-manifest 32 | flake8 33 | readme 34 | pygments 35 | commands = 36 | python setup.py check --strict --metadata --restructuredtext 37 | check-manifest {toxinidir} 38 | flake8 cwr 39 | flake8 data_cwr 40 | flake8 config_cwr 41 | flake8 tests 42 | 43 | [testenv:docs] 44 | changedir = 45 | docs/source 46 | deps = 47 | -r{toxinidir}/requirements.txt 48 | sphinx 49 | commands = 50 | sphinx-build -b linkcheck ./ {envtmpdir}/html 51 | sphinx-build -W -b html -d {envtmpdir}/doctrees ./ {envtmpdir}/html --------------------------------------------------------------------------------