├── .gitattributes ├── .gitignore ├── .gitmodules ├── .gitreview ├── INFO.yaml ├── LICENSE.txt ├── README.rst ├── acumos-package ├── .gitignore ├── MANIFEST.in ├── acumos │ ├── __init__.py │ ├── _version.py │ ├── auth.py │ ├── exc.py │ ├── logging.py │ ├── metadata.py │ ├── modeling.py │ ├── pickler.py │ ├── protogen.py │ ├── session.py │ ├── tests │ │ ├── att.png │ │ ├── connexion_server.py │ │ ├── mock-license.json │ │ ├── mock_server.py │ │ ├── model_loader_helper.py │ │ ├── py36_namedtuple.py │ │ ├── swagger.yaml │ │ ├── test_metadata.py │ │ ├── test_modeling.py │ │ ├── test_pickler.py │ │ ├── test_protogen.py │ │ ├── test_session.py │ │ ├── test_wrapped.py │ │ ├── unpickler_helper.py │ │ ├── user_module.py │ │ ├── user_package │ │ │ ├── __init__.py │ │ │ └── user_package_module.py │ │ └── utils.py │ ├── utils.py │ └── wrapped.py ├── docs ├── examples │ ├── image_example.py │ ├── keras_example.py │ ├── raw_example.py │ ├── sklearn │ │ ├── face_completion_example.py │ │ └── iris_random_forest_example.py │ └── tensorflow_example.py ├── pom.xml ├── setup.cfg ├── setup.py ├── testing │ ├── test-requirements.txt │ ├── tox-requirements.txt │ └── wrap │ │ ├── README.md │ │ ├── dump_example_model.py │ │ ├── listener.py │ │ ├── runner.py │ │ ├── runtime.json │ │ ├── swagger.py │ │ └── talker.py └── tox.ini ├── docs ├── developer-guide.rst ├── images │ └── Acumos_logo_white.png ├── index.rst ├── release-notes.rst ├── tutorial │ └── index.rst └── user-guide.rst └── releases ├── pypi-0.9.1.yaml ├── pypi-0.9.2.yaml ├── pypi-0.9.3.yaml ├── pypi-0.9.4.yaml ├── pypi-0.9.7.yaml ├── pypi-0.9.8.yaml ├── pypi-0.9.9.yaml ├── pypi-1.0.0.yaml ├── pypi-1.0.1.yaml └── release-0.9.0.yaml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Basic .gitattributes for a python repo. 2 | 3 | # Source files 4 | # ============ 5 | *.pxd text 6 | *.py text 7 | *.py3 text 8 | *.pyw text 9 | *.pyx text 10 | 11 | # Binary files 12 | # ============ 13 | *.db binary 14 | *.p binary 15 | *.pkl binary 16 | *.pyc binary 17 | *.pyd binary 18 | *.pyo binary 19 | 20 | # Note: .db, .p, and .pkl files are associated 21 | # with the python modules ``pickle``, ``dbm.*``, 22 | # ``shelve``, ``marshal``, ``anydbm``, & ``bsddb`` 23 | # (among others). 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Sphinx documentation 2 | docs/_build/ 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "acumos-package/acumos/tests/schemas"] 2 | path = acumos-package/acumos/tests/schemas 3 | url = ../model-schema 4 | -------------------------------------------------------------------------------- /.gitreview: -------------------------------------------------------------------------------- 1 | [gerrit] 2 | host=gerrit.acumos.org 3 | port=29418 4 | project=acumos-python-client.git 5 | -------------------------------------------------------------------------------- /INFO.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | project: 'acumos-python-client' 3 | project_creation_date: '2017-11-17' 4 | project_category: '' 5 | lifecycle_state: 'Incubation' 6 | project_lead: &acumos_acumos-python-client_ptl 7 | name: 'Philippe Dooze' 8 | email: 'philippe.dooze@orange.com' 9 | id: 'PhilippDo' 10 | company: 'Orange' 11 | timezone: 'Europe/Paris' 12 | primary_contact: *acumos_acumos-python-client_ptl 13 | issue_tracking: 14 | type: 'jira' 15 | url: 'https://jira.acumos.org/projects/ACUMOS' 16 | key: 'ACUMOS' 17 | mailing_list: 18 | type: 'groups.io' 19 | url: 'https://lists.lfai.foundation/g/acumosai-modelmanagement/' 20 | tag: '<[acumos-python-client]>' 21 | realtime_discussion: 22 | type: 'irc' 23 | server: 'freenode.net' 24 | channel: '#acumos' 25 | meetings: 26 | - type: 'zoom' 27 | agenda: 'https://lists.lfai.foundation/calendar' 28 | url: 'https://zoom.us/j/4906631713' 29 | server: 'n/a' 30 | channel: 'n/a' 31 | repeats: 'weekly monday & thursday' 32 | time: '08:30 am ET' 33 | repositories: 34 | - 'acumos-python-client' 35 | committers: 36 | - <<: *acumos_acumos-python-client_ptl 37 | - name: 'Eric Z' 38 | email: 'ezavesky@research.att.com' 39 | company: 'ATT' 40 | id: 'ezsomething' 41 | timezone: 'America/Chicago' 42 | - name: 'Paul Triantafyllou' 43 | email: 'trianta@research.att.com' 44 | company: 'ATT' 45 | id: 'trianta' 46 | timezone: 'America/New_York' 47 | tsc: 48 | approval: 'https://lists.acumos.org/g/tscgeneral' 49 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | ==========================LICENSE_START========================================== 3 | ================================================================================= 4 | Copyright © 2017 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | ================================================================================= 6 | 7 | Unless otherwise specified, all software contained herein is licensed 8 | under the Apache License, Version 2.0 (the "License"); 9 | you may not use this software except in compliance with the License. 10 | You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | 20 | Unless otherwise specified, all documentation contained herein is licensed 21 | under the Creative Commons License, Attribution 4.0 Intl. (the "License"); 22 | you may not use this documentation except in compliance with the License. 23 | You may obtain a copy of the License at 24 | 25 | https://creativecommons.org/licenses/by/4.0/ 26 | 27 | Unless required by applicable law or agreed to in writing, documentation 28 | distributed under the License is distributed on an "AS IS" BASIS, 29 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 30 | See the License for the specific language governing permissions and 31 | limitations under the License. 32 | 33 | ==========================LICENSE_END============================================ -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. ===============LICENSE_START======================================================= 2 | .. Acumos CC-BY-4.0 3 | .. =================================================================================== 4 | .. Copyright (C) 2017-2020 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | .. =================================================================================== 6 | .. This Acumos documentation file is distributed by AT&T and Tech Mahindra 7 | .. under the Creative Commons Attribution 4.0 International License (the "License"); 8 | .. you may not use this file except in compliance with the License. 9 | .. You may obtain a copy of the License at 10 | .. 11 | .. http://creativecommons.org/licenses/by/4.0 12 | .. 13 | .. This file is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | .. ===============LICENSE_END========================================================= 18 | 19 | ==================== 20 | acumos-python-client 21 | ==================== 22 | 23 | .. image:: docs/images/Acumos_logo_white.png 24 | 25 | |Build Status| 26 | 27 | A client library that allows developers to push their Python models to Acumos. 28 | 29 | See our `documentation `__ to get started. 30 | 31 | .. |Build Status| image:: https://jenkins.acumos.org/buildStatus/icon?job=acumos-python-client-tox-verify-master 32 | :target: https://jenkins.acumos.org/job/acumos-python-client-tox-verify-master/ 33 | -------------------------------------------------------------------------------- /acumos-package/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | xunit-results.xml 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # IPython Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | 82 | # virtualenv 83 | venv/ 84 | ENV/ 85 | 86 | # Spyder project settings 87 | .spyderproject 88 | 89 | # Rope project settings 90 | .ropeproject 91 | 92 | .pytest_cache/ 93 | .spyproject/ 94 | -------------------------------------------------------------------------------- /acumos-package/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include docs *.rst 2 | -------------------------------------------------------------------------------- /acumos-package/acumos/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | from ._version import __version__ # noqa 20 | -------------------------------------------------------------------------------- /acumos-package/acumos/_version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | __version__ = '1.0.1' 20 | -------------------------------------------------------------------------------- /acumos-package/acumos/auth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides authentication utilities 21 | """ 22 | import json 23 | from os import makedirs, environ 24 | from os.path import extsep, isfile, join as path_join 25 | from getpass import getpass 26 | 27 | import requests 28 | from appdirs import user_data_dir 29 | from filelock import FileLock 30 | 31 | import acumos 32 | from acumos.exc import AcumosError 33 | from acumos.logging import get_logger 34 | from acumos.utils import load_artifact, dump_artifact 35 | 36 | 37 | _CONFIG_DIR = user_data_dir('acumos') 38 | _CONFIG_PATH = path_join(_CONFIG_DIR, extsep.join(('config', 'json'))) 39 | _LOCK_PATH = path_join(_CONFIG_DIR, extsep.join(('config', 'lock'))) 40 | 41 | _USERNAME_VAR = 'ACUMOS_USERNAME' 42 | _PASSWORD_VAR = 'ACUMOS_PASSWORD' 43 | _TOKEN_VAR = 'ACUMOS_TOKEN' 44 | 45 | logger = get_logger(__name__) 46 | gettoken = getpass 47 | 48 | 49 | def get_jwt(auth_api): 50 | '''Returns the jwt string from config or authentication''' 51 | jwt = environ.get(_TOKEN_VAR) 52 | if not jwt: 53 | jwt = _get_jwt() 54 | if not jwt: 55 | jwt = _authenticate(auth_api) 56 | _set_jwt(jwt) 57 | return jwt 58 | 59 | 60 | def _authenticate(auth_api): 61 | '''Authenticates and returns the jwt string''' 62 | username = environ.get(_USERNAME_VAR) 63 | password = environ.get(_PASSWORD_VAR) 64 | 65 | # user/pass supported for now. use if explicitly provided instead of prompting for token 66 | if username and password: 67 | if auth_api is None: 68 | raise AcumosError('An authentication API is required if using username & password credentials') 69 | 70 | headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} 71 | request_body = {'request_body': {'username': username, 'password': password}} 72 | r = requests.post(auth_api, json=request_body, headers=headers) 73 | 74 | if r.status_code != 200: 75 | raise AcumosError("Authentication failure: {}".format(r.text)) 76 | 77 | jwt = r.json()['jwtToken'] 78 | else: 79 | jwt = gettoken('Enter onboarding token: ') 80 | 81 | return jwt 82 | 83 | 84 | def clear_jwt(): 85 | '''Clears the jwt from config''' 86 | _set_jwt(None) 87 | 88 | 89 | def _set_jwt(token): 90 | '''Sets the jwt in config''' 91 | _configuration(jwt=token) 92 | 93 | 94 | def _get_jwt(): 95 | '''Gets the jwt from config''' 96 | _configuration().get('jwt') 97 | 98 | 99 | def _configuration(**kwargs): 100 | '''Optionally updates and returns the config dict''' 101 | makedirs(_CONFIG_DIR, exist_ok=True) 102 | lock = FileLock(_LOCK_PATH) 103 | 104 | with lock: 105 | config = dict() if not isfile(_CONFIG_PATH) else load_artifact(_CONFIG_PATH, module=json, mode='r') 106 | 107 | config.update(kwargs) 108 | config['version'] = acumos.__version__ 109 | 110 | with lock: 111 | dump_artifact(_CONFIG_PATH, data=config, module=json, mode='w') 112 | 113 | return config 114 | -------------------------------------------------------------------------------- /acumos-package/acumos/exc.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides acumos exceptions 21 | """ 22 | 23 | 24 | class AcumosError(Exception): 25 | pass 26 | -------------------------------------------------------------------------------- /acumos-package/acumos/logging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides logging utilities 21 | """ 22 | import logging 23 | 24 | import acumos 25 | 26 | 27 | _handler = logging.StreamHandler() 28 | _handler.setFormatter(logging.Formatter('[%(levelname)s] %(name)s : %(message)s')) 29 | 30 | _root = logging.getLogger(acumos.__name__) 31 | _root.setLevel(logging.INFO) 32 | _root.handlers = [_handler, ] 33 | _root.propagate = False 34 | 35 | 36 | def get_logger(name): 37 | '''Returns a logger object''' 38 | return logging.getLogger(name) 39 | -------------------------------------------------------------------------------- /acumos-package/acumos/metadata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides metadata generation utilities 21 | """ 22 | import importlib 23 | import sys 24 | from types import ModuleType 25 | from pkg_resources import get_distribution, DistributionNotFound 26 | from os.path import basename, normpath, sep as pathsep 27 | 28 | from acumos.exc import AcumosError 29 | from acumos.modeling import _is_namedtuple 30 | 31 | 32 | BUILTIN_MODULE_NAMES = set(sys.builtin_module_names) 33 | PACKAGE_DIRS = {'site-packages', 'dist-packages'} 34 | SCHEMA_VERSION = '0.6.0' 35 | _SCHEMA = "acumos.schema.model:{}".format(SCHEMA_VERSION) 36 | SCHEMA_VERSION_CLIO = '0.4.0' 37 | _SCHEMA_CLIO = "acumos.schema.model:{}".format(SCHEMA_VERSION_CLIO) 38 | _MEDIA_TYPES = {str: 'text/plain', bytes: 'application/octet-stream', dict: 'application/json', 'protobuf': 'application/vnd.google.protobuf'} 39 | 40 | # known package mappings because Python packaging is madness 41 | _REQ_MAP = { 42 | 'sklearn': 'scikit-learn', 43 | } 44 | 45 | 46 | class Options(object): 47 | ''' 48 | A collection of options that users may wish to specify along with their Acumos model 49 | 50 | Parameters 51 | ---------- 52 | create_microservice : bool, optional 53 | If True, instructs the Acumos platform to eagerly build the model microservice 54 | deploy : bool, optional 55 | If True, ask to deploy model using Jenkins runner (create_microservice should be also True) 56 | license : str, optional 57 | A license to include with the Acumos model. This parameter may either be a path to a license 58 | file, or a string containing the license content. 59 | ''' 60 | __slots__ = ('create_microservice', 'deploy', 'license') 61 | 62 | def __init__(self, create_microservice=True, deploy=False, license=None): 63 | self.create_microservice = create_microservice or deploy 64 | self.deploy = deploy 65 | self.license = license 66 | 67 | 68 | class Requirements(object): 69 | ''' 70 | A collection of optional user-provided Python requirement metadata 71 | 72 | Parameters 73 | ---------- 74 | reqs : Sequence[str], optional 75 | A sequence of pip-installable Python package names 76 | req_map : Dict[str, str] or Dict[module, str], optional 77 | A corrective mapping of Python modules to pip-installable package names. For example 78 | `req_map={'sklearn': 'scikit-learn'}` or `import sklearn; req_map={sklearn: 'scikit-learn'}`. 79 | packages : Sequence[str], optional 80 | A sequence of paths to Python packages (i.e. directories with an __init__.py). Provided Python 81 | packages will be copied along with your model and added to the PYTHONPATH at runtime. 82 | scripts : Sequence[str], optional 83 | A sequence of paths to Python scripts. A path can point to a Python script, or directory 84 | containing Python scripts. If a directory is provided, all Python scripts within the directory 85 | will be included. Provided Python scripts will be copied along with your model and added to the 86 | PYTHONPATH at runtime. 87 | ''' 88 | 89 | __slots__ = ('reqs', 'req_map', 'packages', 'scripts') 90 | 91 | def __init__(self, reqs=None, req_map=None, packages=None, scripts=None): 92 | self.reqs = set() if reqs is None else _safe_set(reqs) 93 | self.req_map = dict() if req_map is None else req_map.copy() 94 | self.req_map.update(_REQ_MAP) 95 | self.packages = set() if packages is None else _safe_set(packages) 96 | self.scripts = set() if scripts is None else _safe_set(scripts) 97 | 98 | @property 99 | def package_names(self): 100 | return frozenset(basename(p.rstrip(pathsep)) for p in self.packages) 101 | 102 | 103 | def _safe_set(input_): 104 | '''Safely returns a set from input''' 105 | return {input_, } if isinstance(input_, str) else set(input_) 106 | 107 | 108 | def create_model_meta(model, name, requirements, encoding='protobuf'): 109 | '''Returns a model metadata dictionary''' 110 | return {'schema': _SCHEMA, 111 | 'runtime': _create_runtime(requirements, encoding), 112 | 'name': name, 113 | 'methods': {name: {'input': {'name': f.input_type.__name__, 114 | 'media_type': [_MEDIA_TYPES[encoding if _is_namedtuple(f.input_type) else f.input_type.__supertype__._raw_type]], 115 | 'metadata': {} if _is_namedtuple(f.input_type) else f.input_type.__supertype__._metadata, 116 | 'description': '' if _is_namedtuple(f.input_type) else f.input_type.__supertype__._doc}, 117 | 'output': {'name': f.output_type.__name__, 118 | 'media_type': [_MEDIA_TYPES[encoding if _is_namedtuple(f.output_type) else f.output_type.__supertype__._raw_type]], 119 | 'metadata': {} if _is_namedtuple(f.output_type) else f.output_type.__supertype__._metadata, 120 | 'description': '' if _is_namedtuple(f.output_type) else f.output_type.__supertype__._doc}, 121 | 'description': f.description} for name, f in model.methods.items()}} 122 | 123 | 124 | def create_model_meta_clio(model, name, requirements, encoding='protobuf'): 125 | '''Returns a model metadata dictionary''' 126 | return {'schema': _SCHEMA_CLIO, 127 | 'runtime': _create_runtime(requirements, encoding), 128 | 'name': name, 129 | 'methods': {name: {'input': f.input_type.__name__, 130 | 'output': f.output_type.__name__, 131 | 'description': f.description} for name, f in model.methods.items()}} 132 | 133 | 134 | def _create_runtime(requirements, encoding='protobuf'): 135 | '''Returns a runtime dict''' 136 | reqs = _gather_requirements(requirements) 137 | return {'name': 'python', 138 | 'version': '.'.join(map(str, sys.version_info[:3])), 139 | 'dependencies': _create_dependencies(reqs)} 140 | 141 | 142 | def _gather_requirements(requirements): 143 | '''Yields (name, version) tuples of required 3rd party Python packages''' 144 | for req_name in _filter_requirements(requirements): 145 | yield _get_distribution(req_name) 146 | 147 | 148 | def _filter_requirements(requirements): 149 | '''Returns a set required 3rd party Python package names''' 150 | # first get all non-stdlib requirement names 151 | req_names = (n for n in map(_get_requirement_name, requirements.reqs) if not _in_stdlib(n)) 152 | 153 | # then apply any user-provided requirement mappings 154 | req_map = {_get_requirement_name(k): v for k, v in requirements.req_map.items()} 155 | mapped_reqs = {req_map.get(r, r) for r in req_names} 156 | 157 | # finally remove user-provided custom package names, as they won't exist in pip 158 | filtered_reqs = mapped_reqs - requirements.package_names 159 | return filtered_reqs 160 | 161 | 162 | def _get_requirement_name(req): 163 | '''Returns the str name of a requirement''' 164 | if isinstance(req, ModuleType): 165 | name = req.__name__ 166 | elif isinstance(req, str): 167 | name = req 168 | else: 169 | raise AcumosError("Requirement {} is invalid; must be ModuleType or string".format(req)) 170 | return name 171 | 172 | 173 | def _get_distribution(req_name): 174 | '''Returns (name, version) tuple given a requirement''' 175 | try: 176 | return str(get_distribution(req_name).as_requirement()).split('==') 177 | except DistributionNotFound: 178 | raise AcumosError("Module {} was detected as a dependency, but not found as a pip installed package. Use acumos.session.Requirements to declare custom packages or map module names to pip-installable names (e.g. Requirements(req_map=dict(PIL='pillow')) )".format(req_name)) 179 | 180 | 181 | def _in_stdlib(module_name): 182 | '''Returns True if the package name is part of the standard library (not airtight)''' 183 | if module_name in BUILTIN_MODULE_NAMES: 184 | return True 185 | 186 | try: 187 | install_path = importlib.import_module(module_name).__file__ 188 | except ImportError: 189 | return False 190 | 191 | if not install_path.startswith(normpath(sys.base_prefix)): 192 | return False 193 | 194 | dirs = set(normpath(install_path).split(pathsep)) 195 | return not PACKAGE_DIRS & dirs 196 | 197 | 198 | def _create_dependencies(req_tuples): 199 | '''Returns a dict containing model dependency metadata''' 200 | return { 201 | 'pip': { 202 | 'indexes': [], 203 | 'requirements': [{'name': n, 'version': v} for n, v in req_tuples] 204 | }, 205 | 'conda': { 206 | 'channels': [], 207 | 'requirements': [] 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /acumos-package/acumos/modeling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides modeling utilities 21 | """ 22 | from inspect import getfullargspec, getdoc, isclass 23 | from typing import NamedTuple, List, Dict, TypeVar, Generic, NewType 24 | from enum import Enum 25 | from collections import namedtuple, OrderedDict 26 | 27 | try: 28 | from typing import NoReturn # odd import error occurs in Python 3.5 testing 29 | except ImportError: 30 | NoReturn = None 31 | 32 | import numpy as np 33 | 34 | from acumos.exc import AcumosError 35 | from acumos.utils import inspect_type, reraise 36 | 37 | 38 | _NUMPY_PRIMITIVES = {np.int64, np.int32, np.float64, np.float32} 39 | _PYTHON_PRIMITIVES = {int, float, str, bool, bytes} 40 | 41 | _VALID_PRIMITIVES = _PYTHON_PRIMITIVES | _NUMPY_PRIMITIVES 42 | _VALID_TYPES = {NamedTuple, List, Dict, Enum} | _VALID_PRIMITIVES 43 | 44 | _dtype2prim = {np.dtype(t): t for t in _NUMPY_PRIMITIVES} 45 | 46 | 47 | class Empty(tuple): 48 | '''Empty NamedTuple-ish type that consumes any input and returns an empty tuple''' 49 | __slots__ = () 50 | _fields = () 51 | __annotations__ = OrderedDict() 52 | 53 | def __new__(cls, *args, **kwargs): 54 | return super().__new__(cls) 55 | 56 | 57 | _RESERVED_TYPES = {Empty, } 58 | _RESERVED_NAMES = {t.__name__ for t in _RESERVED_TYPES} 59 | 60 | 61 | RawTypeVar = TypeVar('RawTypeVar', str, bytes, dict) 62 | 63 | 64 | class Raw(Generic[RawTypeVar]): 65 | '''Represents raw types that will not result in a generated message''' 66 | def __init__(self, raw_type: RawTypeVar, metadata: dict, doc: str) -> None: 67 | self._raw_type = raw_type 68 | self._metadata = metadata 69 | self._doc = doc 70 | 71 | @property 72 | def raw_type(self): 73 | return self._raw_type 74 | 75 | @property 76 | def metadata(self): 77 | return self._metadata 78 | 79 | @property 80 | def description(self): 81 | return self._doc 82 | 83 | 84 | class Model(object): 85 | ''' 86 | A container of user-provided functions that can be on-boarded as a model in Acumos 87 | 88 | Parameters 89 | ---------- 90 | kwargs : Arbitrary keyword arguments 91 | Keys are used as function names and values are user-defined functions 92 | ''' 93 | 94 | def __init__(self, **kwargs): 95 | if not kwargs: 96 | raise AcumosError('No functions were provided to Model') 97 | 98 | self._methods = {name: _create_function(func, name) if not isinstance(func, Function) else func 99 | for name, func in kwargs.items()} 100 | for k, v in self._methods.items(): 101 | setattr(self, k, v) 102 | 103 | @property 104 | def methods(self): 105 | return self._methods 106 | 107 | 108 | def _create_function(f, name=None): 109 | '''Returns an initialized Function object''' 110 | wrapped, input_type, output_type = _wrap_function(f, name) 111 | return Function(f, wrapped, input_type, output_type) 112 | 113 | 114 | class Function(namedtuple('Function', 'inner wrapped input_type output_type')): 115 | '''Container of original and wrapped functions along with signature metadata''' 116 | 117 | @property 118 | def description(self): 119 | doc = getdoc(self.inner) 120 | return '' if doc is None else doc 121 | 122 | 123 | def _wrap_function(f, name=None): 124 | '''Returns a function that has its arguments and return wrapped in NameTuple types''' 125 | spec = getfullargspec(f) 126 | anno = spec.annotations 127 | 128 | if 'return' not in anno: 129 | raise AcumosError("Function {} must have a return annotation".format(f)) 130 | 131 | for a in spec.args: 132 | if a not in anno: 133 | raise AcumosError("Function argument {} does not have an annotation".format(a)) 134 | 135 | if name is None: 136 | name = f.__name__ 137 | 138 | title = ''.join(s for s in name.title().split('_')) 139 | 140 | field_types = [(a, anno[a]) for a in spec.args] 141 | ret_type = anno['return'] 142 | 143 | args_are_raw = any([is_raw_type(field_type) for field_name, field_type in field_types]) 144 | ret_is_raw = is_raw_type(ret_type) 145 | 146 | if args_are_raw and len(field_types) > 1: 147 | raise AcumosError("Cannot process a function with more than 1 argument when using raw types as input") 148 | 149 | if not args_are_raw: 150 | for field_name, field_type in field_types: 151 | with reraise('Function {} argument {} is invalid', (name, field_name)): 152 | _assert_valid_type(field_type) 153 | 154 | if not ret_is_raw and ret_type not in (None, NoReturn): 155 | if ret_type not in (None, NoReturn): 156 | with reraise('Function {} return type {} is invalid', (name, ret_type)): 157 | _assert_valid_type(ret_type) 158 | 159 | wrap_input = True 160 | wrap_output = True 161 | 162 | if args_are_raw or _already_wrapped(field_types): 163 | input_type = field_types[0][1] 164 | wrap_input = False 165 | else: 166 | input_type = _create_input_type(title, field_types) 167 | with reraise('Function {} wrapped input type is invalid', (name,)): 168 | _assert_valid_type(input_type) 169 | 170 | if ret_is_raw or _is_namedtuple(ret_type): 171 | output_type = ret_type 172 | wrap_output = False 173 | else: 174 | output_type = _create_ret_type(title, ret_type) 175 | with reraise('Function {} wrapped output type is invalid', (name,)): 176 | _assert_valid_type(output_type) 177 | 178 | wrapper = _get_wrapper(wrap_input, wrap_output) 179 | return wrapper(f, input_type, output_type), input_type, output_type 180 | 181 | 182 | def _get_wrapper(wrap_input: bool, wrap_output: bool): 183 | """Find a wrapper for the function""" 184 | if wrap_input: 185 | if wrap_output: 186 | return _create_wrapper_both 187 | return _create_wrapper_args 188 | if wrap_output: 189 | return _create_wrapper_ret 190 | return lambda f, input_type, output_type: f 191 | 192 | 193 | def _already_wrapped(field_types): 194 | '''Returns True if the field types are already considered wrapped''' 195 | return len(field_types) == 1 and _is_namedtuple(field_types[0][1]) 196 | 197 | 198 | def _is_namedtuple(t): 199 | '''Returns True if type `t` is a NamedTuple type''' 200 | return isclass(t) and issubclass(t, tuple) and hasattr(t, '__annotations__') 201 | 202 | 203 | def _is_subclass(c, t): 204 | '''Returns True if c is a subclass of t''' 205 | return isclass(c) and issubclass(c, t) 206 | 207 | 208 | def _create_input_type(name, field_types): 209 | '''Generates a NamedTuple for input arguments''' 210 | return NamedTuple("{}In".format(name), field_types) if field_types else Empty 211 | 212 | 213 | def _create_ret_type(name, ret_type): 214 | '''Generates a NamedTuple for a function return''' 215 | return NamedTuple("{}Out".format(name), [('value', ret_type)]) if ret_type not in (None, NoReturn) else Empty 216 | 217 | 218 | def _create_wrapper_both(f, input_type, output_type): 219 | '''Returns a wrapped function that accepts and returns NamedTuple types''' 220 | def wrapped(in_: input_type) -> output_type: 221 | ret = f(*in_) 222 | return output_type(ret) 223 | return wrapped 224 | 225 | 226 | def _create_wrapper_args(f, input_type, output_type): 227 | '''Returns a wrapped function that accepts and returns NamedTuple types''' 228 | def wrapped(in_: input_type) -> output_type: 229 | return f(*in_) 230 | return wrapped 231 | 232 | 233 | def _create_wrapper_ret(f, input_type, output_type): 234 | '''Returns a wrapped function that accepts and returns NamedTuple types''' 235 | def wrapped(in_: input_type) -> output_type: 236 | ret = f(in_) 237 | return output_type(ret) 238 | return wrapped 239 | 240 | 241 | def _assert_valid_type(t, container=None): 242 | '''Raises AcumosError if the input type contains an invalid type''' 243 | 244 | inspected = inspect_type(t) 245 | 246 | if t in _VALID_PRIMITIVES: 247 | pass 248 | 249 | elif _is_namedtuple(t): 250 | if t.__name__ in _RESERVED_NAMES and t not in _RESERVED_TYPES: 251 | raise AcumosError("NamedTuple {} cannot use a reserved name: {}".format(t, _RESERVED_NAMES)) 252 | 253 | for tt in t.__annotations__.values(): 254 | _assert_valid_type(tt) 255 | 256 | elif _is_subclass(inspected.origin, List): 257 | if container is not None: 258 | raise AcumosError( 259 | "List types cannot be nested within {} types. Use NamedTuple instead" 260 | .format(inspect_type(container).origin.__name__)) 261 | 262 | _assert_valid_type(inspected.args[0], container=List) 263 | 264 | elif _is_subclass(inspected.origin, Dict): 265 | if container is not None: 266 | raise AcumosError( 267 | "Dict types cannot be nested within {} types. Use NamedTuple instead" 268 | .format(inspect_type(container).origin.__name__)) 269 | 270 | key_type, value_type = inspected.args 271 | 272 | if key_type is not str: 273 | raise AcumosError('Dict keys must be str type') 274 | 275 | _assert_valid_type(value_type, container=Dict) 276 | 277 | elif _is_subclass(t, Enum): 278 | pass 279 | 280 | else: 281 | raise AcumosError("Type {} is not one of the supported types: {}".format(t, _VALID_TYPES)) 282 | 283 | 284 | def create_dataframe(name, df): 285 | '''Returns a NamedTuple type corresponding to a pandas DataFrame instance''' 286 | import pandas as pd 287 | 288 | if not isinstance(df, pd.DataFrame): 289 | raise AcumosError('Input `df` must be a pandas.DataFrame') 290 | 291 | dtypes = list(df.dtypes.iteritems()) 292 | for field_name, dtype in dtypes: 293 | if dtype not in _dtype2prim: 294 | raise AcumosError("DataFrame column '{}' has an unsupported type '{}'. Supported types are: {}".format(field_name, dtype, _NUMPY_PRIMITIVES)) 295 | 296 | field_types = [(n, List[_dtype2prim[dt]]) for n, dt in dtypes] 297 | df_type = NamedTuple(name, field_types) 298 | return df_type 299 | 300 | 301 | def create_namedtuple(name, field_types): 302 | '''Returns a NamedTuple type''' 303 | return NamedTuple(name, field_types) 304 | 305 | 306 | def new_type(raw_type, name, metadata=None, doc=None): 307 | '''Returns a user specified raw type''' 308 | return NewType(name, Raw(raw_type, metadata, doc)) 309 | 310 | 311 | def is_raw_type(_type: type) -> bool: 312 | """Checks if a type is Raw""" 313 | try: 314 | return type(_type.__supertype__) == Raw 315 | except AttributeError: 316 | return False 317 | -------------------------------------------------------------------------------- /acumos-package/acumos/pickler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides custom pickle utilities 21 | """ 22 | import sys 23 | import json 24 | import inspect 25 | import contextlib 26 | import tempfile 27 | from os import makedirs 28 | from os.path import basename, isdir, isfile, join as path_join 29 | from copy import deepcopy 30 | from functools import partial 31 | from typing import Dict, List 32 | from typing_inspect import NEW_TYPING 33 | from types import ModuleType 34 | from importlib import import_module 35 | 36 | import dill 37 | 38 | from acumos.modeling import _is_namedtuple, create_namedtuple, Empty 39 | from acumos.exc import AcumosError 40 | from acumos.utils import namedtuple_field_types 41 | 42 | 43 | _DEFAULT_MODULES = ('acumos', 'dill') 44 | _BLACKLIST = {'builtins', } 45 | _DEFAULT = 'default' 46 | 47 | _contexts = dict() 48 | 49 | dump_model = partial(dill.dump, recurse=True) 50 | dumps_model = partial(dill.dumps, recurse=True) 51 | load_model = dill.load 52 | loads_model = dill.loads 53 | 54 | 55 | if not NEW_TYPING: 56 | from typing import GenericMeta 57 | 58 | def _save_annotation(pickler, obj): 59 | '''Workaround for dill annotation serialization bug''' 60 | if obj.__origin__ in (Dict, List): 61 | # recursively save object 62 | t = obj.__origin__ 63 | args = obj.__args__ 64 | pickler.save_reduce(_load_annotation, (t, args), obj=obj) 65 | else: 66 | # eventually hit base type, then use stock pickling logic. temp revert prevents infinite recursion 67 | t = obj 68 | args = None 69 | with _revert_dispatch(GenericMeta): 70 | pickler.save_reduce(_load_annotation, (t, args), obj=obj) 71 | 72 | def _load_annotation(t, args): 73 | '''Workaround for dill annotation serialization bug''' 74 | if t is Dict and args is not None: 75 | return Dict[args[0], args[1]] 76 | elif t is List and args is not None: 77 | return List[args[0]] 78 | else: 79 | return t 80 | 81 | dill.Pickler.dispatch[GenericMeta] = _save_annotation 82 | 83 | 84 | def _save_namedtuple(pickler, obj): 85 | '''Workaround for dill NamedTuple serialization bug''' 86 | field_types = namedtuple_field_types(obj) 87 | pickler.save_reduce(_load_namedtuple, (obj.__name__, field_types), obj=obj) 88 | 89 | 90 | def _load_namedtuple(name, field_types): 91 | '''Workaround for dill NamedTuple serialization bug''' 92 | return create_namedtuple(name, [(k, v) for k, v in field_types.items()]) 93 | 94 | 95 | @contextlib.contextmanager 96 | def _revert_dispatch(t): 97 | '''Temporarily removes a type from the dispatch table to use existing serialization logic''' 98 | f = dill.Pickler.dispatch.pop(t) 99 | try: 100 | yield 101 | finally: 102 | dill.Pickler.dispatch[t] = f 103 | 104 | 105 | def _save_keras(pickler, obj): 106 | '''Serializes a keras model to a context directory''' 107 | base_mod_name = _get_base_module(obj).__name__ 108 | if base_mod_name == 'keras': 109 | import keras 110 | else: 111 | from tensorflow import keras 112 | 113 | backend = keras.backend.backend 114 | load_func = keras.models.load_model 115 | model_cls = keras.models.Model 116 | 117 | context = get_context() 118 | model_subdir = context.create_subdir() 119 | model_abspath, model_relpath = _add_file(model_subdir, 'model.h5') 120 | 121 | obj.save(model_abspath) # /path/to/context/root/abc123/model.h5 122 | context.add_module('h5py') # needed for keras model serialization 123 | context.add_module(backend()) # adds name of active keras backend 124 | 125 | # check for custom or contrib layer modules 126 | custom_objects = {} 127 | for layer in _get_keras_layers(obj, model_cls): 128 | module = _get_base_module(layer) 129 | if module.__name__ != base_mod_name: 130 | context.add_module(module) 131 | layer_cls = layer.__class__ 132 | custom_objects[layer_cls.__name__] = layer_cls 133 | 134 | # store subpath because context root can change, and special layer classes to have them imported before load 135 | pickler.save_reduce(_load_keras, (model_relpath, custom_objects, load_func), obj=obj) 136 | 137 | 138 | def _get_keras_layers(model, model_cls): 139 | '''Recursively walks a keras model and returns its layers''' 140 | for layer in model.layers: 141 | if isinstance(layer, model_cls): 142 | yield from _get_keras_layers(layer, model_cls) 143 | else: 144 | yield layer 145 | 146 | 147 | def _load_keras(model_relpath, custom_objects, load_func): 148 | '''Loads a keras model from a context subdirectory''' 149 | context = get_context() 150 | model_path = context.build_path(*model_relpath) # /different/path/to/context/root/abc123/model.h5 151 | model = load_func(model_path, custom_objects=custom_objects) 152 | return model 153 | 154 | 155 | def _save_tf_tensor(pickler, tensor): 156 | '''Saves a TensorFlow tensor object''' 157 | import tensorflow as tf 158 | pickler.save_reduce(_load_tf_tensor, (tensor.name, tensor.graph, tf.get_default_session()), obj=tensor) 159 | 160 | 161 | def _load_tf_tensor(name, graph, session): 162 | '''Loads a TensorFlow tensor object''' 163 | return graph.get_tensor_by_name(name) 164 | 165 | 166 | def _save_tf_operation(pickler, op): 167 | '''Saves a TensorFlow operation object''' 168 | import tensorflow as tf 169 | pickler.save_reduce(_load_tf_operation, (op.name, op.graph, tf.get_default_session()), obj=op) 170 | 171 | 172 | def _load_tf_operation(name, graph, session): 173 | '''Loads a TensorFlow operation object''' 174 | return graph.get_operation_by_name(name) 175 | 176 | 177 | def _save_tf_session(pickler, session): 178 | '''Saves a TensorFlow session''' 179 | import tensorflow as tf 180 | 181 | context = get_context() 182 | model_subdir = context.create_subdir() 183 | model_abspath, model_relpath = _add_file(model_subdir, 'model') 184 | 185 | graph = session.graph 186 | with graph.as_default(): 187 | saver = tf.train.Saver(allow_empty=True) 188 | saver.save(session, model_abspath, write_meta_graph=False) # don't export meta graph twice 189 | 190 | pickler.save_reduce(_load_tf_session, (model_relpath, session.__class__, graph), obj=session) 191 | 192 | 193 | def _load_tf_session(model_subpath, session_cls, graph): 194 | '''Loads a TensorFlow session''' 195 | import tensorflow as tf 196 | 197 | context = get_context() 198 | model_path = context.build_path(*model_subpath) 199 | 200 | with graph.as_default(): 201 | sess = session_cls() 202 | saver = tf.train.Saver(allow_empty=True) 203 | saver.restore(sess, model_path) 204 | return sess 205 | 206 | 207 | def _save_tf_graph(pickler, graph): 208 | '''Saves a TensorFlow graph''' 209 | import tensorflow as tf 210 | 211 | context = get_context() 212 | model_subdir = context.create_subdir() 213 | graph_abspath, graph_relpath = _add_file(model_subdir, 'graph.meta') 214 | 215 | with graph.as_default(): 216 | tf.train.export_meta_graph(graph_abspath) 217 | pickler.save_reduce(_load_tf_graph, (graph_relpath, ), obj=graph) 218 | 219 | 220 | def _load_tf_graph(graph_relpath): 221 | '''Loads a TensorFlow graph''' 222 | import tensorflow as tf 223 | 224 | context = get_context() 225 | graph_abspath = context.build_path(*graph_relpath) 226 | 227 | graph = tf.Graph() 228 | with graph.as_default(): 229 | tf.train.import_meta_graph(graph_abspath) 230 | return graph 231 | 232 | 233 | def _add_file(subdir, name): 234 | '''Helper function which returns the absolute and context-relative path of a file to be added''' 235 | file_abspath = path_join(subdir, name) 236 | file_relpath = (basename(subdir), name) 237 | return file_abspath, file_relpath 238 | 239 | 240 | _CUSTOM_DISPATCH = { 241 | 'keras.engine.training.Model': _save_keras, 242 | 'tensorflow.python.framework.ops.Tensor': _save_tf_tensor, 243 | 'tensorflow.python.framework.ops.Operation': _save_tf_operation, 244 | 'tensorflow.python.client.session.BaseSession': _save_tf_session, 245 | 'tensorflow.python.framework.ops.Graph': _save_tf_graph, 246 | 'tensorflow.python.keras.engine.training.Model': _save_keras, 247 | } 248 | 249 | 250 | @contextlib.contextmanager 251 | def _patch_dill(): 252 | '''Temporarily patches the dill Pickler dispatch table to support custom serialization within a context''' 253 | try: 254 | dispatch = dill.Pickler.dispatch 255 | dill.Pickler.dispatch = deepcopy(dispatch) 256 | 257 | pickler_save = dill.Pickler.save 258 | 259 | def wrapped_save(pickler, obj, save_persistent_id=True): 260 | '''Hook that intercepts objects about to be saved''' 261 | _catch_object(obj) 262 | 263 | if _is_namedtuple(obj) and obj is not Empty: 264 | _save_namedtuple(pickler, obj) 265 | else: 266 | pickler_save(pickler, obj, save_persistent_id) 267 | 268 | dill.Pickler.save = wrapped_save 269 | yield 270 | finally: 271 | dill.Pickler.dispatch = dispatch 272 | dill.Pickler.save = pickler_save 273 | 274 | 275 | def _catch_object(obj): 276 | '''Inspects object and executes custom serialization / bookkeeping logic''' 277 | 278 | # dynamically extend dispatch table to prevent unnecessary imports / dependencies 279 | obj_type = obj if inspect.isclass(obj) else type(obj) 280 | if obj_type not in dill.Pickler.dispatch: 281 | for path in _get_mro_paths(obj_type): 282 | if path in _CUSTOM_DISPATCH: 283 | dill.Pickler.dispatch[obj_type] = _CUSTOM_DISPATCH[path] 284 | 285 | base_module = _get_base_module(obj) 286 | if base_module is not None and base_module.__name__ not in _BLACKLIST: 287 | context = get_context() 288 | context.add_module(base_module) 289 | 290 | 291 | def _get_base_module(obj): 292 | '''Returns the base module for a given object''' 293 | module = inspect.getmodule(obj) 294 | if module is not None: 295 | base_name, _, _ = module.__name__.partition('.') 296 | base_module = sys.modules[base_name] 297 | else: 298 | base_module = None 299 | return base_module 300 | 301 | 302 | def _get_mro_paths(type_): 303 | '''Yields import path string for each entry in `inspect.getmro`''' 304 | for t in inspect.getmro(type_): 305 | yield "{}.{}".format(t.__module__, t.__name__) 306 | 307 | 308 | class AcumosContext(object): 309 | '''Represents a workspace for a model that is being dumped''' 310 | 311 | def __init__(self, root_dir): 312 | if not isdir(root_dir): 313 | raise AcumosError("AcumosContext root directory {} does not exist".format(root_dir)) 314 | self._modules = set() 315 | self._root_dir = root_dir 316 | self._params_path = path_join(root_dir, 'context.json') 317 | self.parameters = self._load_params() 318 | 319 | for mod in _DEFAULT_MODULES: 320 | self.add_module(mod) 321 | 322 | def create_subdir(self, *paths, exist_ok=False): 323 | '''Creates a new directory within the context root and returns the absolute path''' 324 | if not paths: 325 | tdir = tempfile.mkdtemp(dir=self._root_dir) 326 | else: 327 | tdir = path_join(self._root_dir, *paths) 328 | makedirs(tdir, exist_ok=exist_ok) 329 | return tdir 330 | 331 | def build_path(self, *paths): 332 | '''Returns an absolute path starting from the context root''' 333 | return path_join(self._root_dir, *paths) 334 | 335 | def add_module(self, module): 336 | '''Adds a module to the context module set''' 337 | if isinstance(module, str): 338 | try: 339 | module = import_module(module) 340 | except ImportError: 341 | raise AcumosError("Module '{}' was identified as a dependency, but cannot be imported. Ensure that it is installed and available".format(module)) 342 | elif not isinstance(module, ModuleType): 343 | raise AcumosError("Module must be of type str or types.ModuleType, not {}".format(type(module))) 344 | 345 | self._modules.add(module) 346 | 347 | @property 348 | def abspath(self): 349 | '''Absolute path of the context root directory''' 350 | return self._root_dir 351 | 352 | @property 353 | def basename(self): 354 | '''Base name of the context root directory''' 355 | return basename(self._root_dir) 356 | 357 | @property 358 | def modules(self): 359 | '''The set of all modules (i.e. typing.ModuleType) identified as dependencies''' 360 | return frozenset(self._modules) 361 | 362 | @property 363 | def packages(self): 364 | '''The set of all base packages (i.e. typing.ModuleType with a package) identified as dependencies''' 365 | return frozenset(mod for mod in self._modules if mod.__package__) 366 | 367 | @property 368 | def package_names(self): 369 | '''The set of all base package names identified as dependencies''' 370 | return frozenset(mod.__name__ for mod in self.packages) 371 | 372 | @property 373 | def scripts(self): 374 | '''The set of all scripts (i.e. typing.ModuleType with no package) identified as dependencies''' 375 | return frozenset(script for script in (mod for mod in self._modules if not mod.__package__) if script.__name__ != '__main__') 376 | 377 | @property 378 | def script_names(self): 379 | '''The set of all script names identified as dependencies''' 380 | return frozenset(mod.__name__ for mod in self.scripts) 381 | 382 | def _load_params(self): 383 | '''Returns a parameters dict''' 384 | if isfile(self._params_path): 385 | with open(self._params_path) as f: 386 | return json.load(f) 387 | else: 388 | return dict() 389 | 390 | def save_params(self): 391 | '''Saves a parameters json file within the context root''' 392 | with open(self._params_path, 'w') as f: 393 | json.dump(self.parameters, f) 394 | 395 | 396 | @contextlib.contextmanager 397 | def AcumosContextManager(rootdir=None, name=_DEFAULT): 398 | '''Context manager that provides a AcumosContext object''' 399 | with _patch_dill(): 400 | if name in _contexts: 401 | raise AcumosError("AcumosContext '{}' has already been created. Use `get_context` to access it.".format(name)) 402 | try: 403 | with _DirManager(rootdir) as rootdir: 404 | context = AcumosContext(rootdir) 405 | _contexts[name] = context 406 | yield context 407 | context.save_params() 408 | finally: 409 | del _contexts[name] 410 | 411 | 412 | @contextlib.contextmanager 413 | def _DirManager(dir_=None): 414 | '''Wrapper that passes dir_ through or creates a temporary directory''' 415 | if dir_ is not None: 416 | if not isdir(dir_): 417 | raise AcumosError("Provided AcumosContext rootdir {} does not exist".format(dir_)) 418 | yield dir_ 419 | else: 420 | with tempfile.TemporaryDirectory() as tdir: 421 | yield tdir 422 | 423 | 424 | def get_context(name=_DEFAULT): 425 | '''Returns an existing AcumosContext''' 426 | if name in _contexts: 427 | return _contexts[name] 428 | else: 429 | raise AcumosError("AcumosContext '{}' has not been created".format(name)) 430 | -------------------------------------------------------------------------------- /acumos-package/acumos/protogen.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides protobuf generation utilities 21 | """ 22 | import shutil 23 | from subprocess import PIPE, Popen 24 | from tempfile import TemporaryDirectory 25 | from collections import defaultdict 26 | from itertools import chain as iterchain 27 | from os.path import dirname, isfile, join as path_join 28 | from os import makedirs 29 | 30 | import numpy as np 31 | 32 | from acumos.exc import AcumosError 33 | from acumos.modeling import List, Dict, Enum, _is_namedtuple, is_raw_type, Empty 34 | from acumos.utils import inspect_type, namedtuple_field_types 35 | 36 | 37 | _PROTO_SYNTAX = 'syntax = "proto3";' 38 | 39 | _package_template = 'package {name};' 40 | 41 | _message_template = ''' 42 | message {name} {{ 43 | {msg_def} 44 | }}''' 45 | 46 | _enum_template = ''' 47 | enum {name} {{ 48 | {enum_def} 49 | }}''' 50 | 51 | _service_template = ''' 52 | service {name} {{ 53 | {service_def} 54 | }}''' 55 | 56 | _rpc_template = 'rpc {name} ({msg_in}) returns ({msg_out});' 57 | 58 | 59 | class NestedTypeError(AcumosError): 60 | pass 61 | 62 | 63 | _type_lookup = { 64 | int: 'int64', 65 | str: 'string', 66 | float: 'double', 67 | bool: 'bool', 68 | bytes: 'bytes', 69 | np.int64: 'int64', 70 | np.int32: 'int32', 71 | np.float32: 'float', 72 | np.float64: 'double', 73 | np.bool: 'bool'} 74 | 75 | 76 | def compile_protostr(proto_str, package_name, module_name, out_dir): 77 | '''Compiles a Python module from a protobuf definition str and returns the module abspath''' 78 | _assert_protoc() 79 | 80 | with TemporaryDirectory() as tdir: 81 | protopath = path_join(tdir, package_name, "{}.proto".format(module_name)) 82 | 83 | makedirs(dirname(protopath)) 84 | with open(protopath, 'w') as f: 85 | f.write(proto_str) 86 | 87 | cmd = "protoc --python_out {tdir} --proto_path {tdir} {protopath}".format(tdir=tdir, protopath=protopath).split() 88 | p = Popen(cmd, stderr=PIPE) 89 | _, err = p.communicate() 90 | if p.returncode != 0: 91 | raise AcumosError("A failure occurred while generating source code from protobuf: {}".format(err)) 92 | 93 | gen_module_name = "{}_pb2.py".format(module_name) 94 | gen_module_path = path_join(tdir, package_name, gen_module_name) 95 | if not isfile(gen_module_path): 96 | raise AcumosError("An unknown failure occurred while generating Python module {}".format(gen_module_path)) 97 | 98 | out_module_path = path_join(out_dir, gen_module_name) 99 | shutil.copy(gen_module_path, out_module_path) 100 | return out_module_path 101 | 102 | 103 | def _assert_protoc(): 104 | '''Raises an AcumosError if protoc is not found''' 105 | if shutil.which('protoc') is None: 106 | raise AcumosError('The protocol buffers compiler `protoc` was not found. Verify that it is installed and visible in $PATH') 107 | 108 | 109 | def model2proto(model, package_name): 110 | '''Converts a Model object to a protobuf schema string''' 111 | all_types = (iterchain(*(iterchain(_proto_iter(f.input_type), 112 | _proto_iter(f.output_type)) for f in model.methods.values()))) 113 | 114 | unique_types = _require_unique(all_types) 115 | type_names = set(t.__name__ for t in unique_types) 116 | 117 | msg_defs = tuple(_nt2proto(t, type_names) if _is_namedtuple(t) else _enum2proto(t) for t in unique_types) 118 | service_def = _gen_service(model) 119 | package_def = _package_template.format(name=package_name) 120 | 121 | defs = (_PROTO_SYNTAX, package_def, service_def) + msg_defs 122 | return '\n'.join(defs) 123 | 124 | 125 | def _proto_iter(nt): 126 | '''Recursively yields all types contained within the NamedTuple relevant to protobuf gen''' 127 | if is_raw_type(nt): 128 | # Empty is used here as a placeholder for raw types 129 | yield Empty 130 | return 131 | 132 | if _is_namedtuple(nt): 133 | yield nt 134 | 135 | for t in nt.__annotations__.values(): 136 | inspected = inspect_type(t) 137 | if _is_namedtuple(t): 138 | yield from _proto_iter(t) 139 | elif issubclass(inspected.origin, Enum): 140 | yield t 141 | elif issubclass(inspected.origin, List) or issubclass(inspected.origin, Dict): 142 | for tt in inspected.args: 143 | yield from _proto_iter(tt) 144 | 145 | 146 | def _require_unique(types): 147 | '''Returns a list of unique types. Raises AcumosError if named types are not uniquely defined''' 148 | types_by_name = defaultdict(list) 149 | for _type in types: 150 | types_by_name[_type.__name__].append(_type) 151 | 152 | for name, types in types_by_name.items(): 153 | if len(types) > 1 and not all(_types_equal(types[0], t) for t in types[1:]): 154 | raise AcumosError("Multiple definitions found for type {}: {}".format(name, types)) 155 | 156 | return [types[0] for types in types_by_name.values()] 157 | 158 | 159 | def _types_equal(t1, t2, *, ignore_type_name: bool = False): 160 | '''Returns True if t1 and t2 types are equal. Can't override __eq__ on NamedTuple unfortunately.''' 161 | if _is_namedtuple(t1) and _is_namedtuple(t2): 162 | names_match = ignore_type_name or t1.__name__ == t2.__name__ 163 | ft1, ft2 = namedtuple_field_types(t1), namedtuple_field_types(t2) 164 | keys_match = ft1.keys() == ft2.keys() 165 | values_match = all(_types_equal(v1, v2) for v1, v2 in zip(ft1.values(), ft2.values())) 166 | return names_match and keys_match and values_match 167 | 168 | t1_inspected = inspect_type(t1) 169 | t2_inspected = inspect_type(t2) 170 | 171 | if issubclass(t1_inspected.origin, Enum) and issubclass(t2_inspected.origin, Enum): 172 | names_match = ignore_type_name or t1.__name__ == t2.__name__ 173 | enums_match = [(e.name, e.value) for e in t1] == [(e.name, e.value) for e in t2] 174 | return names_match and enums_match 175 | 176 | else: 177 | return t1 == t2 178 | 179 | 180 | def _nt2proto(nt, type_names): 181 | '''Converts a NamedTuple definition to a protobuf schema str''' 182 | msg_def = '\n'.join(_field2proto(field, type_, index, type_names) 183 | for index, (field, type_) in enumerate(namedtuple_field_types(nt).items(), 1)) 184 | return _message_template.format(name=nt.__name__, msg_def=msg_def) 185 | 186 | 187 | def _enum2proto(enum): 188 | '''Converts an Enum definition to a protobuf schema str''' 189 | names, values = zip(*((e.name, e.value) for e in enum)) 190 | return _gen_enum(names, enum.__name__, values) 191 | 192 | 193 | def _gen_enum(enums, name, values=None): 194 | '''Produces an enum protobuf definition. An UNKNOWN enum is added if values are provided without a 0''' 195 | if values is not None: 196 | if 0 not in values: 197 | enums = ('UNKNOWN', ) + tuple(enums) 198 | values = (0, ) + tuple(values) 199 | 200 | enums, values = zip(*sorted(zip(enums, values), key=lambda x: x[1])) 201 | 202 | # the first enum value must be zero in proto3 203 | zidx = values.index(0) 204 | enums = enums[zidx:] + enums[:zidx] 205 | values = values[zidx:] + values[:zidx] 206 | else: 207 | values = range(len(enums)) 208 | 209 | enum_def = '\n'.join((" {} = {};".format(name, idx) for name, idx in zip(enums, values))) 210 | return _enum_template.format(name=name, enum_def=enum_def) 211 | 212 | 213 | def _field2proto(name, type_, index, type_names, rjust=None): 214 | '''Returns a protobuf schema field str from a NamedTuple field''' 215 | string = None 216 | 217 | inspected = inspect_type(type_) 218 | 219 | if type_ in _type_lookup: 220 | string = "{} {} = {};".format(_type2proto(type_), name, index) 221 | 222 | elif _is_namedtuple(type_) or issubclass(inspected.origin, Enum): 223 | tn = type_.__name__ 224 | if tn not in type_names: 225 | raise AcumosError("Could not build protobuf field using unknown custom type {}".format(tn)) 226 | string = "{} {} = {};".format(tn, name, index) 227 | 228 | elif issubclass(inspected.origin, List): 229 | inner = inspected.args[0] 230 | if _is_container(inner): 231 | raise NestedTypeError("Nested container {} is not yet supported; try using NamedTuple instead".format(type_)) 232 | string = "repeated {}".format(_field2proto(name, inner, index, type_names, 0)) 233 | 234 | elif issubclass(inspected.origin, Dict): 235 | k, v = inspected.args 236 | if any(map(_is_container, (k, v))): 237 | raise NestedTypeError("Nested container {} is not yet supported; try using NamedTuple instead".format(type_)) 238 | string = "map<{}, {}> {} = {};".format(_type2proto(k), _type2proto(v), name, index) 239 | 240 | if string is None: 241 | raise AcumosError("Could not build protobuf field due to unsupported type {}".format(type_)) 242 | 243 | if rjust is None: 244 | rjust = len(string) + 2 245 | 246 | return string.rjust(rjust, ' ') 247 | 248 | 249 | def _is_container(t): 250 | return issubclass(t, Dict) or issubclass(t, List) 251 | 252 | 253 | def _type2proto(t): 254 | '''Returns a string corresponding to the protobuf type''' 255 | if t in _type_lookup: 256 | return _type_lookup[t] 257 | elif _is_namedtuple(t) or issubclass(t, Enum): 258 | return t.__name__ 259 | else: 260 | raise AcumosError("Unknown protobuf mapping for type {}".format(t)) 261 | 262 | 263 | def _gen_service(model, name='Model'): 264 | '''Returns a protobuf service definition string''' 265 | def _type_name(t: type): 266 | "we use empty as placeholders for raw types in protobuf" 267 | return t.__name__ if not is_raw_type(t) else Empty.__name__ 268 | 269 | rpc_comps = ((n, _type_name(f.input_type), _type_name(f.output_type)) 270 | for n, f in model.methods.items() 271 | if _is_namedtuple(f.input_type) or _is_namedtuple(f.output_type)) 272 | rpc_defs = '\n'.join(_gen_rpc(*comps) for comps in rpc_comps) 273 | return _service_template.format(name=name, service_def=rpc_defs) 274 | 275 | 276 | def _gen_rpc(name, in_, out): 277 | '''Returns a protobuf rpc definition string''' 278 | rpc = _rpc_template.format(name=name, msg_in=in_, msg_out=out) 279 | return rpc.rjust(len(rpc) + 2) 280 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/att.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acumos/acumos-python-client/d4faad0ed3fe6da0c8b0bfb23b548fa9ace546e5/acumos-package/acumos/tests/att.png -------------------------------------------------------------------------------- /acumos-package/acumos/tests/connexion_server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | ''' 20 | Provides mock server for unit testing 21 | ''' 22 | import json 23 | import argparse 24 | 25 | import connexion 26 | from flask import request 27 | 28 | 29 | TOKEN = 'secrettoken' 30 | USERS = {'foo': 'bar'} 31 | 32 | 33 | def upload(model, metadata, schema, license=None): 34 | '''Mock upload endpoint''' 35 | # test extra headers 36 | test_header = request.headers.get('X-Test-Header') # made up header to test extra_headers feature 37 | jwt = request.headers.get('Authorization') 38 | if not any(token == TOKEN for token in (jwt, test_header)): 39 | return {'status': 'Unauthorized'}, 401 40 | 41 | # test license file 42 | license_header = request.headers.get('X-Test-License') # made up header to test license content 43 | if license_header is not None: 44 | if not json.loads(license.read().decode())['license'] == license_header: 45 | return 'File license does not match test value', 400 46 | 47 | response = {'status': 'OK'} 48 | 49 | # test microservice creation parameter 50 | create_header = request.headers.get('X-Test-Create') 51 | if create_header is not None: 52 | is_create = request.headers['isCreateMicroservice'] 53 | if is_create != create_header: 54 | return "Header isCreateMicroservice ({}) does not match test value ({})".format(is_create, create_header), 400 55 | response["dockerImageUri"] = "uri/to/image:tag_or_hash" 56 | 57 | return response, 201 58 | 59 | 60 | def authenticate(auth_request): 61 | '''Mock authentication endpoint''' 62 | username = auth_request['request_body']['username'] 63 | password = auth_request['request_body']['password'] 64 | 65 | if USERS.get(username) == password: 66 | return {'jwtToken': TOKEN}, 200 67 | else: 68 | return {'jwtToken': None, 'resultCode': 401}, 401 69 | 70 | 71 | if __name__ == '__main__': 72 | '''Main''' 73 | parser = argparse.ArgumentParser() 74 | parser.add_argument("--port", type=int, default=8887) 75 | parser.add_argument("--https", action='store_true') 76 | pargs = parser.parse_args() 77 | 78 | app = connexion.App(__name__) 79 | app.add_api('swagger.yaml') 80 | 81 | if pargs.https: 82 | app.run(host='localhost', port=pargs.port, ssl_context='adhoc') 83 | else: 84 | app.run(host='localhost', port=pargs.port) 85 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/mock-license.json: -------------------------------------------------------------------------------- 1 | {"license": "Apache-2.0"} -------------------------------------------------------------------------------- /acumos-package/acumos/tests/mock_server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a mock web server 21 | """ 22 | import os 23 | import pexpect 24 | import socket 25 | from collections import namedtuple 26 | 27 | import requests 28 | from requests import ConnectionError 29 | 30 | from utils import TEST_DIR 31 | 32 | 33 | _EXPECT_RE = r'.*Running on (?Phttp://.*:\d+).*' 34 | 35 | 36 | def _find_port(): 37 | '''Returns an open port number''' 38 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 39 | sock.bind(('localhost', 0)) 40 | port = sock.getsockname()[1] 41 | sock.close() 42 | return port 43 | 44 | 45 | class _Config(namedtuple('_Config', ['model_url', 'auth_url', 'ui_url', 'port'])): 46 | 47 | def __new__(cls, port): 48 | base_url = "http://localhost:{}/v2".format(port) 49 | ui_url = "{}/ui".format(base_url) 50 | model_url = "{}/models".format(base_url) 51 | auth_url = "{}/auth".format(base_url) 52 | return super().__new__(cls, model_url, auth_url, ui_url, port) 53 | 54 | 55 | class MockServer(object): 56 | 57 | def __init__(self, timeout=5, use_localhost=True, server=None, extra_envs=None, port=None, https=False): 58 | '''Creates a test server running with a mock upload API in another process''' 59 | self._use_localhost = use_localhost 60 | self._timeout = timeout 61 | self._child = None 62 | self._https = https 63 | self.server = server 64 | 65 | port = _find_port() if port is None else port 66 | self.config = _Config(port) 67 | 68 | app_path = os.path.join(TEST_DIR, 'connexion_server.py') 69 | 70 | cmd = ['python', app_path, '--port', port] 71 | if https: 72 | cmd.append('--https') 73 | self._cmd = ' '.join(map(str, cmd)) 74 | 75 | def __enter__(self): 76 | '''Spawns the child process and waits for server to start until `timeout`''' 77 | assert not _server_running(self.config.ui_url), 'A mock server is already running' 78 | 79 | self._child = pexpect.spawn(self._cmd, env=os.environ) 80 | self._child.expect(_EXPECT_RE, timeout=self._timeout) 81 | server = self._child.match.groupdict()['server'].decode() 82 | if self._use_localhost: 83 | server = server.replace('127.0.0.1', 'localhost') 84 | self.server = server 85 | return self 86 | 87 | def __exit__(self, type, value, tb): 88 | '''Interrupts the server and cleans up''' 89 | if self._child is not None: 90 | self._child.sendintr() 91 | 92 | def api(self, route, route_prefix='', readline=True): 93 | '''Returns the full url with server prepended''' 94 | cmd = "{}{}{}".format(self.server, route_prefix, route) 95 | if self._child is not None and self._child.isalive() and readline: 96 | self._child.readline() # need to read lines so that the child buffer does not get filled 97 | return cmd 98 | 99 | 100 | def _server_running(ui_url): 101 | '''Returns False if test server is not available''' 102 | try: 103 | requests.get(ui_url) 104 | except ConnectionError: 105 | return False 106 | else: 107 | return True 108 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/model_loader_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | """ 21 | Loads a dumped model 22 | """ 23 | import sys 24 | 25 | from acumos.wrapped import load_model 26 | 27 | 28 | if __name__ == '__main__': 29 | '''Main''' 30 | del sys.path[0] # remove acumos/tests dir from python path 31 | 32 | model_dir = sys.argv[1] 33 | module_name = sys.argv[2] 34 | 35 | model = load_model(model_dir) 36 | 37 | custom_package = sys.modules[module_name] # `module_name` should be imported, else KeyError 38 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/py36_namedtuple.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides NamedTuple types defined using Python 3.6 syntax 21 | """ 22 | from typing import NamedTuple 23 | 24 | 25 | class Input(NamedTuple): 26 | x: int 27 | y: int 28 | 29 | 30 | class Output(NamedTuple): 31 | value: int 32 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/swagger.yaml: -------------------------------------------------------------------------------- 1 | swagger: "2.0" 2 | info: 3 | description: "On-boarding server API" 4 | version: "0.3.11" 5 | title: "On-boarding Server" 6 | basePath: "/v2" 7 | tags: 8 | - name: "Model" 9 | - name: "Auth" 10 | schemes: 11 | - "http" 12 | - "https" 13 | paths: 14 | /models: 15 | post: 16 | tags: 17 | - "Model" 18 | summary: "Upload a new model" 19 | operationId: "connexion_server.upload" 20 | consumes: 21 | - multipart/form-data 22 | produces: 23 | - application/json 24 | parameters: 25 | - in: header 26 | name: Authorization 27 | type: string 28 | required: true 29 | description: The JWT string received from /auth 30 | - in: formData 31 | name: model 32 | type: file 33 | required: true 34 | description: The serialized model archive 35 | - in: formData 36 | name: metadata 37 | type: file 38 | required: true 39 | description: The model metadata 40 | - in: formData 41 | name: schema 42 | type: file 43 | required: true 44 | description: The model schema definitions (e.g. proto file, etc.) 45 | - in: formData 46 | name: license 47 | type: file 48 | description: The model license 49 | required: false 50 | - in: header 51 | name: isCreateMicroservice 52 | type: boolean 53 | description: Creates microservice if true 54 | required: false 55 | - in: formData 56 | name: docker 57 | type: file 58 | description: The serialized docker file archive (e.g. a zip file) 59 | required: false 60 | - in: formData 61 | name: docs 62 | type: file 63 | description: A JSON based string for some basic documentation of the model 64 | required: false 65 | - in: formData 66 | name: thumbnail 67 | type: file 68 | description: An image file in PNG encoding with a max resolution of 256x256 69 | required: false 70 | responses: 71 | 201: 72 | description: "Model uploaded" 73 | /auth: 74 | post: 75 | tags: 76 | - "Auth" 77 | summary: "Authenticate" 78 | operationId: "connexion_server.authenticate" 79 | consumes: 80 | - application/json 81 | produces: 82 | - application/json 83 | parameters: 84 | - in: "body" 85 | name: "auth_request" 86 | description: "Authentication request" 87 | required: true 88 | schema: 89 | $ref: "#/definitions/AuthRequest" 90 | responses: 91 | 200: 92 | description: "Authentication successful" 93 | schema: 94 | $ref: "#/definitions/AuthResponse" 95 | 401: 96 | description: "Authentication failure" 97 | definitions: 98 | AuthRequest: 99 | type: "object" 100 | properties: 101 | request_body: 102 | type: "object" 103 | properties: 104 | username: 105 | type: "string" 106 | password: 107 | type: "string" 108 | AuthResponse: 109 | type: "object" 110 | properties: 111 | jwtToken: 112 | type: "string" -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides metadata tests 21 | """ 22 | import pytest 23 | 24 | from acumos.metadata import _gather_requirements, _filter_requirements, Requirements 25 | 26 | 27 | def test_requirements(): 28 | '''Tests usage of requirement utilities''' 29 | req = Requirements(packages=['~/foo', '~/bar/']) 30 | assert req.package_names == {'foo', 'bar'} 31 | 32 | reqs = Requirements(reqs=['foo', 'bar', 'baz'], req_map={'foo': 'bing'}, packages=['~/baz']) 33 | req_set = _filter_requirements(reqs) 34 | assert req_set == {'bing', 'bar'} 35 | 36 | reqs = Requirements(reqs=['PIL', 'scikit-learn', 'keras', 'baz', 'collections', 'time', 'sys'], 37 | req_map={'PIL': 'pillow'}, packages=['~/baz']) 38 | req_names, _ = zip(*_gather_requirements(reqs)) 39 | assert set(map(str.lower, req_names)) == {'pillow', 'scikit-learn', 'keras'} 40 | 41 | 42 | if __name__ == '__main__': 43 | '''Test area''' 44 | pytest.main([__file__, ]) 45 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_modeling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides modeling tests 21 | """ 22 | import pytest 23 | 24 | from acumos.modeling import (_wrap_function, 25 | AcumosError, 26 | NamedTuple, 27 | Model, NoReturn, 28 | Empty, 29 | List, 30 | Dict, 31 | new_type, 32 | is_raw_type,) 33 | from acumos.protogen import _types_equal 34 | 35 | 36 | @pytest.fixture() 37 | def FooIn() -> type: 38 | return NamedTuple('FooIn', [('x', int), ('y', int)]) 39 | 40 | 41 | @pytest.fixture() 42 | def FooOut() -> type: 43 | return NamedTuple('FooOut', [('value', int)]) 44 | 45 | 46 | @pytest.fixture() 47 | def Image() -> type: 48 | return new_type(bytes, 'Image', {'dcae_input_name': 'a', 'dcae_output_name': 'a'}, 'example description') 49 | 50 | 51 | def test_wrap_function_raw_to_raw(FooIn, FooOut, Image): 52 | '''Tests function wrapper utility 53 | check for both user defined raw data type''' 54 | 55 | def test_image_func(image: Image) -> Image: 56 | return Image(image) 57 | 58 | f, raw_in, raw_out = _wrap_function(test_image_func) 59 | 60 | assert is_raw_type(raw_in) 61 | assert is_raw_type(raw_out) 62 | assert f(Image(b"1234")) == Image(b"1234") 63 | 64 | 65 | def test_wrap_function_raw_to_structured(FooIn, FooOut, Image): 66 | '''Tests function wrapper utility 67 | check for user-defined to structured data''' 68 | 69 | def test_get_image_size_func(image: Image) -> int: 70 | return len(image) 71 | 72 | f, raw_in, raw_out = _wrap_function(test_get_image_size_func) 73 | 74 | assert is_raw_type(raw_in) 75 | assert _types_equal(raw_out, FooOut, ignore_type_name=True) 76 | assert f(Image(Image(b"1234"))) == raw_out(4) 77 | 78 | 79 | def test_wrap_function_structured_to_raw(FooIn, FooOut, Image): 80 | '''Tests function wrapper utility 81 | check for structured to user-defined data''' 82 | 83 | def test_create_image_func(x: int, y: int) -> Image: 84 | return Image("b\00" * x * y) 85 | 86 | f, raw_in, raw_out = _wrap_function(test_create_image_func) 87 | 88 | assert _types_equal(raw_in, FooIn, ignore_type_name=True) 89 | assert is_raw_type(raw_out) 90 | assert f(raw_in(2, 2)) == Image("b\00" * 4) 91 | 92 | 93 | def test_wrap_function_structured_to_structured(FooIn, FooOut, Image): 94 | '''Tests function wrapper utility 95 | both args and return need to be wrapped''' 96 | 97 | def foo(x: int, y: int) -> int: 98 | return x + y 99 | 100 | f, in_, out = _wrap_function(foo) 101 | 102 | assert _types_equal(in_, FooIn) 103 | assert _types_equal(out, FooOut) 104 | assert f(FooIn(1, 2)) == FooOut(3) 105 | 106 | 107 | def test_wrap_function_structured_to_structured_already_wrapped(FooIn, FooOut, Image): 108 | '''Tests function wrapper utility 109 | both args and return are already wrapped''' 110 | def bar(msg: FooIn) -> FooOut: 111 | return FooOut(msg.x + msg.y) 112 | 113 | f, in_, out = _wrap_function(bar) 114 | 115 | assert f is bar 116 | assert _types_equal(in_, FooIn) 117 | assert _types_equal(out, FooOut) 118 | assert f(FooIn(1, 2)) == FooOut(3) 119 | 120 | 121 | def test_wrap_function_structured_to_structured_args_already_wrapped(FooIn, FooOut, Image): 122 | '''Tests function wrapper utility 123 | args are already wrapped''' 124 | BazIn = NamedTuple('BazIn', [('x', int), ('y', int)]) 125 | 126 | def baz(x: int, y: int) -> FooOut: 127 | return FooOut(x + y) 128 | 129 | f, in_, out = _wrap_function(baz) 130 | 131 | assert _types_equal(in_, BazIn) 132 | assert _types_equal(out, FooOut) 133 | assert f(BazIn(1, 2)) == FooOut(3) 134 | 135 | 136 | def test_wrap_function_structured_to_structured_return_already_wrapped(FooIn, FooOut, Image): 137 | '''Tests function wrapper utility 138 | return is already wrapped''' 139 | QuxOut = NamedTuple('QuxOut', [('value', int)]) 140 | 141 | def qux(msg: FooIn) -> int: 142 | return msg.x + msg.y 143 | 144 | f, in_, out = _wrap_function(qux) 145 | 146 | assert _types_equal(in_, FooIn) 147 | assert _types_equal(out, QuxOut) 148 | assert f(FooIn(1, 2)) == QuxOut(3) 149 | 150 | 151 | def test_bad_annotations(): 152 | '''Tests bad annotation scenarios''' 153 | 154 | def f1(x: float): 155 | pass 156 | 157 | def f2(x): 158 | pass 159 | 160 | def f3(x) -> float: 161 | pass 162 | 163 | def f4(x: float, y) -> float: 164 | pass 165 | 166 | def f5(x: float) -> float: 167 | pass 168 | 169 | for f in (f1, f2, f3, f4): 170 | with pytest.raises(AcumosError): 171 | _wrap_function(f) 172 | 173 | _wrap_function(f5) 174 | 175 | 176 | def test_model(): 177 | '''Tests Model class''' 178 | 179 | def my_transform(x: int, y: int) -> int: 180 | '''Docstrings also work''' 181 | return x + y 182 | 183 | def another_transform(x: int, y: int) -> int: 184 | return x + y 185 | 186 | model = Model(transform=my_transform, another=another_transform) 187 | 188 | input_type = model.transform.input_type 189 | output_type = model.transform.output_type 190 | 191 | assert input_type.__name__ == 'TransformIn' 192 | assert output_type.__name__ == 'TransformOut' 193 | 194 | assert model.transform.inner(1, 1) == 2 195 | assert model.transform.wrapped(input_type(1, 1)) == output_type(2) 196 | 197 | assert model.transform.description == '''Docstrings also work''' 198 | assert model.another.description == '' 199 | 200 | 201 | def test_null_functions(): 202 | '''Tests the wrapping of a function with no arguments and no returns''' 203 | def f1() -> None: 204 | pass 205 | 206 | def f2() -> NoReturn: 207 | pass 208 | 209 | def f3() -> Empty: 210 | pass 211 | 212 | for f in (f1, f2, f3): 213 | _, in_, out = _wrap_function(f) 214 | assert in_ is Empty 215 | assert out is Empty 216 | 217 | 218 | def test_reserved_name(): 219 | '''Tests that a reserved NamedTuple name cannot be used''' 220 | Empty = NamedTuple('Empty', []) 221 | 222 | def foo(x: Empty) -> Empty: 223 | return Empty() 224 | 225 | with pytest.raises(AcumosError): 226 | _wrap_function(foo) 227 | 228 | 229 | def test_nested_defs(): 230 | '''Tests that nested types raise exceptions''' 231 | 232 | def f1(x: List[List[int]]) -> float: 233 | pass 234 | 235 | def f2(x: List[Dict[str, int]]) -> float: 236 | pass 237 | 238 | def f3(x: Dict[str, List[int]]) -> float: 239 | pass 240 | 241 | def f4(x: Dict[str, Dict[str, int]]) -> float: 242 | pass 243 | 244 | for f in (f1, f2, f3, f4): 245 | with pytest.raises(AcumosError): 246 | _wrap_function(f) 247 | 248 | 249 | if __name__ == '__main__': 250 | '''Test area''' 251 | pytest.main([__file__, ]) 252 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_pickler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Tests custom pickling logic 21 | """ 22 | import os 23 | import sys 24 | from tempfile import TemporaryDirectory 25 | from os.path import join as path_join 26 | 27 | import pytest 28 | import numpy as np 29 | import pandas as pd 30 | import tensorflow as tf 31 | from sklearn.datasets import load_iris 32 | from sklearn.svm import SVC 33 | from keras.models import Sequential 34 | from keras.layers import Dense 35 | from keras_contrib.layers import PELU 36 | 37 | from acumos.pickler import dump_model, load_model, AcumosContextManager, get_context 38 | from acumos.exc import AcumosError 39 | from acumos.modeling import Model 40 | 41 | from utils import run_command, TEST_DIR 42 | from user_module import user_function 43 | 44 | 45 | _UNPICKLER_HELPER = path_join(TEST_DIR, 'unpickler_helper.py') 46 | 47 | 48 | def test_user_script(): 49 | '''Tests that user scripts are identified as dependencies''' 50 | 51 | def predict(x: int) -> int: 52 | return user_function(x) 53 | 54 | model = Model(predict=predict) 55 | 56 | with AcumosContextManager() as context: 57 | model_path = context.build_path('model.pkl') 58 | with open(model_path, 'wb') as f: 59 | dump_model(model, f) 60 | 61 | assert 'user_module' in context.script_names 62 | 63 | # unpickling should fail because `user_module` is not available 64 | with pytest.raises(Exception, match="No module named 'user_module'"): 65 | run_command([sys.executable, _UNPICKLER_HELPER, context.abspath]) 66 | 67 | 68 | def test_keras_contrib(): 69 | '''Tests keras_contrib layer is saved correctly''' 70 | model = Sequential() 71 | model.add(Dense(10, input_shape=(10,))) 72 | model.add(PELU()) 73 | 74 | model.compile(loss='mse', optimizer='adam') 75 | model.fit(x=np.random.random((10, 10)), y=np.random.random((10, 10)), epochs=1, verbose=0) 76 | 77 | with AcumosContextManager() as context: 78 | model_path = context.build_path('model.pkl') 79 | with open(model_path, 'wb') as f: 80 | dump_model(model, f) 81 | assert {'keras', 'dill', 'acumos', 'h5py', 'tensorflow', 'keras_contrib'} == context.package_names 82 | 83 | # verify that the contrib layers don't cause a load error 84 | run_command([sys.executable, _UNPICKLER_HELPER, context.abspath]) 85 | 86 | 87 | def test_function_import(): 88 | '''Tests that a module used by a function is captured correctly''' 89 | import numpy as np 90 | 91 | def foo(): 92 | return np.arange(5) 93 | 94 | with AcumosContextManager() as context: 95 | model_path = context.build_path('model.pkl') 96 | with open(model_path, 'wb') as f: 97 | dump_model(foo, f) 98 | 99 | assert {'dill', 'acumos', 'numpy'} == context.package_names 100 | 101 | with open(model_path, 'rb') as f: 102 | loaded_model = load_model(f) 103 | 104 | assert (loaded_model() == np.arange(5)).all() 105 | 106 | 107 | def test_pickler_keras(): 108 | '''Tests keras dump / load functionality''' 109 | iris = load_iris() 110 | X = iris.data 111 | y_onehot = pd.get_dummies(iris.target).values 112 | 113 | # test both keras and tensorflow.keras packages 114 | keras_pkg_names = {'keras', 'dill', 'acumos', 'h5py', 'tensorflow'} 115 | tf_pkg_names = {'dill', 'acumos', 'h5py', 'tensorflow'} 116 | 117 | for seq_cls, dense_cls, pkg_names in ((Sequential, Dense, keras_pkg_names), 118 | (tf.keras.Sequential, tf.keras.layers.Dense, tf_pkg_names)): 119 | model = seq_cls() 120 | model.add(dense_cls(3, input_dim=4, activation='relu')) 121 | model.add(dense_cls(3, activation='softmax')) 122 | model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) 123 | model.fit(X, y_onehot, verbose=0) 124 | 125 | with TemporaryDirectory() as root: 126 | 127 | with AcumosContextManager(root) as context: 128 | model_path = context.build_path('model.pkl') 129 | with open(model_path, 'wb') as f: 130 | dump_model(model, f) 131 | 132 | assert pkg_names == context.package_names 133 | 134 | with AcumosContextManager(root) as context: 135 | with open(model_path, 'rb') as f: 136 | loaded_model = load_model(f) 137 | 138 | assert (model.predict_classes(X, verbose=0) == loaded_model.predict_classes(X, verbose=0)).all() 139 | 140 | 141 | def test_pickler_sklearn(): 142 | '''Tests sklearn dump / load functionality''' 143 | iris = load_iris() 144 | X = iris.data 145 | y = iris.target 146 | 147 | model = SVC() 148 | model.fit(X, y) 149 | 150 | with TemporaryDirectory() as root: 151 | 152 | with AcumosContextManager(root) as context: 153 | model_path = context.build_path('model.pkl') 154 | with open(model_path, 'wb') as f: 155 | dump_model(model, f) 156 | 157 | assert {'sklearn', 'dill', 'acumos', 'numpy'} == context.package_names 158 | 159 | with AcumosContextManager(root) as context: 160 | with open(model_path, 'rb') as f: 161 | loaded_model = load_model(f) 162 | 163 | assert (model.predict(X) == loaded_model.predict(X)).all() 164 | 165 | 166 | def test_pickler_tensorflow(): 167 | '''Tests tensorflow session and graph serialization''' 168 | tf.set_random_seed(0) 169 | 170 | iris = load_iris() 171 | data = iris.data 172 | target = iris.target 173 | target_onehot = pd.get_dummies(target).values.astype(float) 174 | 175 | with tf.Graph().as_default(): 176 | 177 | # test pickling a session with trained weights 178 | 179 | session = tf.Session() 180 | x, y, prediction = _build_tf_model(session, data, target_onehot) 181 | yhat = session.run([prediction], {x: data})[0] 182 | 183 | with TemporaryDirectory() as model_root: 184 | with AcumosContextManager(model_root) as context: 185 | model_path = context.build_path('model.pkl') 186 | with open(model_path, 'wb') as f: 187 | dump_model(session, f) 188 | 189 | assert {'acumos', 'dill', 'tensorflow'} == context.package_names 190 | 191 | with AcumosContextManager(model_root) as context: 192 | with open(model_path, 'rb') as f: 193 | loaded_session = load_model(f) 194 | 195 | loaded_graph = loaded_session.graph 196 | loaded_prediction = loaded_graph.get_tensor_by_name(prediction.name) 197 | loaded_x = loaded_graph.get_tensor_by_name(x.name) 198 | loaded_yhat = loaded_session.run([loaded_prediction], {loaded_x: data})[0] 199 | 200 | assert loaded_session is not session 201 | assert loaded_graph is not session.graph 202 | assert (yhat == loaded_yhat).all() 203 | 204 | # tests pickling a session with a frozen graph 205 | 206 | with TemporaryDirectory() as frozen_root: 207 | save_path = path_join(frozen_root, 'model') 208 | 209 | with loaded_session.graph.as_default(): 210 | saver = tf.train.Saver() 211 | saver.save(loaded_session, save_path) 212 | 213 | frozen_path = _freeze_graph(frozen_root, ['prediction']) 214 | frozen_graph = _unfreeze_graph(frozen_path) 215 | frozen_session = tf.Session(graph=frozen_graph) 216 | 217 | with TemporaryDirectory() as model_root: 218 | with AcumosContextManager(model_root) as context: 219 | model_path = context.build_path('model.pkl') 220 | with open(model_path, 'wb') as f: 221 | dump_model(frozen_session, f) 222 | 223 | with AcumosContextManager(model_root) as context: 224 | with open(model_path, 'rb') as f: 225 | loaded_frozen_session = load_model(f) 226 | 227 | loaded_frozen_graph = loaded_frozen_session.graph 228 | loaded_frozen_prediction = loaded_frozen_graph.get_tensor_by_name(prediction.name) 229 | loaded_frozen_x = loaded_frozen_graph.get_tensor_by_name(x.name) 230 | loaded_frozen_yhat = loaded_frozen_session.run([loaded_frozen_prediction], {loaded_frozen_x: data})[0] 231 | 232 | assert loaded_frozen_session is not frozen_session 233 | assert loaded_frozen_graph is not frozen_session.graph 234 | assert (yhat == loaded_frozen_yhat).all() 235 | 236 | 237 | def _build_tf_model(session, data, target): 238 | '''Builds and iris tensorflow model and returns the prediction tensor''' 239 | x = tf.placeholder(shape=[None, 4], dtype=tf.float32) 240 | y = tf.placeholder(shape=[None, 3], dtype=tf.float32) 241 | 242 | layer1 = tf.layers.dense(x, 3, activation=tf.nn.relu) 243 | logits = tf.layers.dense(layer1, 3) 244 | 245 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y)) 246 | optimizer = tf.train.GradientDescentOptimizer(0.075).minimize(cost) 247 | 248 | init = tf.global_variables_initializer() 249 | session.run(init) 250 | 251 | for epoch in range(3): 252 | _, loss = session.run([optimizer, cost], feed_dict={x: data, y: target}) 253 | 254 | prediction = tf.argmax(logits, 1, name='prediction') 255 | return x, y, prediction 256 | 257 | 258 | def _freeze_graph(model_dir, output_node_names): 259 | '''Modified from https://blog.metaflow.fr/tensorflow-how-to-freeze-a-model-and-serve-it-with-a-python-api-d4f3596b3adc''' 260 | input_checkpoint = tf.train.get_checkpoint_state(model_dir).model_checkpoint_path 261 | graph_path = "{}.meta".format(input_checkpoint) 262 | output_graph = path_join(model_dir, 'frozen_model.pb') 263 | 264 | with tf.Session(graph=tf.Graph()) as session: 265 | saver = tf.train.import_meta_graph(graph_path, clear_devices=True) 266 | saver.restore(session, input_checkpoint) 267 | output_graph_def = tf.graph_util.convert_variables_to_constants(session, 268 | session.graph.as_graph_def(), 269 | output_node_names) 270 | with tf.gfile.GFile(output_graph, 'wb') as f: 271 | f.write(output_graph_def.SerializeToString()) 272 | return output_graph 273 | 274 | 275 | def _unfreeze_graph(frozen_graph_path): 276 | '''Modified from https://blog.metaflow.fr/tensorflow-how-to-freeze-a-model-and-serve-it-with-a-python-api-d4f3596b3adc''' 277 | with tf.gfile.GFile(frozen_graph_path, 'rb') as f: 278 | graph_def = tf.GraphDef() 279 | graph_def.ParseFromString(f.read()) 280 | 281 | with tf.Graph().as_default() as graph: 282 | tf.import_graph_def(graph_def, name='') 283 | return graph 284 | 285 | 286 | def test_nested_model(): 287 | '''Tests nested models''' 288 | iris = load_iris() 289 | X = iris.data 290 | y = iris.target 291 | y_onehot = pd.get_dummies(iris.target).values 292 | 293 | m1 = Sequential() 294 | m1.add(Dense(3, input_dim=4, activation='relu')) 295 | m1.add(Dense(3, activation='softmax')) 296 | m1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) 297 | m1.fit(X, y_onehot, verbose=0) 298 | 299 | m2 = SVC() 300 | m2.fit(X, y) 301 | 302 | # lambda watch out 303 | crazy_good_model = lambda x: m1.predict_classes(x) + m2.predict(x) # noqa 304 | out1 = crazy_good_model(X) 305 | 306 | with TemporaryDirectory() as root: 307 | 308 | with AcumosContextManager(root) as context: 309 | model_path = context.build_path('model.pkl') 310 | with open(model_path, 'wb') as f: 311 | dump_model(crazy_good_model, f) 312 | 313 | assert {'sklearn', 'keras', 'dill', 'acumos', 'numpy', 'h5py', 'tensorflow'} == context.package_names 314 | 315 | with AcumosContextManager(root) as context: 316 | with open(model_path, 'rb') as f: 317 | loaded_model = load_model(f) 318 | 319 | out2 = loaded_model(X) 320 | assert (out1 == out2).all() 321 | 322 | 323 | def test_context(): 324 | '''Tests basic AcumosContextManager functionality''' 325 | with AcumosContextManager() as c1: 326 | c2 = get_context() 327 | assert c1 is c2 328 | assert {'dill', 'acumos'} == c1.package_names 329 | 330 | # default context already exists 331 | with pytest.raises(AcumosError): 332 | with AcumosContextManager(): 333 | pass 334 | 335 | assert os.path.isdir(c1.abspath) 336 | 337 | abspath = c1.create_subdir() 338 | assert os.path.isdir(abspath) 339 | 340 | # context removes a temporary directory it creates 341 | assert not os.path.isdir(c1.abspath) 342 | 343 | # default context doesn't exist outside of CM 344 | with pytest.raises(AcumosError): 345 | get_context() 346 | 347 | 348 | def test_context_provided_root(): 349 | '''Tests AcumosContextManager with a provided root directory''' 350 | with TemporaryDirectory() as root: 351 | with AcumosContextManager(root) as c1: 352 | abspath = c1.create_subdir() 353 | assert os.path.isdir(abspath) 354 | 355 | # context does not remove a provided directory 356 | assert os.path.isdir(c1.abspath) 357 | assert os.path.isdir(abspath) 358 | 359 | 360 | if __name__ == '__main__': 361 | '''Test area''' 362 | pytest.main([__file__, ]) 363 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_protogen.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides modeling tests 21 | """ 22 | import tempfile 23 | 24 | import pytest 25 | import numpy as np 26 | import pandas as pd 27 | 28 | from acumos.modeling import Model, Enum, List, NamedTuple, create_dataframe 29 | from acumos.protogen import _types_equal, _require_unique, _nt2proto, model2proto, compile_protostr 30 | from acumos.exc import AcumosError 31 | 32 | 33 | def test_type_equality(): 34 | '''Tests that type equality function works as expected''' 35 | t1 = NamedTuple('T1', [('x', int), ('y', int)]) 36 | t2 = NamedTuple('T1', [('x', int), ('y', float)]) 37 | t3 = NamedTuple('T2', [('x', int), ('y', t1)]) 38 | t4 = NamedTuple('T2', [('x', int), ('y', t2)]) 39 | t5 = NamedTuple('T3', [('x', int), ('y', t1)]) 40 | t6 = NamedTuple('T2', [('x', int), ('y', t1)]) 41 | t7 = NamedTuple('T2', [('x', int), ('z', t1)]) 42 | 43 | assert not _types_equal(t1, t2) # type differs 44 | assert not _types_equal(t3, t4) # type differs 45 | assert not _types_equal(t3, t5) # name differs 46 | assert not _types_equal(t3, t7) # field differs 47 | assert _types_equal(t3, t6) 48 | 49 | 50 | def test_require_unique(): 51 | '''Tests that unique types are tested for''' 52 | t1 = NamedTuple('T1', [('x', int), ('y', int)]) 53 | t2 = NamedTuple('T1', [('x', int), ('y', float)]) 54 | t3 = NamedTuple('T1', [('x', int), ('y', int)]) 55 | 56 | with pytest.raises(AcumosError): 57 | _require_unique((t1, t2, t3)) # t2 is a different definition of T1 58 | 59 | uniq = _require_unique((t1, t3)) 60 | assert len(uniq) == 1 61 | assert t1 in uniq or t3 in uniq 62 | 63 | 64 | def test_nt2proto(): 65 | '''Tests the generation of protobuf messages from NamedTuple''' 66 | Foo = NamedTuple('Foo', [('x', int), ('y', int)]) 67 | Bar = NamedTuple('Bar', [('x', Foo)]) 68 | 69 | _nt2proto(Foo, set()) 70 | 71 | # dependence on Foo which has not been declared 72 | with pytest.raises(AcumosError): 73 | _nt2proto(Bar, set()) 74 | 75 | _nt2proto(Bar, {Foo.__name__, }) 76 | 77 | 78 | def test_model2proto(): 79 | '''Tests the generation of protobuf messages from a Model''' 80 | T1 = NamedTuple('T1', [('x', int), ('y', int)]) 81 | T2 = NamedTuple('T2', [('data', int)]) 82 | 83 | Thing = Enum('Thing', 'a b c d e') 84 | 85 | def f1(x: int, y: int) -> int: 86 | return x + y 87 | 88 | def f2(data: T1) -> T2: 89 | return T2(data.x + data.y) 90 | 91 | def f3(data: List[Thing]) -> Thing: 92 | return data[0] 93 | 94 | def f4(data: List[T1]) -> None: 95 | pass 96 | 97 | def f5(x: List[np.int32]) -> np.int32: 98 | return np.sum(x) 99 | 100 | df = pd.DataFrame({'x': [1, 2, 3], 'y': [4, 5, 6]}) 101 | TestDataFrame = create_dataframe('TestDataFrame', df) 102 | 103 | def f6(in_: TestDataFrame) -> None: 104 | pass 105 | 106 | model = Model(f1=f1, f2=f2, f3=f3, f4=f4, f5=f5, f6=f6) 107 | module = 'model' 108 | package = 'pkg' 109 | protostr = model2proto(model, package) 110 | 111 | # main test is to make sure that compilation doesn't fail 112 | with tempfile.TemporaryDirectory() as tdir: 113 | compile_protostr(protostr, package, module, tdir) 114 | 115 | 116 | if __name__ == '__main__': 117 | '''Test area''' 118 | pytest.main([__file__, ]) 119 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_session.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides session tests 21 | """ 22 | import contextlib 23 | import logging 24 | from pathlib import Path 25 | from typing import Optional 26 | 27 | import mock 28 | import tempfile 29 | import json 30 | import sys 31 | from os import environ, listdir 32 | from os.path import isfile, join as path_join 33 | 34 | import pytest 35 | import numpy as np 36 | import pandas as pd 37 | from sklearn.datasets import load_iris 38 | from sklearn.ensemble import RandomForestClassifier 39 | from keras.models import Sequential 40 | from keras.layers import Dense 41 | from jsonschema import validate 42 | 43 | from acumos.wrapped import _infer_model_dir 44 | from acumos.modeling import Model, List, create_namedtuple, create_dataframe 45 | from acumos.session import AcumosSession, _dump_model 46 | from acumos.exc import AcumosError 47 | from acumos.utils import load_artifact 48 | from acumos.auth import clear_jwt, _USERNAME_VAR, _PASSWORD_VAR, _TOKEN_VAR, _set_jwt 49 | from acumos.metadata import SCHEMA_VERSION, Requirements, Options 50 | 51 | from mock_server import MockServer 52 | from utils import run_command, TEST_DIR 53 | from user_package import user_package_module 54 | from user_module import user_function 55 | 56 | 57 | _USER_PACKAGE_DIR = path_join(TEST_DIR, 'user_package') 58 | _MODEL_LOADER_HELPER = path_join(TEST_DIR, 'model_loader_helper.py') 59 | _REQ_FILES = ('model.zip', 'model.proto', 'metadata.json', 'metadata_clio.json') 60 | _FAKE_USERNAME = 'foo' 61 | _FAKE_PASSWORD = 'bar' 62 | _FAKE_TOKEN = 'secrettoken' 63 | _FAKE_LICENSE = path_join(TEST_DIR, 'mock-license.json') 64 | 65 | 66 | @pytest.mark.parametrize('create_microservice', [True, False]) 67 | def test_create_microservice(create_microservice, caplog): 68 | '''Tests that the microservice creation parameter is correctly specified''' 69 | with _patch_auth(): 70 | extra_headers = {'X-Test-Create': str(create_microservice).lower()} 71 | opts = Options(create_microservice=create_microservice) 72 | maybe_docker_image_uri = _push_dummy_model(extra_headers, options=opts) 73 | 74 | if create_microservice: 75 | assert "Acumos model docker image successfully created" in caplog.text, "docker image Uri was not displayed" 76 | assert maybe_docker_image_uri == "uri/to/image:tag_or_hash" 77 | 78 | 79 | def test_license(): 80 | '''Tests that a user-provided license is correctly pushed''' 81 | with _patch_auth(): 82 | license_str = load_artifact(_FAKE_LICENSE, module=json, mode='r')['license'] 83 | extra_headers = {'X-Test-License': license_str} 84 | 85 | opts = Options(license=_FAKE_LICENSE) 86 | _push_dummy_model(extra_headers, options=opts) 87 | 88 | opts = Options(license=license_str) 89 | _push_dummy_model(extra_headers, options=opts) 90 | 91 | 92 | def test_session_push(): 93 | '''Tests various session push scenarios''' 94 | # allow users to push using username and password env vars 95 | clear_jwt() 96 | with _patch_environ(**{_USERNAME_VAR: _FAKE_USERNAME, _PASSWORD_VAR: _FAKE_PASSWORD}): 97 | _push_dummy_model(use_auth_url=True) 98 | 99 | # verify auth url is required for username and password auth 100 | clear_jwt() 101 | with _patch_environ(**{_USERNAME_VAR: _FAKE_USERNAME, _PASSWORD_VAR: _FAKE_PASSWORD}): 102 | with pytest.raises(AcumosError): 103 | _push_dummy_model(use_auth_url=False) 104 | 105 | # allow users to push using a token env var 106 | clear_jwt() 107 | with _patch_environ(**{_TOKEN_VAR: _FAKE_TOKEN}): 108 | _push_dummy_model() 109 | 110 | # allow users to push by providing a token via interactive prompt 111 | clear_jwt() 112 | with _patch_auth(): 113 | _push_dummy_model() 114 | 115 | # verify we can recover from a stale token 116 | _set_jwt('invalidtoken') 117 | with _patch_auth(): 118 | _push_dummy_model() 119 | 120 | 121 | def test_extra_header(): 122 | '''Tests that extra headers are correctly sent to the onboarding server''' 123 | clear_jwt() 124 | # in the mock onboarding server, this extra test header acts as another auth header 125 | with _patch_environ(**{_TOKEN_VAR: 'wrongtoken'}): 126 | extra_headers = {'X-Test-Header': _FAKE_TOKEN} 127 | _push_dummy_model(extra_headers) 128 | 129 | clear_jwt() 130 | with pytest.raises(AcumosError): 131 | with _patch_environ(**{_TOKEN_VAR: 'wrongtoken'}): 132 | extra_headers = {'X-Test-Header': 'wrongtoken'} 133 | _push_dummy_model(extra_headers) 134 | 135 | 136 | def _push_dummy_model(extra_headers=None, use_model_url=True, use_auth_url=False, options=None) -> Optional[str]: 137 | '''Generic dummy model push routine''' 138 | 139 | def my_transform(x: int, y: int) -> int: 140 | return x + y 141 | 142 | model = Model(transform=my_transform) 143 | 144 | with MockServer() as server: 145 | _model_url, _auth_url, _, _ = server.config 146 | model_url = _model_url if use_model_url else None 147 | auth_url = _auth_url if use_auth_url else None 148 | 149 | session = AcumosSession(model_url, auth_url) 150 | return session.push(model, name='my-model', extra_headers=extra_headers, options=options) 151 | 152 | 153 | def test_custom_package(): 154 | '''Tests that custom packages can be included, wrapped, and loaded''' 155 | 156 | def my_transform(x: int, y: int) -> int: 157 | return user_package_module.add_numbers(x, y) 158 | 159 | model = Model(transform=my_transform) 160 | model_name = 'my-model' 161 | 162 | # load should fail without requirements 163 | with pytest.raises(Exception, match='Module user_package was detected as a dependency'): 164 | with _dump_model(model, model_name) as dump_dir: 165 | pass 166 | 167 | reqs = Requirements(packages=[_USER_PACKAGE_DIR]) 168 | 169 | with _dump_model(model, model_name, reqs) as dump_dir: 170 | run_command([sys.executable, _MODEL_LOADER_HELPER, dump_dir, 'user_package']) 171 | 172 | 173 | def test_custom_script(): 174 | '''Tests that custom modules can be included, wrapped, and loaded''' 175 | 176 | def predict(x: int) -> int: 177 | return user_function(x) 178 | 179 | model = Model(predict=predict) 180 | model_name = 'my-model' 181 | 182 | with _dump_model(model, model_name) as dump_dir: 183 | run_command([sys.executable, _MODEL_LOADER_HELPER, dump_dir, 'user_module']) 184 | 185 | 186 | def test_script_req(): 187 | '''Tests that Python scripts can be included using Requirements''' 188 | 189 | def predict(x: int) -> int: 190 | return x 191 | 192 | model = Model(predict=predict) 193 | model_name = 'my-model' 194 | 195 | # tests that individual script and directory of scripts are both gathered 196 | reqs = Requirements(scripts=_abspath('./user_module.py', './user_package')) 197 | 198 | with _dump_model(model, model_name, reqs) as dump_dir: 199 | _verify_files(dump_dir, ('scripts/user_provided/user_package_module.py', 200 | 'scripts/user_provided/__init__.py', 201 | 'scripts/user_provided/user_module.py')) 202 | 203 | bad_reqs = Requirements(scripts=_abspath('./user_module.py', './user_package', 'not_real.py')) 204 | 205 | with pytest.raises(AcumosError, match='does not exist'): 206 | with _dump_model(model, model_name, bad_reqs) as dump_dir: 207 | pass 208 | 209 | bad_reqs = Requirements(scripts=_abspath('./user_module.py', './user_package', './att.png')) 210 | 211 | with pytest.raises(AcumosError, match='is invalid'): 212 | with _dump_model(model, model_name, bad_reqs) as dump_dir: 213 | pass 214 | 215 | 216 | def _abspath(*files): 217 | '''Returns the absolute path of a local test files''' 218 | return tuple(path_join(TEST_DIR, file) for file in files) 219 | 220 | 221 | @pytest.mark.parametrize(["replace"], argvalues=[ 222 | pytest.param(True, id="replace"), 223 | pytest.param(False, id="noreplace"), 224 | ]) 225 | def test_session_dump(replace: bool): 226 | '''Tests session dump''' 227 | 228 | def my_transform(x: int, y: int) -> int: 229 | return x + y 230 | 231 | model = Model(transform=my_transform) 232 | model_name = 'my-model' 233 | 234 | session = AcumosSession() 235 | 236 | with tempfile.TemporaryDirectory() as tdir: 237 | 238 | session.dump(model, model_name, tdir) 239 | model_dir = path_join(tdir, model_name) 240 | assert set(listdir(model_dir)) == set(_REQ_FILES) 241 | if replace is False: 242 | with pytest.raises(AcumosError): 243 | session.dump(model, model_name, tdir) # file already exists 244 | else: 245 | session.dump(model, model_name, tdir, replace=replace) # file already exists but it will be replaced 246 | 247 | 248 | @pytest.mark.parametrize(["replace"], argvalues=[ 249 | pytest.param(True, id="replace"), 250 | pytest.param(False, id="noreplace"), 251 | ]) 252 | def test_session_dump_zip(replace: bool): 253 | '''Tests session dump zip''' 254 | 255 | def my_transform(x: int, y: int) -> int: 256 | return x + y 257 | 258 | model = Model(transform=my_transform) 259 | model_name = 'my-model' 260 | 261 | session = AcumosSession() 262 | 263 | with tempfile.TemporaryDirectory() as tdir: 264 | model_zip_path = Path(tdir) / f"{model_name}.zip" 265 | 266 | session.dump_zip(model, model_name, model_zip_path) 267 | import zipfile 268 | with zipfile.ZipFile(model_zip_path, "r") as model_zip: 269 | assert set(model_zip.namelist()) == set(_REQ_FILES) 270 | 271 | if replace is False: 272 | with pytest.raises(AcumosError): 273 | session.dump_zip(model, model_name, model_zip_path) # file already exists 274 | else: 275 | session.dump_zip(model, model_name, model_zip_path, replace=replace) # file already exists but it will be replaced 276 | 277 | 278 | def test_dump_model(): 279 | '''Tests dump model utility, including generated artifacts''' 280 | 281 | def predict(x: int) -> int: 282 | return user_function(x) 283 | 284 | model = Model(predict=predict) 285 | model_name = 'my-model' 286 | 287 | reqs = Requirements(reqs=['wronglib'], req_map={'wronglib': 'scipy'}, packages=[_USER_PACKAGE_DIR]) 288 | 289 | with _dump_model(model, model_name, reqs) as dump_dir: 290 | 291 | assert set(listdir(dump_dir)) == set(_REQ_FILES) 292 | 293 | metadata = load_artifact(dump_dir, 'metadata.json', module=json, mode='r') 294 | schema = _load_schema(SCHEMA_VERSION) 295 | validate(metadata, schema) 296 | 297 | # test that a user-provided library was included and correctly mapped 298 | assert 'scipy' in {r['name'] for r in metadata['runtime']['dependencies']['pip']['requirements']} 299 | 300 | # test that custom package was bundled 301 | _verify_files(dump_dir, ('scripts/user_provided/user_package/user_package_module.py', 302 | 'scripts/user_provided/user_package/__init__.py', 303 | 'scripts/user_provided/user_module.py')) 304 | 305 | 306 | def _verify_files(dump_dir, files): 307 | '''Asserts that `files` exist in `dump_dir`''' 308 | model_dir = _infer_model_dir(dump_dir) 309 | for file in files: 310 | assert isfile(path_join(model_dir, file)) 311 | 312 | 313 | def _load_schema(version): 314 | '''Returns a jsonschema dict from the model-schema submodule''' 315 | path = path_join(TEST_DIR, 'schemas', "schema-{}.json".format(version)) 316 | with open(path) as f: 317 | schema = json.load(f) 318 | return schema 319 | 320 | 321 | def test_session_push_sklearn(): 322 | '''Tests basic model pushing functionality with sklearn''' 323 | with _patch_auth(): 324 | with MockServer() as server: 325 | iris = load_iris() 326 | X = iris.data 327 | y = iris.target 328 | 329 | clf = RandomForestClassifier(random_state=0) 330 | clf.fit(X, y) 331 | 332 | columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] 333 | X_df = pd.DataFrame(X, columns=columns) 334 | 335 | DataFrame = create_dataframe('DataFrame', X_df) 336 | Predictions = create_namedtuple('Predictions', [('predictions', List[int])]) 337 | 338 | def predict(df: DataFrame) -> Predictions: 339 | '''Predicts the class of iris''' 340 | X = np.column_stack(df) 341 | yhat = clf.predict(X) 342 | preds = Predictions(predictions=yhat) 343 | return preds 344 | 345 | model = Model(predict=predict) 346 | 347 | model_url, auth_url, _, _ = server.config 348 | session = AcumosSession(model_url, auth_url) 349 | session.push(model, name='sklearn_iris_push') 350 | 351 | 352 | def test_session_push_keras(): 353 | '''Tests basic model pushing functionality with keras''' 354 | with _patch_auth(): 355 | with MockServer() as server: 356 | iris = load_iris() 357 | X = iris.data 358 | y = pd.get_dummies(iris.target).values 359 | 360 | clf = Sequential() 361 | clf.add(Dense(3, input_dim=4, activation='relu')) 362 | clf.add(Dense(3, activation='softmax')) 363 | clf.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) 364 | clf.fit(X, y) 365 | 366 | columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] 367 | X_df = pd.DataFrame(X, columns=columns) 368 | 369 | DataFrame = create_dataframe('DataFrame', X_df) 370 | Predictions = create_namedtuple('Predictions', [('predictions', List[int])]) 371 | 372 | def predict(df: DataFrame) -> Predictions: 373 | '''Predicts the class of iris''' 374 | X = np.column_stack(df) 375 | yhat = clf.predict(X) 376 | preds = Predictions(predictions=yhat) 377 | return preds 378 | 379 | model = Model(predict=predict) 380 | 381 | model_url, auth_url, _, _ = server.config 382 | session = AcumosSession(model_url, auth_url) 383 | session.push(model, name='keras_iris_push') 384 | 385 | 386 | @contextlib.contextmanager 387 | def _patch_auth(clear_token=True): 388 | '''Convenience CM to patch interactive prompts used for authentication''' 389 | if clear_token: 390 | clear_jwt() 391 | 392 | with mock.patch('acumos.auth.gettoken', lambda x: _FAKE_TOKEN): 393 | yield 394 | 395 | 396 | @contextlib.contextmanager 397 | def _patch_environ(**kwargs): 398 | '''Temporarily adds kwargs to os.environ''' 399 | try: 400 | orig_vars = {k: environ[k] for k in kwargs.keys() if k in environ} 401 | environ.update(kwargs) 402 | yield 403 | finally: 404 | environ.update(orig_vars) 405 | for extra_key in (kwargs.keys() - orig_vars.keys()): 406 | del environ[extra_key] 407 | 408 | 409 | @contextlib.contextmanager 410 | def _acumos_logs_propagated(): 411 | '''Configures Acumos logger to propagate logs entry so that pytest's caplog can read them''' 412 | acumos_root_logger = logging.getLogger("acumos") 413 | old_propagate_value = acumos_root_logger.propagate 414 | acumos_root_logger.propagate = True 415 | try: 416 | yield 417 | finally: 418 | acumos_root_logger.propagate = old_propagate_value 419 | 420 | 421 | @pytest.fixture 422 | def caplog(caplog): 423 | '''Overrides pytest's caplog fixture with acumos logs propagated''' 424 | with _acumos_logs_propagated(): 425 | yield caplog 426 | 427 | 428 | if __name__ == '__main__': 429 | '''Test area''' 430 | pytest.main([__file__, ]) 431 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/test_wrapped.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides wrapped model tests 21 | """ 22 | import io 23 | import sys 24 | import json 25 | import logging 26 | from os.path import join as path_join 27 | from collections import Counter 28 | from operator import eq 29 | from tempfile import TemporaryDirectory 30 | 31 | import pytest 32 | import PIL 33 | import pandas as pd 34 | import numpy as np 35 | import tensorflow as tf 36 | from sklearn.datasets import load_iris 37 | from sklearn.ensemble import RandomForestClassifier 38 | from google.protobuf.json_format import MessageToJson, MessageToDict 39 | 40 | from acumos.wrapped import _unpack_pb_msg, load_model, _pack_pb_msg 41 | from acumos.modeling import Model, create_dataframe, List, Dict, create_namedtuple, new_type 42 | from acumos.session import _dump_model, _copy_dir, Requirements 43 | 44 | from test_pickler import _build_tf_model 45 | from utils import TEST_DIR 46 | 47 | 48 | _IMG_PATH = path_join(TEST_DIR, 'att.png') 49 | 50 | logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 51 | logger = logging.getLogger(__name__) 52 | 53 | 54 | @pytest.mark.skipif(sys.version_info < (3, 6), reason='Requires python3.6') 55 | def test_py36_namedtuple(): 56 | '''Tests to make sure that new syntax for NamedTuple works with wrapping''' 57 | from py36_namedtuple import Input, Output 58 | 59 | def adder(data: Input) -> Output: 60 | return Output(data.x + data.y) 61 | 62 | _generic_test(adder, Input(1, 2), Output(3)) 63 | 64 | 65 | @pytest.mark.flaky(reruns=5) 66 | def test_wrapped_prim_type(): 67 | '''Tests model wrap and load functionality''' 68 | 69 | def f1(x: int, y: int) -> int: 70 | return x + y 71 | 72 | def f2(x: int, y: int) -> None: 73 | pass 74 | 75 | def f3() -> None: 76 | pass 77 | 78 | def f4() -> int: 79 | return 3330 80 | 81 | def f5(data: bytes) -> str: 82 | '''Something more complex''' 83 | buffer = io.BytesIO(data) 84 | img = PIL.Image.open(buffer) 85 | return img.format 86 | 87 | def f6(x: List[int]) -> int: 88 | return sum(x) 89 | 90 | def f7(x: List[str]) -> Dict[str, int]: 91 | return Counter(x) 92 | 93 | def f8(x: List[np.int32]) -> np.int32: 94 | return np.sum(x) 95 | 96 | # input / output "answers" 97 | f1_in = (1, 2) 98 | f1_out = (3, ) 99 | 100 | f2_in = (1, 2) 101 | f2_out = () 102 | 103 | f3_in = () 104 | f3_out = () 105 | 106 | f4_in = (0, ) 107 | f4_out = (3330, ) 108 | 109 | with open(_IMG_PATH, 'rb') as f: 110 | f5_in = (f.read(), ) 111 | f5_out = ('PNG', ) 112 | 113 | f6_in = ([1, 2, 3], ) 114 | f6_out = (6, ) 115 | 116 | f7_in = (['a', 'a', 'b'], ) 117 | f7_out = ({'a': 2, 'b': 1}, ) 118 | 119 | f8_in = ([1, 2, 3], ) 120 | f8_out = (6, ) 121 | 122 | for func, in_, out in ((f1, f1_in, f1_out), (f2, f2_in, f2_out), (f3, f3_in, f3_out), 123 | (f4, f4_in, f4_out), (f6, f6_in, f6_out), (f8, f8_in, f8_out)): 124 | _generic_test(func, in_, out) 125 | 126 | _generic_test(f5, f5_in, f5_out, reqs=Requirements(req_map={'PIL': 'pillow'})) 127 | _generic_test(f7, f7_in, f7_out, skip=_dict_skips) 128 | 129 | 130 | @pytest.mark.flaky(reruns=5) 131 | def test_wrapped_nested_type(): 132 | '''Tests to make sure that nested NamedTuple messages are unpacked correctly''' 133 | Inner = create_namedtuple('Inner', [('x', int), ('y', int), ('z', int)]) 134 | 135 | N1 = create_namedtuple('N1', [('dict_data', Dict[str, int])]) 136 | N2 = create_namedtuple('N2', [('n1s', List[N1])]) 137 | 138 | def f1(x: List[Inner]) -> Inner: 139 | '''Returns the component-wise sum of a sequence of Inner''' 140 | sums = np.vstack(x).sum(axis=0) 141 | return Inner(*sums) 142 | 143 | def f2(n2_in: N2) -> N2: 144 | '''Returns another N2 type using data from the input N2 type''' 145 | n1_in = n2_in.n1s[0] 146 | dict_data = dict(**n1_in.dict_data) # shallow copy 147 | dict_data['b'] = 2 148 | n1_out = N1(dict_data=dict_data) 149 | n2_out = N2(n1s=[n1_out, n1_out]) 150 | return n2_out 151 | 152 | f1_in = ([Inner(1, 2, 3), ] * 5, ) 153 | f1_out = (5, 10, 15) 154 | 155 | n1 = N1(dict_data={'a': 1}) 156 | n1_out = N1(dict_data={'a': 1, 'b': 2}) 157 | f2_in = N2(n1s=[n1]) 158 | f2_out = N2(n1s=[n1_out, n1_out]) 159 | 160 | _generic_test(f1, f1_in, f1_out) 161 | _generic_test(f2, f2_in, f2_out, skip=_dict_skips) 162 | 163 | 164 | def _dict_skips(as_, from_): 165 | '''Skips byte and json str output comparison due to odd failures, perhaps related to dict ordering''' 166 | return as_ in {'as_pb_bytes', 'as_json'} 167 | 168 | 169 | Text = new_type(str, 'Text', {'dcae_input_name': 'a', 'dcae_output_name': 'a'}, 'example description') 170 | Image = new_type(bytes, 'Image', {'dcae_input_name': 'a', 'dcae_output_name': 'a'}, 'example description') 171 | Dictionary = new_type(dict, 'Dictionary', {'dcae_input_name': 'a', 'dcae_output_name': 'a'}, 'example description') 172 | 173 | 174 | def f1(text: Text) -> Text: 175 | '''Return a raw text''' 176 | return Text(text) 177 | 178 | 179 | def f2(image: Image) -> Image: 180 | '''Return an image''' 181 | return Image(image) 182 | 183 | 184 | def f3(dictionary: Dictionary) -> Dictionary: 185 | '''Return a raw dictionary''' 186 | return Dictionary(dictionary) 187 | 188 | 189 | def f4(image: Image) -> int: 190 | '''Return the size in bytes of the image''' 191 | return len(image) 192 | 193 | 194 | def f5(x: int, y: int) -> Image: 195 | '''Return an empty image''' 196 | return Image(b"\00" * x * y) 197 | 198 | 199 | @pytest.mark.parametrize( 200 | ["func", "f_in", "f_out", "in_media_type", "out_media_type", "in_is_raw", "out_is_raw"], ( 201 | pytest.param(f1, "test string", "test string", ["text/plain"], ["text/plain"], True, True, id="string"), 202 | pytest.param(f2, b'test bytes', b'test bytes', ["application/octet-stream"], ["application/octet-stream"], True, True, id="bytes"), 203 | pytest.param(f3, {'a': 1, 'b': 2}, {'a': 1, 'b': 2}, ["application/json"], ["application/json"], True, True, id="dict"), 204 | pytest.param(f4, b'test bytes', 10, ["application/octet-stream"], ["application/vnd.google.protobuf"], True, False, id="bytes->int"), 205 | pytest.param(f5, (2, 2), b"\00\00\00\00", ["application/vnd.google.protobuf"], ["application/octet-stream"], False, True, id="int->bytes"), 206 | )) 207 | def test_raw_type(func, f_in, f_out, in_media_type, out_media_type, in_is_raw, out_is_raw): 208 | '''Tests to make sure that supported raw data type models are working correctly''' 209 | model = Model(transform=func) 210 | model_name = 'my-model' 211 | 212 | with TemporaryDirectory() as tdir: 213 | with _dump_model(model, model_name) as dump_dir: 214 | _copy_dir(dump_dir, tdir, model_name) 215 | 216 | copied_dump_dir = path_join(tdir, model_name) 217 | metadata_file_path = path_join(copied_dump_dir, 'metadata.json') 218 | 219 | with open(metadata_file_path) as metadata_file: 220 | metadata_json = json.load(metadata_file) 221 | 222 | assert metadata_json['methods']['transform']['input']['media_type'] == in_media_type 223 | assert metadata_json['methods']['transform']['output']['media_type'] == out_media_type 224 | 225 | wrapped_model = load_model(copied_dump_dir) 226 | if in_is_raw: 227 | wrapped_return = wrapped_model.transform.from_raw(f_in) 228 | else: 229 | arguments = model.transform.input_type(*f_in) 230 | arguments_pb_msg = _pack_pb_msg(arguments, wrapped_model.transform._module) 231 | wrapped_return = wrapped_model.transform.from_pb_msg(arguments_pb_msg) 232 | 233 | if out_is_raw: 234 | ret = wrapped_return.as_raw() 235 | else: 236 | ret_pb_msg = wrapped_return.as_pb_msg() 237 | ret = _unpack_pb_msg(model.transform.output_type, ret_pb_msg).value 238 | 239 | assert ret == f_out 240 | 241 | 242 | @pytest.mark.flaky(reruns=5) 243 | def test_wrapped_sklearn(): 244 | '''Tests model wrap and load functionality''' 245 | 246 | iris = load_iris() 247 | X = iris.data 248 | y = iris.target 249 | 250 | clf = RandomForestClassifier(random_state=0) 251 | clf.fit(X, y) 252 | 253 | yhat = clf.predict(X) 254 | 255 | columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] 256 | X_df = pd.DataFrame(X, columns=columns) 257 | IrisDataFrame = create_dataframe('IrisDataFrame', X_df) 258 | 259 | def f1(data: IrisDataFrame) -> List[int]: 260 | '''Creates a numpy ndarray and predicts''' 261 | X = np.column_stack(data) 262 | return clf.predict(X) 263 | 264 | def f2(data: IrisDataFrame) -> List[int]: 265 | '''Creates a pandas DataFrame and predicts''' 266 | X = np.column_stack(data) 267 | df = pd.DataFrame(X, columns=columns) 268 | return clf.predict(df.values) 269 | 270 | in_ = tuple(col for col in X.T) 271 | out = (yhat, ) 272 | 273 | for func in (f1, f2): 274 | _generic_test(func, in_, out, wrapped_eq=lambda a, b: (a[0] == b[0]).all()) 275 | 276 | 277 | @pytest.mark.flaky(reruns=5) 278 | def test_wrapped_tensorflow(): 279 | '''Tests model wrap and load functionality''' 280 | tf.set_random_seed(0) 281 | 282 | iris = load_iris() 283 | data = iris.data 284 | target = iris.target 285 | target_onehot = pd.get_dummies(target).values.astype(float) 286 | 287 | # ============================================================================= 288 | # test with explicit session 289 | # ============================================================================= 290 | 291 | tf.reset_default_graph() 292 | 293 | session = tf.Session() 294 | x, y, prediction = _build_tf_model(session, data, target_onehot) 295 | yhat = session.run([prediction], {x: data})[0] 296 | 297 | X_df = pd.DataFrame(data, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) 298 | IrisDataFrame = create_dataframe('IrisDataFrame', X_df) 299 | 300 | def f1(df: IrisDataFrame) -> List[int]: 301 | '''Tests with explicit session provided''' 302 | X = np.column_stack(df) 303 | return prediction.eval({x: X}, session) 304 | 305 | in_ = tuple(col for col in data.T) 306 | out = (yhat, ) 307 | 308 | _generic_test(f1, in_, out, wrapped_eq=lambda a, b: (a[0] == b[0]).all(), preload=tf.reset_default_graph) 309 | 310 | # ============================================================================= 311 | # test with implicit default session 312 | # ============================================================================= 313 | 314 | tf.reset_default_graph() 315 | 316 | session = tf.InteractiveSession() 317 | x, y, prediction = _build_tf_model(session, data, target_onehot) 318 | yhat = session.run([prediction], {x: data})[0] 319 | 320 | def f2(df: IrisDataFrame) -> List[int]: 321 | '''Tests with implicit default session''' 322 | X = np.column_stack(df) 323 | return prediction.eval({x: X}) 324 | 325 | in_ = tuple(col for col in data.T) 326 | out = (yhat, ) 327 | 328 | _generic_test(f2, in_, out, wrapped_eq=lambda a, b: (a[0] == b[0]).all(), preload=tf.reset_default_graph) 329 | 330 | 331 | def _generic_test(func, in_, out, wrapped_eq=eq, pb_mg_eq=eq, pb_bytes_eq=eq, dict_eq=eq, json_eq=eq, preload=None, reqs=None, skip=None): 332 | '''Reusable wrap test routine with swappable equality functions''' 333 | 334 | model = Model(transform=func) 335 | model_name = 'my-model' 336 | 337 | with TemporaryDirectory() as tdir: 338 | with _dump_model(model, model_name, reqs) as dump_dir: 339 | _copy_dir(dump_dir, tdir, model_name) 340 | 341 | if preload is not None: 342 | preload() 343 | 344 | copied_dump_dir = path_join(tdir, model_name) 345 | wrapped_model = load_model(copied_dump_dir) 346 | 347 | TransIn = model.transform.input_type 348 | TransOut = model.transform.output_type 349 | 350 | trans_in = TransIn(*in_) 351 | trans_out = TransOut(*out) 352 | 353 | trans_in_pb = _pack_pb_msg(trans_in, wrapped_model.transform._module) 354 | trans_out_pb = _pack_pb_msg(trans_out, wrapped_model.transform._module) 355 | 356 | trans_in_pb_bytes = trans_in_pb.SerializeToString() 357 | trans_out_pb_bytes = trans_out_pb.SerializeToString() 358 | 359 | trans_in_dict = MessageToDict(trans_in_pb) 360 | trans_out_dict = MessageToDict(trans_out_pb) 361 | 362 | trans_in_json = MessageToJson(trans_in_pb, indent=0) 363 | trans_out_json = MessageToJson(trans_out_pb, indent=0) 364 | 365 | # test all from / as combinations 366 | for as_method_name, as_data_expected, eq_func in (('as_wrapped', trans_out, wrapped_eq), 367 | ('as_pb_msg', trans_out_pb, pb_mg_eq), 368 | ('as_pb_bytes', trans_out_pb_bytes, pb_bytes_eq), 369 | ('as_dict', trans_out_dict, dict_eq), 370 | ('as_json', trans_out_json, json_eq)): 371 | for from_method_name, from_data in (('from_wrapped', trans_in), 372 | ('from_pb_msg', trans_in_pb), 373 | ('from_pb_bytes', trans_in_pb_bytes), 374 | ('from_dict', trans_in_dict), 375 | ('from_json', trans_in_json)): 376 | 377 | if skip is not None and skip(as_method_name, from_method_name): 378 | logger.info("Skipping {} -> {}".format(from_method_name, as_method_name)) 379 | continue 380 | 381 | from_method = getattr(wrapped_model.transform, from_method_name) 382 | resp = from_method(from_data) 383 | as_data_method = getattr(resp, as_method_name) 384 | as_data = as_data_method() 385 | assert eq_func(as_data, as_data_expected) 386 | 387 | 388 | if __name__ == '__main__': 389 | '''Test area''' 390 | pytest.main([__file__, ]) 391 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/unpickler_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | """ 21 | Unpickles a model, given a context directory 22 | """ 23 | import sys 24 | 25 | from acumos.pickler import load_model, AcumosContextManager 26 | 27 | 28 | if __name__ == '__main__': 29 | '''Main''' 30 | del sys.path[0] # remove acumos/tests dir from python path 31 | 32 | context_dir = sys.argv[1] 33 | 34 | with AcumosContextManager(context_dir) as c: 35 | model_path = c.build_path('model.pkl') 36 | with open(model_path, 'rb') as f: 37 | model = load_model(f) # success if this doesn't explode 38 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/user_module.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a custom user-defined function in a lone module 21 | """ 22 | 23 | 24 | def user_function(x): 25 | return x 26 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/user_package/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/user_package/user_package_module.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | ''' 20 | Module containing custom code to be used in a model 21 | ''' 22 | 23 | 24 | def add_numbers(x, y): 25 | return x + y 26 | -------------------------------------------------------------------------------- /acumos-package/acumos/tests/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | """ 21 | Provides testing utils 22 | """ 23 | import subprocess 24 | import os 25 | from os.path import dirname 26 | 27 | import acumos 28 | 29 | 30 | TEST_DIR = dirname(__file__) 31 | ACUMOS_DIR = dirname(dirname(acumos.__file__)) 32 | 33 | 34 | def run_command(cmd): 35 | '''Runs a given command and raises AcumosError on process failure''' 36 | env = {'PATH': os.environ['PATH'], 'PYTHONPATH': ACUMOS_DIR} 37 | proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, env=env) 38 | _, stderr = proc.communicate() 39 | assert proc.returncode == 0, stderr.decode() 40 | -------------------------------------------------------------------------------- /acumos-package/acumos/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides model wrapping utilities 21 | """ 22 | from importlib.util import spec_from_file_location, module_from_spec 23 | import os 24 | import inspect 25 | import contextlib 26 | from collections import OrderedDict, namedtuple 27 | import typing_inspect 28 | 29 | from acumos.exc import AcumosError 30 | 31 | 32 | def namedtuple_field_types(nt): 33 | '''Returns an OrderedDict corresponding to NamedTuple field types''' 34 | field_types = nt.__annotations__ 35 | return OrderedDict((field, field_types[field]) for field in nt._fields) 36 | 37 | 38 | def load_module(fullname, path): 39 | '''Imports and returns a module from path for Python 3.5+''' 40 | spec = spec_from_file_location(fullname, path) 41 | module = module_from_spec(spec) 42 | spec.loader.exec_module(module) 43 | return module 44 | 45 | 46 | def load_artifact(*path, module, mode): 47 | '''Artifact loader helper''' 48 | with open(os.path.join(*path), mode) as f: 49 | return module.load(f) 50 | 51 | 52 | def dump_artifact(*path, data, module, mode): 53 | '''Artifact saver helper''' 54 | with open(os.path.join(*path), mode) as f: 55 | if module is None: 56 | f.write(data) 57 | else: 58 | module.dump(data, f) 59 | 60 | 61 | def get_qualname(o): 62 | if inspect.isclass(o): 63 | return "{}.{}".format(o.__module__, o.__name__) 64 | else: 65 | return get_qualname(o.__class__) 66 | 67 | 68 | @contextlib.contextmanager 69 | def reraise(prefix, prefix_args): 70 | '''Reraises an exception with a more informative prefix''' 71 | try: 72 | yield 73 | except AcumosError as e: 74 | raise AcumosError("{}: {}".format(prefix.format(*prefix_args), e)).with_traceback(e.__traceback__) 75 | 76 | 77 | InspectedType = namedtuple('InspectedType', ['type', # The real type 78 | 'origin', # The unsubscribed / unaliased typed 79 | 'args', # The type arguments 80 | ]) 81 | 82 | 83 | def inspect_type(t) -> InspectedType: 84 | """Returns basic information on a type as an InspectedType""" 85 | args = typing_inspect.get_args(t) 86 | if typing_inspect.is_generic_type(t): 87 | origin = typing_inspect.get_origin(t) 88 | else: 89 | origin = t 90 | return InspectedType(type=t, origin=origin, args=args) 91 | -------------------------------------------------------------------------------- /acumos-package/acumos/wrapped.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides wrapped model utilities 21 | """ 22 | import sys 23 | from zipfile import ZipFile 24 | from os import listdir 25 | from os.path import isfile, isdir, join as path_join 26 | 27 | from google.protobuf.json_format import Parse as ParseJson, ParseDict, MessageToJson, MessageToDict 28 | 29 | from acumos.modeling import _is_namedtuple, List, Dict 30 | from acumos.pickler import AcumosContextManager, load_model as _load_model 31 | from acumos.utils import inspect_type, load_module 32 | from acumos.exc import AcumosError 33 | 34 | 35 | def load_model(path): 36 | '''Returns a WrappedModel previously dumped to `path`''' 37 | model_dir = _infer_model_dir(path) 38 | with AcumosContextManager(model_dir) as c: 39 | _extend_syspath(c) # extend path before we unpickle user model 40 | 41 | with open(c.build_path('model.pkl'), 'rb') as f: 42 | model = _load_model(f) 43 | 44 | pkg = c.parameters['protobuf_package'] 45 | module_path = c.build_path(path_join('scripts', 'acumos_gen', pkg, 'model_pb2.py')) 46 | module = load_module('model_pb2', module_path) 47 | return WrappedModel(model, module) 48 | 49 | 50 | def _infer_model_dir(path): 51 | '''Returns an absolute path to the model dir. Unzips the model archive if `path` contains it''' 52 | model_zip_path = path_join(path, 'model.zip') 53 | if isfile(model_zip_path): 54 | model_dir = path_join(path, 'model') 55 | zip_file = ZipFile(model_zip_path) 56 | zip_file.extractall(model_dir) 57 | else: 58 | model_dir = path 59 | 60 | pkl_path = path_join(model_dir, 'model.pkl') 61 | if not isfile(pkl_path): 62 | raise AcumosError("Provided path {} does not contain an Acumos model".format(path)) 63 | 64 | return model_dir 65 | 66 | 67 | def _extend_syspath(context): 68 | '''Adds user-provided packages to the system path''' 69 | provided_abspath = context.build_path(path_join('scripts', 'user_provided')) 70 | if provided_abspath not in sys.path: 71 | sys.path.append(provided_abspath) 72 | 73 | for pkg_name in listdir(provided_abspath): 74 | pkg_abspath = path_join(provided_abspath, pkg_name) 75 | if not isdir(pkg_abspath): 76 | continue 77 | 78 | if pkg_abspath not in sys.path: 79 | sys.path.append(pkg_abspath) 80 | 81 | 82 | class WrappedModel(object): 83 | '''Container of WrappedFunction objects''' 84 | 85 | def __init__(self, model, module): 86 | self._methods = {name: WrappedFunction(func, module) for name, func in model.methods.items()} 87 | for k, v in self._methods.items(): 88 | setattr(self, k, v) 89 | 90 | @property 91 | def methods(self): 92 | return self._methods 93 | 94 | 95 | class WrappedFunction(object): 96 | '''A function wrapper with various consumption options''' 97 | 98 | def __init__(self, func, module): 99 | self._func = func 100 | self._module = module 101 | self._input_type = func.input_type 102 | self._output_type = func.output_type 103 | self._pb_input_type = getattr(module, func.input_type.__name__) if _is_namedtuple(self._input_type) else None 104 | self._pb_output_type = getattr(module, func.output_type.__name__) if _is_namedtuple(self._output_type) else None 105 | 106 | def from_pb_bytes(self, pb_bytes_in): 107 | '''Consumes a binary Protobuf message and returns a WrappedResponse object''' 108 | pb_msg_in = self._pb_input_type.FromString(pb_bytes_in) 109 | return self.from_pb_msg(pb_msg_in) 110 | 111 | def from_pb_msg(self, pb_msg_in): 112 | '''Consumes a Protobuf message object and returns a WrappedResponse object''' 113 | wrapped_in = _unpack_pb_msg(self._input_type, pb_msg_in) 114 | return self.from_wrapped(wrapped_in) 115 | 116 | def from_native(self, *args, **kwargs): 117 | '''Consumes the original inner function arguments and returns a WrappedResponse object''' 118 | wrapped_in = self._input_type(*args, **kwargs) 119 | return self.from_wrapped(wrapped_in) 120 | 121 | def from_wrapped(self, wrapped_in): 122 | '''Consumes a NamedTuple wrapped type and returns a WrappedResponse object''' 123 | wrapped_out = self._func.wrapped(wrapped_in) 124 | return WrappedResponse(wrapped_out, self._module, self._pb_output_type) 125 | 126 | def from_dict(self, dict_in): 127 | '''Consumes a dict and returns a WrappedResponse object''' 128 | pb_msg_in = ParseDict(dict_in, self._pb_input_type()) 129 | return self.from_pb_msg(pb_msg_in) 130 | 131 | def from_json(self, json_in): 132 | '''Consumes a json str and returns a WrappedResponse object''' 133 | pb_msg_in = ParseJson(json_in, self._pb_input_type()) 134 | return self.from_pb_msg(pb_msg_in) 135 | 136 | def from_raw(self, raw_in): 137 | '''Consumes a raw type data and returns a WrappedResponse object''' 138 | return self.from_wrapped(raw_in) 139 | 140 | @property 141 | def pb_input_type(self): 142 | return self._pb_input_type 143 | 144 | @property 145 | def pb_output_type(self): 146 | return self._pb_output_type 147 | 148 | 149 | class WrappedResponse(object): 150 | '''A WrappedFunction response with various return options''' 151 | 152 | def __init__(self, resp, module, pb_output_type): 153 | self._resp = resp 154 | self._module = module 155 | self._pb_output_type = pb_output_type 156 | 157 | def as_pb_bytes(self): 158 | '''Returns a Protobuf binary string representation of the model response''' 159 | return self.as_pb_msg().SerializeToString() 160 | 161 | def as_pb_msg(self): 162 | '''Returns a Protobuf message representation of the model response''' 163 | return _pack_pb_msg(self._resp, self._module) 164 | 165 | def as_wrapped(self): 166 | '''Returns a Python NamedTuple representation of the model response''' 167 | return self._resp 168 | 169 | def as_dict(self): 170 | '''Returns a dict representation of the model response''' 171 | pb_msg_out = self.as_pb_msg() 172 | return MessageToDict(pb_msg_out, self._pb_output_type()) 173 | 174 | def as_json(self): 175 | '''Returns a json str representation of the model response''' 176 | pb_msg_out = self.as_pb_msg() 177 | return MessageToJson(pb_msg_out, self._pb_output_type(), indent=0) 178 | 179 | def as_raw(self): 180 | '''Returns a raw data type representation of the model response''' 181 | return self._resp 182 | 183 | 184 | def _pack_pb_msg(wrapped_in, module): 185 | '''Returns a protobuf message object from a NamedTuple instance''' 186 | wrapped_type = type(wrapped_in) 187 | field_types = wrapped_type.__annotations__ 188 | pb_type = getattr(module, wrapped_type.__name__) 189 | return pb_type(**{f: _set_pb_value(field_types[f], v, module) for f, v in zip(wrapped_in._fields, wrapped_in)}) 190 | 191 | 192 | def _set_pb_value(wrapped_type, value, module): 193 | '''Recursively traverses the NamedTuple instance to ensure nested NamedTuples become protobuf messages''' 194 | 195 | inspected_type = inspect_type(wrapped_type) 196 | 197 | if _is_namedtuple(wrapped_type): 198 | return _pack_pb_msg(value, module) 199 | 200 | elif issubclass(inspected_type.origin, Dict): 201 | _, val_type = inspected_type.args 202 | if _is_namedtuple(val_type): 203 | return {k: _pack_pb_msg(v, module) for k, v in value.items()} 204 | 205 | elif issubclass(inspected_type.origin, List): 206 | list_type = inspected_type.args[0] 207 | if _is_namedtuple(list_type): 208 | return [_pack_pb_msg(v, module) for v in value] 209 | 210 | return value 211 | 212 | 213 | def _unpack_pb_msg(input_type, pb_msg): 214 | '''Returns a NamedTuple from protobuf message''' 215 | values = {f: _get_pb_value(t, getattr(pb_msg, f)) for f, t in input_type.__annotations__.items()} 216 | return input_type(**values) 217 | 218 | 219 | def _get_pb_value(wrapped_type, pb_value): 220 | '''Recursively traverses the protobuf message to ensure nested messages become NamedTuples''' 221 | 222 | inspected_type = inspect_type(wrapped_type) 223 | 224 | if _is_namedtuple(wrapped_type): 225 | return _unpack_pb_msg(wrapped_type, pb_value) 226 | 227 | elif issubclass(inspected_type.origin, Dict): 228 | _, val_type = inspected_type.args 229 | if _is_namedtuple(val_type): 230 | return {k: _unpack_pb_msg(val_type, v) for k, v in pb_value.items()} 231 | 232 | elif issubclass(inspected_type.origin, List): 233 | list_type = inspected_type.args[0] 234 | if _is_namedtuple(list_type): 235 | return [_unpack_pb_msg(list_type, v) for v in pb_value] 236 | 237 | return pb_value 238 | -------------------------------------------------------------------------------- /acumos-package/docs: -------------------------------------------------------------------------------- 1 | ../docs -------------------------------------------------------------------------------- /acumos-package/examples/image_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides an image Acumos model example 21 | 22 | This model returns metadata about input images 23 | """ 24 | import io 25 | 26 | import PIL 27 | 28 | from acumos.modeling import Model, create_namedtuple 29 | from acumos.session import AcumosSession 30 | 31 | 32 | ImageShape = create_namedtuple('ImageShape', [('width', int), ('height', int)]) 33 | 34 | 35 | def get_format(data: bytes) -> str: 36 | '''Returns the format of an image''' 37 | buffer = io.BytesIO(data) 38 | img = PIL.Image.open(buffer) 39 | return img.format 40 | 41 | 42 | def get_shape(data: bytes) -> ImageShape: 43 | '''Returns the width and height of an image''' 44 | buffer = io.BytesIO(data) 45 | img = PIL.Image.open(buffer) 46 | shape = ImageShape(width=img.width, height=img.height) 47 | return shape 48 | 49 | 50 | model = Model(get_format=get_format, get_shape=get_shape) 51 | 52 | session = AcumosSession() 53 | session.dump_zip(model, 'image-model', './image-model.zip', replace=True) # creates ./image-model.zip 54 | -------------------------------------------------------------------------------- /acumos-package/examples/keras_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a keras Acumos model example 21 | """ 22 | import numpy as np 23 | import pandas as pd 24 | from sklearn.datasets import load_iris 25 | from keras.models import Sequential 26 | from keras.layers import Dense 27 | 28 | from acumos.session import AcumosSession 29 | from acumos.modeling import Model, List, create_dataframe 30 | 31 | 32 | iris = load_iris() 33 | X = iris.data 34 | y = pd.get_dummies(iris.target).values 35 | 36 | clf = Sequential() 37 | clf.add(Dense(3, input_dim=4, activation='relu')) 38 | clf.add(Dense(3, activation='softmax')) 39 | clf.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) 40 | clf.fit(X, y) 41 | 42 | X_df = pd.DataFrame(X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) 43 | IrisDataFrame = create_dataframe('IrisDataFrame', X_df) 44 | 45 | 46 | def classify_iris(df: IrisDataFrame) -> List[int]: 47 | '''Returns an array of iris classifications''' 48 | X = np.column_stack(df) 49 | return clf.predict_classes(X) 50 | 51 | 52 | model = Model(classify=classify_iris) 53 | 54 | session = AcumosSession() 55 | session.dump_zip(model, 'keras', './keras.zip', replace=True) # creates ./keras.zip 56 | -------------------------------------------------------------------------------- /acumos-package/examples/raw_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2018-2019 Huawei Technologies Co. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by Huawei Technologies Co. 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | ''' 20 | Dumps an example model for illustrating acumos_model_runner usage 21 | ''' 22 | 23 | from acumos.session import AcumosSession 24 | from acumos.modeling import Model, new_type 25 | 26 | # allow users to specify a "raw" bytes type. no protobuf message is generated here 27 | Image = new_type(bytes, 'Image', {'dcae_input_name': 'a', 'dcae_output_name': 'a'}, 'example description') 28 | 29 | 30 | def image_func(image: Image) -> Image: 31 | '''Return an image''' 32 | return Image(image) 33 | 34 | 35 | if __name__ == '__main__': 36 | '''Main''' 37 | model = Model(imgae_func=image_func) 38 | 39 | session = AcumosSession() 40 | session.dump_zip(model, 'raw', './raw.zip', replace=True) # creates ./raw.zip 41 | -------------------------------------------------------------------------------- /acumos-package/examples/sklearn/face_completion_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a scikit-learn Acumos model example 21 | 22 | Example adapted from http://scikit-learn.org/stable/auto_examples/plot_multioutput_face_completion.html 23 | """ 24 | import numpy as np 25 | 26 | from sklearn.datasets import fetch_olivetti_faces 27 | from sklearn.utils.validation import check_random_state 28 | 29 | from sklearn.neighbors import KNeighborsRegressor 30 | 31 | from acumos.modeling import Model, List, create_namedtuple 32 | from acumos.session import AcumosSession 33 | 34 | 35 | # ============================================================================= 36 | # from the original example above 37 | # ============================================================================= 38 | 39 | # Load the faces datasets 40 | data = fetch_olivetti_faces() 41 | targets = data.target 42 | 43 | data = data.images.reshape((len(data.images), -1)) 44 | train = data[targets < 30] 45 | test = data[targets >= 30] # Test on independent people 46 | 47 | # Test on a subset of people 48 | n_faces = 5 49 | rng = check_random_state(4) 50 | face_ids = rng.randint(test.shape[0], size=(n_faces, )) 51 | test = test[face_ids, :] 52 | 53 | n_pixels = data.shape[1] 54 | # Upper half of the faces 55 | X_train = train[:, :(n_pixels + 1) // 2] 56 | # Lower half of the faces 57 | y_train = train[:, n_pixels // 2:] 58 | X_test = test[:, :(n_pixels + 1) // 2] 59 | y_test = test[:, n_pixels // 2:] 60 | 61 | knn = KNeighborsRegressor() 62 | knn.fit(X_train, y_train) 63 | 64 | # ============================================================================= 65 | # Acumos specific code 66 | # ============================================================================= 67 | 68 | # represents a single "flattened" [1 x n] image array 69 | FlatImage = create_namedtuple('FlatImage', [('image', List[float])]) 70 | 71 | # represents a collection of flattened image arrays 72 | FlatImages = create_namedtuple('FlatImages', [('images', List[FlatImage])]) 73 | 74 | 75 | def complete_faces(images: FlatImages) -> FlatImages: 76 | '''Predicts the bottom half of each input image''' 77 | X = np.vstack(images).squeeze() # creates an [m x n] matrixs with m images and n pixels 78 | yhat = knn.predict(X) 79 | return FlatImages([FlatImage(row) for row in yhat]) 80 | 81 | 82 | model = Model(complete_faces=complete_faces) 83 | 84 | session = AcumosSession() 85 | session.dump(model, 'face-model', '.') # creates ./face-model 86 | -------------------------------------------------------------------------------- /acumos-package/examples/sklearn/iris_random_forest_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a scikit-learn Acumos model example 21 | """ 22 | import numpy as np 23 | 24 | from sklearn.datasets import load_iris 25 | from sklearn.ensemble import RandomForestClassifier 26 | 27 | from acumos.modeling import Model, List, create_namedtuple 28 | from acumos.session import AcumosSession 29 | 30 | 31 | iris = load_iris() 32 | X = iris.data 33 | y = iris.target 34 | 35 | clf = RandomForestClassifier(random_state=0) 36 | clf.fit(X, y) 37 | 38 | IrisDataFrame = create_namedtuple('IrisDataFrame', [('sepal_length', List[float]), 39 | ('sepal_width', List[float]), 40 | ('petal_length', List[float]), 41 | ('petal_width', List[float])]) 42 | 43 | # ============================================================================= 44 | # # starting in Python 3.6, you can alternatively use this simpler syntax: 45 | # 46 | # from acumos.modeling import NamedTuple 47 | # 48 | # class IrisDataFrame(NamedTuple): 49 | # '''DataFrame corresponding to the Iris dataset''' 50 | # sepal_length: List[float] 51 | # sepal_width: List[float] 52 | # petal_length: List[float] 53 | # petal_width: List[float] 54 | # ============================================================================= 55 | 56 | # ============================================================================= 57 | # # A pandas DataFrame can also be used to infer appropriate NamedTuple types: 58 | # 59 | # import pandas as pd 60 | # from acumos.modeling import create_dataframe 61 | # 62 | # X_df = pd.DataFrame(X, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) 63 | # IrisDataFrame = create_dataframe('IrisDataFrame', X_df) 64 | # ============================================================================= 65 | 66 | 67 | def classify_iris(df: IrisDataFrame) -> List[int]: 68 | '''Returns an array of iris classifications''' 69 | X = np.column_stack(df) 70 | return clf.predict(X) 71 | 72 | 73 | model = Model(classify=classify_iris) 74 | 75 | session = AcumosSession() 76 | session.dump(model, 'my-iris', '.') # creates ./my-iris 77 | -------------------------------------------------------------------------------- /acumos-package/examples/tensorflow_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | """ 20 | Provides a TensorFlow Acumos model example 21 | """ 22 | import numpy as np 23 | import pandas as pd 24 | import tensorflow as tf 25 | from sklearn.datasets import load_iris 26 | from sklearn.metrics import classification_report 27 | 28 | from acumos.session import AcumosSession 29 | from acumos.modeling import Model, List, create_dataframe 30 | 31 | 32 | iris = load_iris() 33 | data = iris.data 34 | target = iris.target 35 | target_onehot = pd.get_dummies(target).values.astype(float) 36 | 37 | # ============================================================================= 38 | # build model 39 | # ============================================================================= 40 | 41 | x = tf.placeholder(shape=[None, 4], dtype=tf.float32) 42 | y = tf.placeholder(shape=[None, 3], dtype=tf.float32) 43 | 44 | layer1 = tf.layers.dense(x, 3, activation=tf.nn.relu) 45 | layer2 = tf.layers.dense(layer1, 3, activation=tf.nn.relu) 46 | logits = tf.layers.dense(layer2, 3) 47 | 48 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) 49 | optimizer = tf.train.GradientDescentOptimizer(0.075).minimize(cost) 50 | 51 | # ============================================================================= 52 | # train model & predict 53 | # ============================================================================= 54 | 55 | init = tf.global_variables_initializer() 56 | sess = tf.Session() 57 | sess.run([init]) 58 | 59 | for epoch in range(1000): 60 | _, loss = sess.run([optimizer, cost], feed_dict={x: data, y: target_onehot}) 61 | print("Epoch {} | Loss {}".format(epoch, loss)) 62 | 63 | prediction = tf.argmax(logits, 1) 64 | yhat = sess.run([prediction], {x: data})[0] 65 | 66 | # note: this predicts on the training set for illustration purposes only 67 | print(classification_report(target, yhat)) 68 | 69 | # ============================================================================= 70 | # create a acumos model from the tensorflow model 71 | # ============================================================================= 72 | 73 | X_df = pd.DataFrame(data, columns=['sepal_length', 'sepal_width', 'petal_length', 'petal_width']) 74 | IrisDataFrame = create_dataframe('IrisDataFrame', X_df) 75 | 76 | 77 | def classify_iris(df: IrisDataFrame) -> List[int]: 78 | '''Returns an array of iris classifications''' 79 | X = np.column_stack(df) 80 | return prediction.eval({x: X}, sess) 81 | 82 | 83 | model = Model(classify=classify_iris) 84 | 85 | session = AcumosSession() 86 | session.dump_zip(model, 'tensorflow', './tensorflow.zip', replace=True) # creates ./tensorflow.zip 87 | -------------------------------------------------------------------------------- /acumos-package/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 4.0.0 6 | org.acumos.acumos-python-client 7 | acumos-python-client 8 | 1.0.1 9 | 10 | UTF-8 11 | . 12 | xunit-results.xml 13 | coverage.xml 14 | py 15 | python 16 | **/**.py 17 | **/tests/**.py,**/testing/**.py,examples/**.py,setup.py 18 | 19 | 20 | -------------------------------------------------------------------------------- /acumos-package/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test = tox 3 | 4 | [bdist_wheel] 5 | universal = 1 6 | -------------------------------------------------------------------------------- /acumos-package/setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | from os.path import dirname, abspath, join as path_join 20 | from setuptools import setup, find_packages 21 | 22 | 23 | SETUP_DIR = abspath(dirname(__file__)) 24 | DOCS_DIR = path_join(SETUP_DIR, 'docs') 25 | 26 | with open(path_join(SETUP_DIR, 'acumos', '_version.py')) as file: 27 | globals_dict = dict() 28 | exec(file.read(), globals_dict) 29 | __version__ = globals_dict['__version__'] 30 | 31 | 32 | def _long_descr(): 33 | '''Yields the content of documentation files for the long description''' 34 | for file in ('user-guide.rst', 'tutorial/index.rst', 'release-notes.rst', 'developer-guide.rst'): 35 | doc_path = path_join(DOCS_DIR, file) 36 | with open(doc_path) as f: 37 | yield f.read() 38 | 39 | 40 | setup( 41 | author='Paul Triantafyllou', 42 | author_email='trianta@research.att.com', 43 | classifiers=[ 44 | 'Development Status :: 4 - Beta', 45 | 'Intended Audience :: Developers', 46 | 'Intended Audience :: Science/Research', 47 | 'Programming Language :: Python :: 3', 48 | 'Programming Language :: Python :: 3.6', 49 | 'Programming Language :: Python :: 3.7', 50 | 'Programming Language :: Python :: 3.8', 51 | 'Programming Language :: Python :: 3.9', 52 | 'License :: OSI Approved :: Apache Software License', 53 | ], 54 | description='Acumos client library for building and pushing Python models', 55 | install_requires=['protobuf', 56 | 'requests', 57 | 'numpy', 58 | 'dill', 59 | 'appdirs', 60 | 'filelock', 61 | 'grpcio', 62 | 'zipp', 63 | 'typing_inspect'], 64 | keywords='acumos machine learning model modeling artificial intelligence ml ai', 65 | license='Apache License 2.0', 66 | long_description='\n'.join(_long_descr()), 67 | long_description_content_type="text/x-rst", 68 | name='acumos', 69 | packages=find_packages(), 70 | python_requires='>=3.6, <3.10', 71 | url='https://gerrit.acumos.org/r/gitweb?p=acumos-python-client.git', 72 | version=__version__, 73 | ) 74 | -------------------------------------------------------------------------------- /acumos-package/testing/test-requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | -------------------------------------------------------------------------------- /acumos-package/testing/tox-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-rerunfailures 3 | pytest-cov 4 | connexion==1.5.2 5 | werkzeug<1.0.0 6 | pandas<0.25 7 | scikit-learn~=0.20.3 8 | jsonschema==2.6.0 9 | keras==2.2.4 10 | h5py~=2.10 # h5py 3.0 was released and it seems to cause issues with our version of keras 11 | tensorflow==1.15.2 12 | pexpect==4.6.0 13 | pillow 14 | mock 15 | #Keras-contrib is deprecated. Use TensorFlow Addons. 16 | git+https://github.com/keras-team/keras-contrib.git#egg=keras_contrib 17 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/README.md: -------------------------------------------------------------------------------- 1 | # testing/wrap 2 | This directory provides example applications that demonstrate how wrapped models work 3 | 4 | # Scripts 5 | ## dump\_example\_model.py 6 | This script trains a scikit-learn model on the iris dataset and dumps the model to the present working directory. 7 | 8 | ``` 9 | $ cd testing/wrap 10 | $ python dump_example_model.py 11 | ``` 12 | 13 | This creates additional files that would be used for pushing the dumped model to the Acumos upload server: 14 | 15 | ``` 16 | . 17 | ├── model 18 | │   ├── metadata.json 19 | │   ├── model.pkl 20 | │   ├── model.py 21 | │   ├── wrap.json 22 | │   ├── model.proto 23 | │   └── model.zip 24 | 25 | ``` 26 | 27 | ## talker.py 28 | This script continuously sends DataFrame messages to the model runner script every 5 seconds by default. 29 | 30 | ## runner.py 31 | This script loads the dumped model and uses it to dynamically add `flask` endpoints. In this instance, the scikit-learn model implements the `transform` API which results in a `/transform` endpoint. 32 | 33 | ``` 34 | usage: runner.py [-h] [--port PORT] [--modeldir MODELDIR] [--json_io] 35 | [--return_output] 36 | 37 | optional arguments: 38 | -h, --help show this help message and exit 39 | --port PORT 40 | --modeldir MODELDIR specify the model directory to load 41 | --json_io input+output rich JSON instead of protobuf 42 | --return_output return output in response instae of just downstream 43 | ``` 44 | 45 | To test JSON-based endpoints, you can specify the flag `--json_io` and the app will attempt ot decode and encode outputs in JSON. 46 | 47 | Note that the downstream applications that are being "published" to are defined in `runtime.json` file via the `downstream` key. However, you can 48 | also request that the output is included in the response with the flag `--return_output`. 49 | 50 | The following examples are provided for curl-based evaluation from a command-line. 51 | 52 | ``` 53 | (as GET) 54 | curl -X GET "http://localhost:3330/transform?x0=123&x2=0.31&x1=0.77&x3=0.12" 55 | 56 | (as POST) 57 | curl --data x1=123 -d x0=0.2 -d x2=0.5 -d x3=0.1 -X POST http://localhost:3330/transform 58 | 59 | (as POST, with multiple) 60 | curl --data x1=123 -d x0=0.2 -d x2=0.5 -d x3=0.1 -d x1=2 -d x0=0.1 -d x2=3 -d x3=0.4 -X POST http://localhost:3330/transform 61 | ``` 62 | 63 | 64 | ## listen.py 65 | This script receives Prediction messages produced by the model runner script and prints them to console. 66 | 67 | # Running the example 68 | Run all three applications together to create the pipeline: 69 | 70 | ``` 71 | $ python talker.py &> /dev/null & 72 | $ python runner.py &> /dev/null & 73 | $ python listener.py 74 | ``` 75 | 76 | ## Running your own example. 77 | To aide in the testing of your own work, each of these scripts have an additional 78 | argument `--modeldir` that can be used to point to your own `model` directory 79 | Additionally, while you are encouraged to derive your own `talker` script, you can 80 | also utilize this script to feed test samples to your script by providing 81 | a CSV-based file. This changes the run patterns for these scripts as thus. 82 | 83 | ``` 84 | $ python talker.py --modeldir /some/path/model --csvdata /some/path/data.csv &> /dev/null & 85 | $ python runner.py --modeldir /some/path/model &> /dev/null & 86 | $ python listener.py --modeldir /some/path/model 87 | ``` 88 | 89 | # Swagger and Wrapper example 90 | To aide in the development and export of models to a swagger/webapp interface 91 | a sample script was created to inspect models and generate python `dict` wrapper. To call 92 | this sample jut point it at your target model directory and a simple output will be 93 | generated for all methods. If you don't have a model a simple model will be dumped to the 94 | target directory. 95 | 96 | ``` 97 | $ python swagger.py --modeldir /some/path/model 98 | 99 | (output) 100 | [{ 101 | 'name': 'transform', 102 | 'out': { 103 | 'predictions': 104 | }, 105 | 'in': { 106 | 'x1': , 107 | 'x3': , 108 | 'x0': , 109 | 'x2': 110 | } 111 | }] 112 | ``` 113 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/dump_example_model.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | ''' 20 | Generates a model and dumps it to present working directory for testing purposes 21 | ''' 22 | import pandas as pd 23 | import numpy as np 24 | from sklearn.datasets import load_iris 25 | from sklearn.ensemble import RandomForestClassifier 26 | 27 | from acumos.modeling import Model, List, create_namedtuple, create_dataframe 28 | from acumos.session import AcumosSession 29 | 30 | 31 | if __name__ == '__main__': 32 | '''Main''' 33 | 34 | iris = load_iris() 35 | X = iris.data 36 | y = iris.target 37 | 38 | clf = RandomForestClassifier(random_state=0) 39 | clf.fit(X, y) 40 | 41 | columns = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth'] 42 | X_df = pd.DataFrame(X, columns=columns) 43 | 44 | DataFrame = create_dataframe('DataFrame', X_df) 45 | Predictions = create_namedtuple('Predictions', [('predictions', List[int])]) 46 | 47 | def predict(df: DataFrame) -> Predictions: 48 | '''Predicts the class of iris''' 49 | X = np.column_stack(df) 50 | yhat = clf.predict(X) 51 | preds = Predictions(predictions=yhat) 52 | return preds 53 | 54 | model = Model(transform=predict) 55 | 56 | s = AcumosSession(None) 57 | s.dump(model, 'model', '.') 58 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/listener.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | ''' 21 | Provides a "listener" application that listens for Prediction messages 22 | ''' 23 | import argparse 24 | 25 | from flask import Flask, request 26 | 27 | from acumos.wrapped import load_model 28 | 29 | 30 | if __name__ == '__main__': 31 | '''Main''' 32 | parser = argparse.ArgumentParser() 33 | parser.add_argument("--port", type=int, default=3331) 34 | parser.add_argument("--modeldir", type=str, default='model') 35 | pargs = parser.parse_args() 36 | 37 | model = load_model(pargs.modeldir) 38 | Prediction = model.transform.pb_output_type # need the Prediction message definition to deserialize 39 | 40 | app = Flask(__name__) 41 | 42 | @app.route('/listen', methods=['POST']) 43 | def listen(): 44 | bytes_in = request.data 45 | msg = Prediction.FromString(bytes_in) 46 | print("Received Prediction message: {}".format(msg.predictions)) 47 | return 'OK', 201 48 | 49 | print("Running Flask server on port {:}".format(pargs.port)) 50 | app.run(port=pargs.port) 51 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | ''' 21 | Provides a model runner application that subscribes to iris DataFrame messages and publishes Prediction messages 22 | ''' 23 | import argparse 24 | import json 25 | from functools import partial 26 | from os import path 27 | 28 | import requests 29 | from flask import Flask, request, make_response, current_app 30 | from google.protobuf import json_format 31 | from gunicorn.app.base import BaseApplication 32 | 33 | from acumos.wrapped import load_model 34 | 35 | 36 | def invoke_method(model_method, downstream): 37 | '''Consumes and produces protobuf binary data''' 38 | app = current_app 39 | content_type = "text/plain;charset=UTF-8" 40 | bytes_in = request.data 41 | if not bytes_in: 42 | if request.form: 43 | bytes_in = dict(request.form) 44 | elif request.args: 45 | bytes_in = dict(request.args) 46 | if type(bytes_in) == dict: # attempt to push arguments into JSON for more tolerant parsing 47 | bytes_in = json.dumps(bytes_in) 48 | bytes_out = None 49 | 50 | try: 51 | if app.json_io: 52 | msg_in = json_format.Parse(bytes_in, model_method.pb_input_type()) # attempt to decode JSON 53 | msg_out = model_method.from_pb_msg(msg_in) 54 | else: 55 | msg_out = model_method.from_pb_bytes(bytes_in) # default from-bytes method 56 | if app.json_io: 57 | bytes_out = json_format.MessageToJson(msg_out.as_pb_msg()) 58 | content_type = "application/javascript;charset=UTF-8" 59 | else: 60 | bytes_out = msg_out.as_pb_bytes() 61 | except json_format.ParseError as e: 62 | type_input = list(model_method.pb_input_type.DESCRIPTOR.fields_by_name.keys()) 63 | str_reply = "[invoke_method]: Value specification error, expected {}, {}".format(type_input, e) 64 | print(str_reply) 65 | resp = make_response(str_reply, 400) 66 | except (ValueError, TypeError) as e: 67 | str_reply = "[invoke_method]: Value conversion error: {}".format(e) 68 | print(str_reply) 69 | resp = make_response(str_reply, 400) 70 | 71 | if bytes_out is not None: 72 | resp = None 73 | for url in downstream: 74 | try: 75 | req_response = requests.post(url, data=bytes_out) 76 | if resp is None: # save only first response from downstream list 77 | resp = make_response(req_response.content, req_response.status_code) 78 | for header_test in ['Content-Type', 'content-type']: # test for content type to copy from downstream 79 | if header_test in req_response: 80 | content_type = req_response[header_test] 81 | except Exception as e: 82 | print("Failed to publish to downstream url {} : {}".format(url, e)) 83 | if app.return_output: 84 | if resp is None: # only if not received from downstream 85 | resp = make_response(bytes_out, 201) 86 | else: 87 | resp = make_response('OK', 201) 88 | 89 | resp.headers['Access-Control-Allow-Origin'] = '*' 90 | resp.headers['Content-Type'] = content_type 91 | return resp 92 | 93 | 94 | class StandaloneApplication(BaseApplication): 95 | '''Custom gunicorn app. Modified from http://docs.gunicorn.org/en/stable/custom.html''' 96 | 97 | def __init__(self, pargs): 98 | self.parsed_args = pargs 99 | self.options = {'bind': "{}:{}".format(pargs.host, pargs.port), 'workers': pargs.workers, 'timeout': pargs.timeout} 100 | super().__init__() 101 | 102 | def load_config(self): 103 | config = dict([(key, value) for key, value in self.options.items() 104 | if key in self.cfg.settings and value is not None]) 105 | for key, value in config.items(): 106 | self.cfg.set(key.lower(), value) 107 | 108 | def load(self): 109 | return build_app(self.parsed_args) 110 | 111 | 112 | def build_app(pargs): 113 | '''Builds and returns a Flask app''' 114 | downstream = [] 115 | if path.exists('runtime.json'): 116 | with open('runtime.json') as f: 117 | runtime = json.load(f) # ad-hoc way of giving the app runtime parameters 118 | if not pargs.no_downstream: 119 | downstream = runtime['downstream'] # list of IP:port/path urls 120 | print("Found downstream forward routes {}".format(downstream)) 121 | else: 122 | pargs.return_output = True 123 | 124 | model = load_model(pargs.modeldir) # refers to ./model dir in pwd. generated by helper script also in this dir 125 | 126 | app = Flask(__name__) 127 | app.json_io = pargs.json_io # store io flag 128 | app.return_output = not pargs.no_output # store output 129 | 130 | # dynamically add handlers depending on model capabilities 131 | for method_name, method in model.methods.items(): 132 | 133 | handler = partial(invoke_method, model_method=method, downstream=downstream) 134 | url = "/{}".format(method_name) 135 | app.add_url_rule(url, method_name, handler, methods=['POST', 'GET']) 136 | 137 | # render down the input in few forms 138 | typeInput = list(method.pb_input_type.DESCRIPTOR.fields_by_name.keys()) 139 | 140 | # render down the output in few forms 141 | typeOutput = list(method.pb_output_type.DESCRIPTOR.fields_by_name.keys()) 142 | 143 | print("Adding route {} [input:{}, output:{}]".format(url, typeInput, typeOutput)) 144 | 145 | return app 146 | 147 | 148 | if __name__ == '__main__': 149 | '''Main''' 150 | parser = argparse.ArgumentParser() 151 | parser.add_argument("--host", type=str, default='0.0.0.0') 152 | parser.add_argument("--port", type=int, default=3330) 153 | parser.add_argument("--timeout", type=int, default=120) 154 | parser.add_argument("--workers", type=int, default=2) 155 | parser.add_argument("--modeldir", type=str, default='model', help='specify the model directory to load') 156 | parser.add_argument("--json_io", action='store_true', help='input+output rich JSON instead of protobuf') 157 | parser.add_argument("--no_output", action='store_true', help='do not return output in response, only send downstream') 158 | parser.add_argument("--no_downstream", action='store_true', help='ignore downstream arguments even if in runtime') 159 | 160 | pargs = parser.parse_args() 161 | 162 | StandaloneApplication(pargs).run() 163 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/runtime.json: -------------------------------------------------------------------------------- 1 | {"downstream": ["http://127.0.0.1:3331/listen"]} 2 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/swagger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | ''' 21 | Provides a quick way to inspect protobuf and generate swagger example 22 | ''' 23 | import argparse 24 | from os.path import exists 25 | 26 | import pandas as pd 27 | 28 | # import yaml # note special dependency here 29 | 30 | from acumos.wrap.load import load_model 31 | from acumos.wrap.dump import dump_keras_model 32 | from google.protobuf.descriptor import FieldDescriptor 33 | 34 | 35 | _pb_type_lookup = { 36 | FieldDescriptor.TYPE_DOUBLE: float, 37 | FieldDescriptor.TYPE_FLOAT: float, 38 | FieldDescriptor.TYPE_INT64: int, 39 | FieldDescriptor.TYPE_UINT64: int, 40 | FieldDescriptor.TYPE_INT32: int, 41 | FieldDescriptor.TYPE_FIXED64: int, 42 | FieldDescriptor.TYPE_FIXED32: int, 43 | FieldDescriptor.TYPE_BOOL: bool, 44 | FieldDescriptor.TYPE_STRING: str, 45 | # TYPE_GROUP = 10 46 | # TYPE_MESSAGE = 11 47 | # TYPE_BYTES = 12 48 | FieldDescriptor.TYPE_UINT32: int, 49 | FieldDescriptor.TYPE_ENUM: int, 50 | FieldDescriptor.TYPE_SFIXED32: int, 51 | FieldDescriptor.TYPE_SFIXED64: int, 52 | FieldDescriptor.TYPE_SINT32: int, 53 | FieldDescriptor.TYPE_SINT64: int 54 | } 55 | 56 | 57 | def dump_yaml(list_methods): 58 | """Dumps a simple dictionary to a yaml file with the following expected format... 59 | list_methods = [ {'name':'transform_EXAMPLE', 'in':{'x1':float, 'x2':float, ...}, 'out':{'prediction':float}} ... ] 60 | """ 61 | print(list_methods) 62 | 63 | # TODO: method fill for actual yaml formatting 64 | # print(yaml.dump(list_methods)) 65 | 66 | 67 | def main(): 68 | parser = argparse.ArgumentParser() 69 | parser.add_argument("--modeldir", type=str, default='model') 70 | pargs = parser.parse_args() 71 | 72 | if not exists(pargs.modeldir): 73 | from sklearn.datasets import load_iris 74 | from keras.models import Sequential 75 | from keras.layers import Dense 76 | 77 | print("No local model directory found, creating a simple model... '{:}'".format(pargs.modeldir)) 78 | iris = load_iris() 79 | X = iris.data 80 | y = pd.get_dummies(iris.target).values 81 | 82 | model = Sequential() 83 | model.add(Dense(8, input_dim=4, activation='relu')) 84 | model.add(Dense(3, activation='softmax')) 85 | model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) 86 | model.fit(X, y) 87 | 88 | dump_keras_model(model, X, pargs.modeldir, name='keras-iris') 89 | 90 | print("Parsing model source directory... '{:}'".format(pargs.modeldir)) 91 | model = load_model(pargs.modeldir) # refers to ./model dir in pwd. generated by helper script also in this dir 92 | 93 | listMethod = [] 94 | for method_name in model.methods: # iterate through known methods 95 | objMethod = getattr(model, method_name) 96 | dictInput = {field.name: _pb_type_lookup[field.type] for field in objMethod.msg_in.DESCRIPTOR.fields} 97 | dictOutput = {field.name: _pb_type_lookup[field.type] for field in objMethod.msg_out.DESCRIPTOR.fields} 98 | listMethod.append({'name': method_name, 'in': dictInput, 'out': dictOutput}) 99 | dump_yaml(listMethod) 100 | 101 | 102 | if __name__ == '__main__': 103 | main() 104 | -------------------------------------------------------------------------------- /acumos-package/testing/wrap/talker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # ===============LICENSE_START======================================================= 3 | # Acumos Apache-2.0 4 | # =================================================================================== 5 | # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 6 | # =================================================================================== 7 | # This Acumos software file is distributed by AT&T and Tech Mahindra 8 | # under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # This file is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ===============LICENSE_END========================================================= 19 | # -*- coding: utf-8 -*- 20 | """ 21 | Provides a "talker" application that sends iris DataFrame protobuf messages 22 | """ 23 | import argparse 24 | import time 25 | 26 | import requests 27 | from sklearn.datasets import load_iris 28 | 29 | from acumos.wrapped import load_model 30 | 31 | 32 | if __name__ == '__main__': 33 | '''Main''' 34 | 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument("--uri", default='http://127.0.0.1:3330/transform') 37 | parser.add_argument("--sleep", default=5) 38 | parser.add_argument("--modeldir", type=str, default='model') 39 | parser.add_argument("--csvdata", type=str, default='') 40 | pargs = parser.parse_args() 41 | 42 | model = load_model(pargs.modeldir) # refers to ./model dir in pwd. generated by helper script also in this dir 43 | 44 | if pargs.csvdata: 45 | import pandas as pd 46 | dfRaw = pd.read_csv(pargs.csvdata) 47 | X = dfRaw.as_matrix() 48 | else: 49 | iris = load_iris() 50 | X = iris.data 51 | 52 | # build protobuf message that model consumes 53 | DataFrame = model.transform.pb_input_type 54 | 55 | X_msg = DataFrame() 56 | for col, field in enumerate(DataFrame.DESCRIPTOR.fields): 57 | getattr(X_msg, field.name).extend(X[:, col].tolist()) 58 | 59 | X_bytes = X_msg.SerializeToString() 60 | 61 | # eat errors and talk forever 62 | while True: 63 | try: 64 | requests.post(pargs.uri, data=X_bytes) 65 | except Exception as e: 66 | print("[ERROR] {}".format(e)) 67 | finally: 68 | time.sleep(pargs.sleep) 69 | -------------------------------------------------------------------------------- /acumos-package/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{36, 37},flake8 3 | 4 | [testenv] 5 | recreate = true 6 | setenv = 7 | PYTHONHASHSEED = 3330 8 | PYTHONPATH={toxinidir} 9 | passenv = * 10 | deps = -rtesting/tox-requirements.txt 11 | commands = pytest --junitxml xunit-results.xml --cov-fail-under=75 --cov=acumos --cov-report xml acumos/tests 12 | 13 | [testenv:flake8] 14 | basepython = python3.7 15 | skip_install = true 16 | deps = flake8 17 | commands = flake8 setup.py acumos testing examples 18 | 19 | [flake8] 20 | ignore = E501 21 | -------------------------------------------------------------------------------- /docs/developer-guide.rst: -------------------------------------------------------------------------------- 1 | .. ===============LICENSE_START======================================================= 2 | .. Acumos CC-BY-4.0 3 | .. =================================================================================== 4 | .. Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | .. =================================================================================== 6 | .. This Acumos documentation file is distributed by AT&T and Tech Mahindra 7 | .. under the Creative Commons Attribution 4.0 International License (the "License"); 8 | .. you may not use this file except in compliance with the License. 9 | .. You may obtain a copy of the License at 10 | .. 11 | .. http://creativecommons.org/licenses/by/4.0 12 | .. 13 | .. This file is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | .. ===============LICENSE_END========================================================= 18 | 19 | ==================================== 20 | Acumos Python Client Developer Guide 21 | ==================================== 22 | 23 | Testing 24 | ======= 25 | 26 | We use a combination of ``tox``, ``pytest``, and ``flake8`` to test 27 | ``acumos``. Code which is not PEP8 compliant (aside from E501) will be 28 | considered a failing test. You can use tools like ``autopep8`` to 29 | “clean” your code as follows: 30 | 31 | .. code:: bash 32 | 33 | $ pip install autopep8 34 | $ cd acumos-python-client 35 | $ autopep8 -r --in-place --ignore E501 acumos/ testing/ examples/ 36 | 37 | Run tox directly: 38 | 39 | .. code:: bash 40 | 41 | $ cd acumos-python-client 42 | $ export WORKSPACE=$(pwd) # env var normally provided by Jenkins 43 | $ tox 44 | 45 | You can also specify certain tox environments to test: 46 | 47 | .. code:: bash 48 | 49 | $ tox -e py36 # only test against Python 3.6 50 | $ tox -e flake8 # only lint code 51 | 52 | A set of integration test is also available in ``acumos-package/testing/integration_tests``. 53 | To run those, use ``acumos-package/testing/tox-integration.ini`` as tox config (-c flag), 54 | onboarding tests will be ran with python 3.6 to 3.9. 55 | You will need to set your user credentials and platform configuration in ``tox-integration.ini``. 56 | 57 | .. code:: bash 58 | 59 | $ tox -c acumos-package/testing/integration_tests 60 | 61 | 62 | Packaging 63 | ========= 64 | 65 | The RST files in the docs/ directory are used to publish HTML pages to 66 | ReadTheDocs.io and to build the package long description in setup.py. 67 | The symlink from the subdirectory acumos-package to the docs/ directory 68 | is required for the Python packaging tools. Those tools build a source 69 | distribution from files in the package root, the directory acumos-package. 70 | The MANIFEST.in file directs the tools to pull files from directory docs/, 71 | and the symlink makes it possible because the tools only look within the 72 | package root. 73 | -------------------------------------------------------------------------------- /docs/images/Acumos_logo_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/acumos/acumos-python-client/d4faad0ed3fe6da0c8b0bfb23b548fa9ace546e5/docs/images/Acumos_logo_white.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. ===============LICENSE_START======================================================= 2 | .. Acumos CC-BY-4.0 3 | .. =================================================================================== 4 | .. Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | .. =================================================================================== 6 | .. This Acumos documentation file is distributed by AT&T and Tech Mahindra 7 | .. under the Creative Commons Attribution 4.0 International License (the "License"); 8 | .. you may not use this file except in compliance with the License. 9 | .. You may obtain a copy of the License at 10 | .. 11 | .. http://creativecommons.org/licenses/by/4.0 12 | .. 13 | .. This file is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | .. ===============LICENSE_END========================================================= 18 | 19 | ==================== 20 | Acumos Python Client 21 | ==================== 22 | 23 | The User Guide is located on `PyPI `_. 24 | (Recommended version for Clio release is 0.8.0) 25 | 26 | .. toctree:: 27 | :maxdepth: 1 28 | 29 | release-notes 30 | developer-guide 31 | user-guide 32 | tutorial/index 33 | -------------------------------------------------------------------------------- /docs/release-notes.rst: -------------------------------------------------------------------------------- 1 | .. ===============LICENSE_START======================================================= 2 | .. Acumos CC-BY-4.0 3 | .. =================================================================================== 4 | .. Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | .. =================================================================================== 6 | .. This Acumos documentation file is distributed by AT&T and Tech Mahindra 7 | .. under the Creative Commons Attribution 4.0 International License (the "License"); 8 | .. you may not use this file except in compliance with the License. 9 | .. You may obtain a copy of the License at 10 | .. 11 | .. http://creativecommons.org/licenses/by/4.0 12 | .. 13 | .. This file is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | .. ===============LICENSE_END========================================================= 18 | 19 | ================================== 20 | Acumos Python Client Release Notes 21 | ================================== 22 | 23 | v1.0.1, 27 April 2021 24 | ===================== 25 | 26 | * use acumos-python-client > 0.8.0 with Acumos clio `ACUMOS-4330 `_ 27 | 28 | v1.0.0, 13 April 2021 29 | ===================== 30 | 31 | * Fix Type issue with python 3.9 `ACUMOS-4323 `_ 32 | 33 | v0.9.9, 13 April 2021 34 | ===================== 35 | 36 | * Take into account "deploy" parameter in acumos python client `ACUMOS-4303 `_ 37 | 38 | v0.9.8, 06 November 2020 39 | ======================== 40 | 41 | * Return docker URI & added an optional flag to replace and existing model when dumping `ACUMOS-4298 `_ 42 | * The model bundle can now be dumped directly as a zip file `ACUMOS-4273 `_ 43 | * Allow installation on python 3.9 `ACUMOS-4123 `_ 44 | 45 | v0.9.7, 27 August 2020 46 | ====================== 47 | 48 | * Add support of python 3.7 & 3.8 `ACUMOS-4123 `_ 49 | * Display acumos logo on github `ACUMOS-4094 `_ 50 | 51 | v0.9.4, 05 April 2020 52 | ===================== 53 | 54 | * Give image tag URL from python client `ACUMOS-3961 `_ 55 | 56 | v0.9.3, 30 Mar 2020 57 | =================== 58 | 59 | * Modify unstructured type section in pypi `ACUMOS-3956 `_ 60 | * Raise an Error when using asymetric type `ACUMOS-3956 `_ 61 | 62 | v0.9.2, 31 Jan 2020 63 | =================== 64 | 65 | * Remove support for python 3.5 `Gerrit-6275 `_ 66 | 67 | v0.9.1 68 | ====== 69 | 70 | * add raw format support `ACUMOS-2712 `_ 71 | * publish content type for long description `Gerrit-5504 `_ 72 | 73 | v0.8.0 74 | ====== 75 | (This is the recommended version for the Clio release) 76 | 77 | - Enhancements 78 | 79 | - Users may now specify additional options when pushing their Acumos model. See the options section in the tutorial for more information. 80 | - ``acumos`` now supports Keras models built with ``tensorflow.keras`` 81 | 82 | - Support changes 83 | 84 | - ``acumos`` no longer supports Python 3.4 85 | 86 | 87 | v0.7.2 88 | ====== 89 | 90 | - Bug fixes 91 | 92 | - The deprecated authentication API is now considered optional 93 | - A more portable path solution is now used when saving models, to avoid issues with models developed in Windows 94 | 95 | 96 | v0.7.1 97 | ====== 98 | 99 | - Authentication 100 | 101 | - Username and password authentication has been deprecated 102 | - Users are now interactively prompted for an onboarding token, as opposed to a username and password 103 | 104 | v0.7.0 105 | ====== 106 | 107 | - Requirements 108 | 109 | - Python script dependencies can now be specified using a Requirements object 110 | - Python script dependencies found during the introspection stage are now included with the model 111 | 112 | v0.6.5 113 | ====== 114 | 115 | - Bug fixes 116 | 117 | - Don't attempt to use an empty auth token (avoids blank strings to be set in environment) 118 | 119 | v0.6.4 120 | ====== 121 | 122 | - Bug fixes 123 | 124 | - The normalized path of the system base prefix is now used for identifying stdlib packages 125 | 126 | v0.6.3 127 | ====== 128 | 129 | - Bug fixes 130 | 131 | - Improved dependency inspection when using a virtualenv 132 | - Removed custom packages from model metadata, as it caused image build failures 133 | - Fixed Python 3.5.2 ordering bug in wrapped model usage 134 | 135 | v0.6.2 136 | ====== 137 | 138 | - TensorFlow 139 | 140 | - Fixed a serialization issue that occurred when using a frozen graph 141 | 142 | v0.6.1 143 | ====== 144 | 145 | - Model upload 146 | 147 | - The JWT is now cleared immediately after a failed upload 148 | - Additional HTTP information is now included in the error message 149 | 150 | v0.6.0 151 | ====== 152 | 153 | - Authentication token 154 | 155 | - A new environment variable ``ACUMOS_TOKEN`` can be used to short-circuit 156 | the authentication process 157 | 158 | - Extra headers 159 | 160 | - ``AcumosSession.push`` now accepts an optional ``extra_headers`` argument, 161 | which will allow users and systems to include additional information when 162 | pushing models to the onboarding server 163 | 164 | v0.5.0 165 | ====== 166 | 167 | - Modeling 168 | 169 | - Python 3.6 NamedTuple syntax support now tested 170 | - User documentation includes example of new NamedTuple syntax 171 | 172 | - Model wrapper 173 | 174 | - Model wrapper now has APIs for consuming and producing Python 175 | dicts and JSON strings 176 | 177 | - Protobuf and protoc 178 | 179 | - An explicit check for protoc is now made, which raises a more 180 | informative error message 181 | - User documentation is more clear about dependence on protoc, and 182 | provides an easier way to install protoc via Anaconda 183 | 184 | - Keras 185 | 186 | - The active keras backend is now included as a tracked module 187 | - keras_contrib layers are now supported 188 | 189 | v0.4.0 190 | ====== 191 | 192 | - Replaced library-specific onboarding functions with “new-style” 193 | models 194 | 195 | - Support for arbitrary Python functions using type hints 196 | - Support for custom user-defined types 197 | - Support for TensorFlow models 198 | - Improved dependency introspection 199 | - Improved object serialization mechanisms 200 | -------------------------------------------------------------------------------- /docs/user-guide.rst: -------------------------------------------------------------------------------- 1 | .. ===============LICENSE_START======================================================= 2 | .. Acumos CC-BY-4.0 3 | .. =================================================================================== 4 | .. Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved. 5 | .. =================================================================================== 6 | .. This Acumos documentation file is distributed by AT&T and Tech Mahindra 7 | .. under the Creative Commons Attribution 4.0 International License (the "License"); 8 | .. you may not use this file except in compliance with the License. 9 | .. You may obtain a copy of the License at 10 | .. 11 | .. http://creativecommons.org/licenses/by/4.0 12 | .. 13 | .. This file is distributed on an "AS IS" BASIS, 14 | .. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | .. See the License for the specific language governing permissions and 16 | .. limitations under the License. 17 | .. ===============LICENSE_END========================================================= 18 | 19 | =============================== 20 | Acumos Python Client User Guide 21 | =============================== 22 | 23 | 24 | |Build Status| 25 | 26 | ``acumos`` is a client library that allows modelers to push their Python models 27 | to the `Acumos platform `__. 28 | 29 | Installation 30 | ============ 31 | 32 | You will need a Python 3.6 or 3.7 environment in order to install ``acumos``. 33 | Python 3.8 and later can also be used starting with version 0.9.5, some AI 34 | framework like Tensor Flow was not supported in Python 3.8 and later. 35 | You can use `Anaconda `__ 36 | (preferred) or `pyenv `__ to install and 37 | manage Python environments. 38 | 39 | If you’re new to Python and need an IDE to start developing, we 40 | recommend using `Spyder `__ which 41 | can easily be installed with Anaconda. 42 | 43 | The ``acumos`` package can be installed with pip: 44 | 45 | .. code:: bash 46 | 47 | pip install acumos 48 | 49 | 50 | Protocol Buffers 51 | ---------------- 52 | 53 | The ``acumos`` package uses protocol buffers and **assumes you have 54 | the protobuf compiler** ``protoc`` **installed**. Please visit the `protobuf 55 | repository `__ 56 | and install the appropriate ``protoc`` for your operating system. 57 | Installation is as easy as downloading a binary release and adding it to 58 | your system ``$PATH``. This is a temporary requirement that will be 59 | removed in a future version of ``acumos``. 60 | 61 | **Anaconda Users**: You can easily install ``protoc`` from `an Anaconda 62 | package `__ via: 63 | 64 | .. code:: bash 65 | 66 | conda install -c anaconda libprotobuf 67 | 68 | 69 | .. |Build Status| image:: https://jenkins.acumos.org/buildStatus/icon?job=acumos-python-client-tox-verify-master 70 | :target: https://jenkins.acumos.org/job/acumos-python-client-tox-verify-master/ 71 | -------------------------------------------------------------------------------- /releases/pypi-0.9.1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.6' 4 | version: 0.9.1 5 | log_dir: acumos-python-client-pypi-merge-master/7 6 | -------------------------------------------------------------------------------- /releases/pypi-0.9.2.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.6' 4 | version: 0.9.2 5 | log_dir: acumos-python-client-pypi-merge-master/10 6 | -------------------------------------------------------------------------------- /releases/pypi-0.9.3.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.6' 4 | version: 0.9.3 5 | log_dir: acumos-python-client-pypi-merge-master/11 6 | -------------------------------------------------------------------------------- /releases/pypi-0.9.4.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.6' 4 | version: 0.9.4 5 | log_dir: acumos-python-client-pypi-merge-master/14 6 | -------------------------------------------------------------------------------- /releases/pypi-0.9.7.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.7' 4 | version: 0.9.7 5 | log_dir: acumos-python-client-pypi-merge-master/23 -------------------------------------------------------------------------------- /releases/pypi-0.9.8.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.9' 4 | version: 0.9.8 5 | log_dir: acumos-python-client-pypi-merge-master/26 -------------------------------------------------------------------------------- /releases/pypi-0.9.9.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.9' 4 | version: 0.9.9 5 | log_dir: acumos-python-client-pypi-merge-master/27 6 | -------------------------------------------------------------------------------- /releases/pypi-1.0.0.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.9' 4 | version: 1.0.0 5 | log_dir: acumos-python-client-pypi-merge-master/28 6 | -------------------------------------------------------------------------------- /releases/pypi-1.0.1.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pypi_project: acumos 3 | python_version: '3.9' 4 | version: 1.0.1 5 | log_dir: acumos-python-client-pypi-merge-master/29 6 | -------------------------------------------------------------------------------- /releases/release-0.9.0.yaml: -------------------------------------------------------------------------------- 1 | bution_type: maven 2 | version: 0.9.0 3 | project: acumos-python-client 4 | log_dir: acumos-python-client-pypi-merge-master/2 5 | --------------------------------------------------------------------------------