├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .gitmodules ├── .travis.yml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── rekcurd ├── __init__.py ├── _project.py ├── _version.py ├── console_scripts │ ├── __init__.py │ ├── errors.py │ └── startapp_handler.py ├── core │ ├── __init__.py │ ├── rekcurd_dashboard_servicer.py │ ├── rekcurd_worker.py │ └── rekcurd_worker_servicer.py ├── data_servers │ ├── __init__.py │ ├── aws_s3_handler.py │ ├── ceph_handler.py │ ├── data_handler.py │ ├── gcs_handler.py │ └── local_handler.py ├── logger │ ├── __init__.py │ ├── logger_fluent.py │ ├── logger_interface.py │ └── logger_jsonlogger.py ├── protobuf ├── template │ ├── app.py-tpl │ ├── requirements.txt-tpl │ ├── settings.yml-tpl │ └── start.sh-tpl └── utils │ ├── __init__.py │ └── rekcurd_config.py ├── requirements.txt ├── scripts └── release.sh ├── setup.py ├── test-requirements.txt ├── test ├── __init__.py ├── core │ ├── __init__.py │ ├── test_dashboard_servicer.py │ ├── test_rekcurd_worker.py │ └── test_worker_servicer.py ├── data_servers │ ├── __init__.py │ ├── test_aws_s3_handler.py │ ├── test_ceph_handler.py │ ├── test_data_server.py │ ├── test_gcs_handler.py │ └── test_local_handler.py ├── logger │ ├── __init__.py │ ├── test_logger_fluent.py │ └── test_logger_jsonlogger.py ├── test-settings.yml ├── test_app.py └── utils │ ├── __init__.py │ └── test_rekcurd_config.py └── tox.ini /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Motivation 2 | 3 | write the motivation 4 | 5 | ## Goal 6 | - write the goal 7 | 8 | write the description 9 | 10 | ## Implementation 11 | - feature 12 | - feature 13 | - resource 14 | - testing 15 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What is this PR for? 2 | 3 | write a reason 4 | 5 | ## This PR includes 6 | 7 | - item 8 | - item 9 | - item 10 | 11 | ## What type of PR is it? 12 | 13 | Feature/Bugfix/.... 14 | 15 | ## What is the issue? 16 | 17 | N/A 18 | 19 | ## How should this be tested? 20 | 21 | write a method and a sample of command 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # PyCharm 104 | .idea/ 105 | 106 | # Rekcurd 107 | model/ 108 | rekcurd-eval/ 109 | 110 | # sqlite 111 | *.sqlite3 112 | 113 | # Mac OS temporary file 114 | .DS_Store -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "rekcurd/grpc"] 2 | path = rekcurd/grpc 3 | url = https://github.com/rekcurd/grpc-proto.git 4 | branch = master 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: xenial 3 | sudo: true 4 | services: 5 | - docker 6 | matrix: 7 | include: 8 | - python: 3.6 9 | env: TOXENV=py36 10 | - python: 3.6 11 | env: TOXENV=coverage,codecov 12 | - python: 3.7 13 | env: TOXENV=py37 14 | install: 15 | - pip install tox 16 | before_install: 17 | - export BOTO_CONFIG=/dev/null 18 | script: 19 | - tox 20 | notifications: 21 | email: false 22 | slack: 23 | secure: qXXJJKQCThatS5lv1r0nBaHSUsgDgLcLnGodcr6se/Q4SEnoKo60yHYctWKqS67x0OwKX7JpCgdK1YQ5r6uZdzUy8xQyLCD+qGe1XZK0ndJ1XuV5jHvwnzK+ODVK71win75NmGc4g2cblfOYO83Oz9gFDOOLThNXiw2yirKyw9tPmClLK7OU1VwuH6vDywXQHG6Ii5ub5MoOxgwQ3P2v0dothYrtnoDVZrUMUNact+ymYBzcWQcrwSLCStXTgwQ9sRAj3y3wHDkGy/hXMHepM0nNn9A5OgvqRgb2jG3LUjtRZ9vxvHYWUR/c+DgVNp0zpH59OlYv53L/cN4KxcQL3vObxf8O3hd/VzNq9PmPn104epfYF0UokgrtGEmNjTeNRMzwCKAYul4wZsYZ40UhHHBomMfO0ZN3ftx3ofRTOHxAJrdwDP6gZCdDFFWeuKcGBhoU7GLHtYXVOOmuW+/XtTDfpY3t9HTfdBfNusT0jaseUVPwmO+iOXXBRHsg/sMuLl48jCdkLPu5qI6t6EOuRYddk9PAtJ1uTtk4n4hvv3mgktDFfIVhOjuhTNcCDInssMJaX2kY+7Iqw37FpQw+E4nJ+gvEeNRQH04D5VoZfhKZeMYNj2Nvr312pCTVBfjbfc5Shxpqmy6XCFnwukJaPmAeNr0vh7JokuLaNneVvEI= 24 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [v1.0.1](https://github.com/rekcurd/rekcurd-python/tree/v1.0.1) 4 | 5 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v1.0.0...v1.0.1) 6 | 7 | **Closed issues:** 8 | 9 | - Add storage option to load model \(local / s3 / gcs / PV\) [\#30](https://github.com/rekcurd/rekcurd-python/issues/30) 10 | 11 | **Merged pull requests:** 12 | 13 | - Specify `long\_description\_content\_type` [\#49](https://github.com/rekcurd/rekcurd-python/pull/49) ([keigohtr](https://github.com/keigohtr)) 14 | - Add GCS support [\#48](https://github.com/rekcurd/rekcurd-python/pull/48) ([keigohtr](https://github.com/keigohtr)) 15 | 16 | ## [v1.0.0](https://github.com/rekcurd/rekcurd-python/tree/v1.0.0) (2019-04-26) 17 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.5...v1.0.0) 18 | 19 | **Implemented enhancements:** 20 | 21 | - Large Evaluation data may cause OOM error [\#25](https://github.com/rekcurd/rekcurd-python/issues/25) 22 | - Pipnize the packages [\#2](https://github.com/rekcurd/rekcurd-python/issues/2) 23 | 24 | **Closed issues:** 25 | 26 | - Delete Evaluation file [\#29](https://github.com/rekcurd/rekcurd-python/issues/29) 27 | 28 | **Merged pull requests:** 29 | 30 | - Update documents [\#47](https://github.com/rekcurd/rekcurd-python/pull/47) ([keigohtr](https://github.com/keigohtr)) 31 | - quit saving evaluate result as file [\#46](https://github.com/rekcurd/rekcurd-python/pull/46) ([yuki-mt](https://github.com/yuki-mt)) 32 | - prevent OOM due to evaluation data [\#44](https://github.com/rekcurd/rekcurd-python/pull/44) ([yuki-mt](https://github.com/yuki-mt)) 33 | - Pass object to update `predictor` [\#43](https://github.com/rekcurd/rekcurd-python/pull/43) ([keigohtr](https://github.com/keigohtr)) 34 | - Output error log [\#42](https://github.com/rekcurd/rekcurd-python/pull/42) ([keigohtr](https://github.com/keigohtr)) 35 | - Add `service\_insecure\_host` option [\#41](https://github.com/rekcurd/rekcurd-python/pull/41) ([keigohtr](https://github.com/keigohtr)) 36 | - Remove sqlalchemy library [\#40](https://github.com/rekcurd/rekcurd-python/pull/40) ([keigohtr](https://github.com/keigohtr)) 37 | - Use `SERVICE\_ID` instead `SERVICE\_NAME` [\#39](https://github.com/rekcurd/rekcurd-python/pull/39) ([keigohtr](https://github.com/keigohtr)) 38 | - Add AWS S3 data server [\#38](https://github.com/rekcurd/rekcurd-python/pull/38) ([keigohtr](https://github.com/keigohtr)) 39 | - Add template generation script [\#37](https://github.com/rekcurd/rekcurd-python/pull/37) ([keigohtr](https://github.com/keigohtr)) 40 | - Add dataserver, remove models [\#36](https://github.com/rekcurd/rekcurd-python/pull/36) ([keigohtr](https://github.com/keigohtr)) 41 | - add label to metrics [\#34](https://github.com/rekcurd/rekcurd-python/pull/34) ([yuki-mt](https://github.com/yuki-mt)) 42 | - fix type hinting [\#33](https://github.com/rekcurd/rekcurd-python/pull/33) ([yuki-mt](https://github.com/yuki-mt)) 43 | - Rename from `drucker` to `rekcurd` [\#32](https://github.com/rekcurd/rekcurd-python/pull/32) ([keigohtr](https://github.com/keigohtr)) 44 | - Rename repository link [\#31](https://github.com/rekcurd/rekcurd-python/pull/31) ([keigohtr](https://github.com/keigohtr)) 45 | 46 | ## [v0.4.5](https://github.com/rekcurd/rekcurd-python/tree/v0.4.5) (2019-01-30) 47 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.4...v0.4.5) 48 | 49 | **Merged pull requests:** 50 | 51 | - Unittest py37 support [\#28](https://github.com/rekcurd/rekcurd-python/pull/28) ([keigohtr](https://github.com/keigohtr)) 52 | 53 | ## [v0.4.4](https://github.com/rekcurd/rekcurd-python/tree/v0.4.4) (2019-01-15) 54 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.3...v0.4.4) 55 | 56 | **Merged pull requests:** 57 | 58 | - Add slack notification [\#26](https://github.com/rekcurd/rekcurd-python/pull/26) ([keigohtr](https://github.com/keigohtr)) 59 | - implement method for EvaluationResult protocol [\#24](https://github.com/rekcurd/rekcurd-python/pull/24) ([yuki-mt](https://github.com/yuki-mt)) 60 | - Close `open\(file\)` [\#23](https://github.com/rekcurd/rekcurd-python/pull/23) ([keigohtr](https://github.com/keigohtr)) 61 | 62 | ## [v0.4.3](https://github.com/rekcurd/rekcurd-python/tree/v0.4.3) (2018-12-26) 63 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.2...v0.4.3) 64 | 65 | **Merged pull requests:** 66 | 67 | - Update README.md [\#22](https://github.com/rekcurd/rekcurd-python/pull/22) ([keigohtr](https://github.com/keigohtr)) 68 | - \[Hotfix\] fix ServiceEnvType if statement and make check strict [\#21](https://github.com/rekcurd/rekcurd-python/pull/21) ([yuki-mt](https://github.com/yuki-mt)) 69 | - separate evaluate and upload test data [\#20](https://github.com/rekcurd/rekcurd-python/pull/20) ([yuki-mt](https://github.com/yuki-mt)) 70 | - Istio support [\#19](https://github.com/rekcurd/rekcurd-python/pull/19) ([keigohtr](https://github.com/keigohtr)) 71 | 72 | ## [v0.4.2](https://github.com/rekcurd/rekcurd-python/tree/v0.4.2) (2018-11-28) 73 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.1...v0.4.2) 74 | 75 | **Closed issues:** 76 | 77 | - Need file path check for saving ML model [\#16](https://github.com/rekcurd/rekcurd-python/issues/16) 78 | 79 | **Merged pull requests:** 80 | 81 | - Check if invalid filepath specified then raise Exception [\#17](https://github.com/rekcurd/rekcurd-python/pull/17) ([keigohtr](https://github.com/keigohtr)) 82 | 83 | ## [v0.4.1](https://github.com/rekcurd/rekcurd-python/tree/v0.4.1) (2018-11-20) 84 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.4.0...v0.4.1) 85 | 86 | **Merged pull requests:** 87 | 88 | - Release prepare/v0.4.1 [\#15](https://github.com/rekcurd/rekcurd-python/pull/15) ([keigohtr](https://github.com/keigohtr)) 89 | - \[Hotfix\] Transform to string when `yaml.load` is boolean value [\#14](https://github.com/rekcurd/rekcurd-python/pull/14) ([keigohtr](https://github.com/keigohtr)) 90 | - \[Hotfix\] boolean checker [\#12](https://github.com/rekcurd/rekcurd-python/pull/12) ([keigohtr](https://github.com/keigohtr)) 91 | - evaluate model [\#11](https://github.com/rekcurd/rekcurd-python/pull/11) ([yuki-mt](https://github.com/yuki-mt)) 92 | - \[Hotfix\] Fix invalid variable [\#10](https://github.com/rekcurd/rekcurd-python/pull/10) ([keigohtr](https://github.com/keigohtr)) 93 | 94 | ## [v0.4.0](https://github.com/rekcurd/rekcurd-python/tree/v0.4.0) (2018-11-07) 95 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.3.4...v0.4.0) 96 | 97 | **Merged pull requests:** 98 | 99 | - Add badge [\#9](https://github.com/rekcurd/rekcurd-python/pull/9) ([keigohtr](https://github.com/keigohtr)) 100 | - Create CONTRIBUTING.md [\#8](https://github.com/rekcurd/rekcurd-python/pull/8) ([syleeeee](https://github.com/syleeeee)) 101 | - Create CODE\_OF\_CONDUCT.md [\#7](https://github.com/rekcurd/rekcurd-python/pull/7) ([syleeeee](https://github.com/syleeeee)) 102 | - Pipnize drucker [\#6](https://github.com/rekcurd/rekcurd-python/pull/6) ([keigohtr](https://github.com/keigohtr)) 103 | 104 | ## [v0.3.4](https://github.com/rekcurd/rekcurd-python/tree/v0.3.4) (2018-08-29) 105 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.3.3...v0.3.4) 106 | 107 | ## [v0.3.3](https://github.com/rekcurd/rekcurd-python/tree/v0.3.3) (2018-08-27) 108 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.3.2...v0.3.3) 109 | 110 | **Merged pull requests:** 111 | 112 | - Add sandbox env [\#5](https://github.com/rekcurd/rekcurd-python/pull/5) ([keigohtr](https://github.com/keigohtr)) 113 | - Refactor `sys.path.append` related code [\#3](https://github.com/rekcurd/rekcurd-python/pull/3) ([keigohtr](https://github.com/keigohtr)) 114 | 115 | ## [v0.3.2](https://github.com/rekcurd/rekcurd-python/tree/v0.3.2) (2018-08-15) 116 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.3.1...v0.3.2) 117 | 118 | ## [v0.3.1](https://github.com/rekcurd/rekcurd-python/tree/v0.3.1) (2018-08-09) 119 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.3.0...v0.3.1) 120 | 121 | **Merged pull requests:** 122 | 123 | - \[Hotfix\] Change code generator [\#1](https://github.com/rekcurd/rekcurd-python/pull/1) ([keigohtr](https://github.com/keigohtr)) 124 | 125 | ## [v0.3.0](https://github.com/rekcurd/rekcurd-python/tree/v0.3.0) (2018-07-18) 126 | [Full Changelog](https://github.com/rekcurd/rekcurd-python/compare/v0.2.0...v0.3.0) 127 | 128 | ## [v0.2.0](https://github.com/rekcurd/rekcurd-python/tree/v0.2.0) (2018-07-17) 129 | 130 | 131 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at dl_oss_dev@linecorp.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How to contribute to Rekcurd project 2 | 3 | First of all, thank you so much for taking your time to contribute! Rekcurd is not very different from any other open 4 | source projects you are aware of. It will be amazing if you could help us by doing any of the following: 5 | 6 | - File an issue in [the issue tracker](https://github.com/rekcurd/rekcurd-python/issues) to report bugs and propose new features and 7 | improvements. 8 | - Ask a question by creating a new issue in [the issue tracker](https://github.com/rekcurd/rekcurd-python/issues). 9 | - Browse [the list of previously asked questions](https://github.com/rekcurd/rekcurd-python/issues?q=label%3Aquestion). 10 | - Contribute your work by sending [a pull request](https://github.com/rekcurd/rekcurd-python/pulls). 11 | 12 | ### Contributor license agreement 13 | 14 | When you are sending a pull request and it's a non-trivial change beyond fixing typos, please sign 15 | [the ICLA (individual contributor license agreement)](https://cla-assistant.io/rekcurd/rekcurd-python). Please 16 | [contact us](dl_oss_dev@linecorp.com) if you need the CCLA (corporate contributor license agreement). 17 | 18 | ### Code of conduct 19 | 20 | We expect contributors to follow [our code of conduct](https://github.com/rekcurd/rekcurd-python/blob/master/CODE_OF_CONDUCT.md). 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include rekcurd/template/* 2 | include requirements.txt 3 | include README.md 4 | include LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rekcurd 2 | 3 | [![Build Status](https://travis-ci.com/rekcurd/rekcurd-python.svg?branch=master)](https://travis-ci.com/rekcurd/rekcurd-python) 4 | [![PyPI version](https://badge.fury.io/py/rekcurd.svg)](https://badge.fury.io/py/rekcurd) 5 | [![codecov](https://codecov.io/gh/rekcurd/rekcurd-python/branch/master/graph/badge.svg)](https://codecov.io/gh/rekcurd/rekcurd-python "Non-generated packages only") 6 | [![pypi supported versions](https://img.shields.io/pypi/pyversions/rekcurd.svg)](https://pypi.python.org/pypi/rekcurd) 7 | 8 | Rekcurd is the Project for serving ML module. This is a gRPC micro-framework and it can be used like [Django](https://docs.djangoproject.com/) and [Flask](http://flask.pocoo.org/). 9 | 10 | 11 | ## Parent Project 12 | https://github.com/rekcurd/community 13 | 14 | 15 | ## Components 16 | - [Rekcurd](https://github.com/rekcurd/rekcurd-python): Project for serving ML module. 17 | - [Rekcurd-dashboard](https://github.com/rekcurd/dashboard): Project for managing ML model and deploying ML module. 18 | - [Rekcurd-client](https://github.com/rekcurd/python-client): Project for integrating ML module. 19 | 20 | 21 | ## Installation 22 | From source: 23 | 24 | ```bash 25 | $ git clone --recursive https://github.com/rekcurd/rekcurd-python.git 26 | $ cd rekcurd-python 27 | $ pip install -e . 28 | ``` 29 | 30 | From [PyPi](https://pypi.org/project/rekcurd/) directly: 31 | 32 | ```bash 33 | $ pip install rekcurd 34 | ``` 35 | 36 | ## How to use 37 | Example is available [here](https://github.com/rekcurd/rekcurd-example/tree/master/python/sklearn-digits). You can generate Rekcurd template and implement necessary methods. 38 | 39 | ```bash 40 | $ rekcurd startapp {Your application name} 41 | $ cd {Your application name} 42 | $ vi app.py 43 | $ python app.py 44 | ``` 45 | 46 | 47 | ## Unittest 48 | ``` 49 | $ python -m unittest 50 | ``` 51 | 52 | 53 | ## Kubernetes support 54 | Rekcurd can be run on Kubernetes. See [community repository](https://github.com/rekcurd/community). 55 | 56 | 57 | ## Type definition 58 | ### `PredictLabel` type 59 | *V* is the length of feature vector. 60 | 61 | |Field |Type |Description | 62 | |:---|:---|:---| 63 | |input
(required) |One of below
- string
- bytes
- string[*V*]
- int[*V*]
- double[*V*] |Input data for inference.
- "Nice weather." for a sentiment analysis.
- PNG file for an image transformation.
- ["a", "b"] for a text summarization.
- [1, 2] for a sales forcast.
- [0.9, 0.1] for mnist data. | 64 | |option |string| Option field. Must be json format. | 65 | 66 | The "option" field needs to be a json format. Any style is Ok but we have some reserved fields below. 67 | 68 | |Field |Type |Description | 69 | |:---|:---|:---| 70 | |suppress_log_input |bool |True: NOT print the input and output to the log message.
False (default): Print the input and outpu to the log message. | 71 | |YOUR KEY |any |YOUR VALUE | 72 | 73 | ### `PredictResult` type 74 | *M* is the number of classes. If your algorithm is a binary classifier, you set *M* to 1. If your algorithm is a multi-class classifier, you set *M* to the number of classes. 75 | 76 | |Field |Type |Description | 77 | |:---|:---|:---| 78 | |label
(required) |One of below
-string
-bytes
-string[*M*]
-int[*M*]
-double[*M*] |Result of inference.
-"positive" for a sentiment analysis.
-PNG file for an image transformation.
-["a", "b"] for a multi-class classification.
-[1, 2] for a multi-class classification.
-[0.9, 0.1] for a multi-class classification. | 79 | |score
(required) |One of below
-double
-double[*M*] |Score of result.
-0.98 for a binary classification.
-[0.9, 0.1] for a multi-class classification. | 80 | |option |string |Option field. Must be json format. | 81 | 82 | ### `EvaluateResult` type 83 | `EvaluateResult` is the evaluation score. *N* is the number of evaluation data. *M* is the number of classes. If your algorithm is a binary classifier, you set *M* to 1. If your algorithm is a multi-class classifier, you set *M* to the number of classes. 84 | 85 | |Field |Type |Description | 86 | |:---|:---|:---| 87 | |num
(required)|int |Number of evaluation data. | 88 | |accuracy
(required) |double |Accuracy. | 89 | |precision
(required) |double[*M*] |Precision. | 90 | |recall
(required) |double[*M*] |Recall. | 91 | |fvalue
(required) |double[*M*] |F1 value. | 92 | 93 | ### `EvaluateDetail` type 94 | `EvaluateDetail` is the details of evaluation result. 95 | 96 | |Field |Type |Description | 97 | |:---|:---|:---| 98 | |result
(required) |PredictResult |Prediction result. | 99 | |is_correct
(required) |bool |Correct or not. | 100 | -------------------------------------------------------------------------------- /rekcurd/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Rekcurd Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from rekcurd import _project 16 | from rekcurd import _version 17 | 18 | __project__ = _project.__project__ 19 | __version__ = _version.__version__ 20 | 21 | from rekcurd.core import ( 22 | Rekcurd, RekcurdPack, RekcurdInput, RekcurdOutput, 23 | RekcurdWorkerServicer, RekcurdDashboardServicer 24 | ) 25 | -------------------------------------------------------------------------------- /rekcurd/_project.py: -------------------------------------------------------------------------------- 1 | __project__ = 'rekcurd' 2 | -------------------------------------------------------------------------------- /rekcurd/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.0.2a0' 2 | -------------------------------------------------------------------------------- /rekcurd/console_scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | import argparse 5 | 6 | from .startapp_handler import startapp_handler 7 | from rekcurd import _version 8 | 9 | 10 | def create_parser(): 11 | parser = argparse.ArgumentParser(description='rekcurd command') 12 | parser.add_argument( 13 | '--version', '-v', action='version', version=_version.__version__) 14 | subparsers = parser.add_subparsers() 15 | 16 | # startapp 17 | parser_startapp = subparsers.add_parser( 18 | 'startapp', help='see `rekcurd startapp -h`') 19 | parser_startapp.add_argument( 20 | 'name', help='Name of the application or project.') 21 | parser_startapp.add_argument( 22 | '--dir', required=False, help='Optional destination directory', default='./') 23 | parser_startapp.set_defaults(handler=startapp_handler) 24 | 25 | return parser 26 | 27 | 28 | def main() -> None: 29 | parser = create_parser() 30 | args = parser.parse_args() 31 | 32 | if hasattr(args, 'handler'): 33 | args.handler(args) 34 | else: 35 | parser.print_help() 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /rekcurd/console_scripts/errors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class CommandError(Exception): 5 | """ 6 | Exception class indicating a problem while executing a console 7 | scripts. 8 | """ 9 | pass 10 | -------------------------------------------------------------------------------- /rekcurd/console_scripts/startapp_handler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | import os 5 | import rekcurd 6 | import shutil 7 | import stat 8 | 9 | from pathlib import Path 10 | 11 | from .errors import CommandError 12 | 13 | 14 | def _make_writeable(filename): 15 | """ 16 | Make sure that the file is writeable. 17 | Useful if our source is read-only. 18 | """ 19 | if not os.access(filename, os.W_OK): 20 | st = os.stat(filename) 21 | new_permissions = stat.S_IMODE(st.st_mode) | stat.S_IWUSR 22 | os.chmod(filename, new_permissions) 23 | 24 | 25 | def startapp_handler(args): 26 | if Path(args.name).name != args.name: 27 | raise TypeError("Invalid project name: "+args.name) 28 | destination = Path(args.dir, args.name) 29 | if destination.exists(): 30 | raise CommandError("'{}' already exists".format(destination.absolute())) 31 | destination.mkdir(parents=True, exist_ok=False) 32 | 33 | base_name = "RekcurdAppTemplate" 34 | template_suffix = "-tpl" 35 | template_dir = Path(rekcurd.__path__[0], 'template') 36 | 37 | for root, dirs, files in os.walk(template_dir): 38 | for dirname in dirs[:]: 39 | if dirname.startswith('.') or dirname == '__pycache__': 40 | dirs.remove(dirname) 41 | 42 | for filename in files: 43 | if filename.endswith(('.pyo', '.pyc', '.py.class')): 44 | # Ignore some files as they cause various breakages. 45 | continue 46 | old_path = Path(root, filename) 47 | new_path = Path(destination, filename) 48 | if str(new_path).endswith(template_suffix): 49 | new_path = Path(str(new_path)[:-len(template_suffix)]) 50 | 51 | if new_path.exists(): 52 | raise CommandError("{} already exists, overlaying a " 53 | "project into an existing directory" 54 | "won't replace conflicting files".format(new_path)) 55 | 56 | with old_path.open(mode='r', encoding='utf-8') as template_file: 57 | content = template_file.read().replace(base_name, args.name) 58 | with new_path.open(mode='w', encoding='utf-8') as new_file: 59 | new_file.write(content) 60 | 61 | try: 62 | shutil.copymode(str(old_path), str(new_path)) 63 | _make_writeable(str(new_path)) 64 | except OSError: 65 | print( 66 | "Notice: Couldn't set permission bits on {}. You're " 67 | "probably using an uncommon filesystem setup. No " 68 | "problem.".format(new_path) 69 | ) 70 | -------------------------------------------------------------------------------- /rekcurd/core/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | from .rekcurd_worker import Rekcurd, RekcurdPack 5 | from .rekcurd_worker_servicer import RekcurdInput, RekcurdOutput, RekcurdWorkerServicer 6 | from .rekcurd_dashboard_servicer import RekcurdDashboardServicer 7 | -------------------------------------------------------------------------------- /rekcurd/core/rekcurd_dashboard_servicer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import traceback 6 | 7 | import grpc 8 | import types 9 | import pickle 10 | import sys 11 | 12 | from grpc import ServicerContext 13 | from typing import Iterator, Union, List 14 | 15 | from .rekcurd_worker import RekcurdPack 16 | from rekcurd.protobuf import rekcurd_pb2, rekcurd_pb2_grpc 17 | from rekcurd.utils import PredictInput, PredictLabel, PredictScore 18 | 19 | 20 | def error_handling(error_response): 21 | """ Decorator for handling error 22 | 23 | Apply following processing on Servicer methods 24 | to handle errors. 25 | 26 | - DB transaction decorating for Servicer class. 27 | Confirm to call :func:``db.session.commit`` 28 | on success operation and :func:``db.session.rollback`` 29 | on fail. 30 | - Error setup for gRPC errors 31 | - Call :func:``on_error`` method (if defined) in the class 32 | to postprocess something on error 33 | 34 | Parameters 35 | ---------- 36 | error_response 37 | gRPC response instance on error 38 | 39 | """ 40 | 41 | def _wrapper_maker(func): 42 | def _wrapper(*args, **kwargs): 43 | try: 44 | return func(*args, **kwargs) 45 | except Exception as error: 46 | # gRPC 47 | context = args[2] 48 | context.set_code(grpc.StatusCode.UNKNOWN) 49 | context.set_details(str(error)) 50 | 51 | servicer = args[0] 52 | if hasattr(servicer, 'on_error'): 53 | assert isinstance(servicer.on_error, types.MethodType), \ 54 | 'You must define on_error as method' 55 | servicer.on_error(error) 56 | return error_response 57 | 58 | return _wrapper 59 | 60 | return _wrapper_maker 61 | 62 | 63 | class RekcurdDashboardServicer(rekcurd_pb2_grpc.RekcurdDashboardServicer): 64 | """ gRPC servicer to manage environment 65 | 66 | - Applications 67 | Machine leagning applications 68 | - Services 69 | Unit to deploy machine learning models 70 | (Corresponding to Service of K8S) 71 | - Models 72 | Machine learning model 73 | """ 74 | 75 | CHUNK_SIZE = 100 76 | BYTE_LIMIT = 4190000 77 | 78 | def __init__(self, rekcurd_pack: RekcurdPack): 79 | self.rekcurd_pack = rekcurd_pack 80 | self.logger = rekcurd_pack.app.system_logger 81 | 82 | def on_error(self, error: Exception): 83 | """ Postprocessing on error 84 | 85 | For detail, see :func:``on_error`` 86 | 87 | Parameters 88 | ---------- 89 | error : Exception 90 | Error to be handled 91 | """ 92 | self.logger.error(str(error)) 93 | self.logger.error(traceback.format_exc()) 94 | 95 | def ServiceInfo(self, 96 | request: rekcurd_pb2.ServiceInfoRequest, 97 | context: ServicerContext 98 | ) -> rekcurd_pb2.ServiceInfoResponse: 99 | """ Get service info. 100 | 101 | :param request: 102 | :param context: 103 | :return: 104 | """ 105 | self.logger.info("Run ServiceInfo.") 106 | return rekcurd_pb2.ServiceInfoResponse( 107 | application_name=self.rekcurd_pack.app.config.APPLICATION_NAME, 108 | service_name=self.rekcurd_pack.app.config.SERVICE_ID, 109 | service_level=self.rekcurd_pack.app.config.SERVICE_LEVEL) 110 | 111 | @error_handling(rekcurd_pb2.ModelResponse(status=0, message='Error: Uploading model file.')) 112 | def UploadModel(self, 113 | request_iterator: Iterator[rekcurd_pb2.UploadModelRequest], 114 | context: ServicerContext 115 | ) -> rekcurd_pb2.ModelResponse: 116 | """ Upload your latest ML model. 117 | 118 | :param request_iterator: 119 | :param context: 120 | :return: 121 | """ 122 | self.logger.info("Run UploadModel.") 123 | self.rekcurd_pack.app.data_server.upload_model(request_iterator) 124 | return rekcurd_pb2.ModelResponse(status=1, 125 | message='Success: Uploading model file.') 126 | 127 | @error_handling(rekcurd_pb2.ModelResponse(status=0, message='Error: Switching model file.')) 128 | def SwitchModel(self, 129 | request: rekcurd_pb2.SwitchModelRequest, 130 | context: ServicerContext 131 | ) -> rekcurd_pb2.ModelResponse: 132 | """ Switch your ML model. 133 | 134 | :param request: 135 | :param context: 136 | :return: 137 | """ 138 | self.logger.info("Run SwitchModel.") 139 | filepath = request.path 140 | local_filepath = self.rekcurd_pack.app.data_server.switch_model(filepath) 141 | self.rekcurd_pack.predictor = self.rekcurd_pack.app.load_model(local_filepath) 142 | return rekcurd_pb2.ModelResponse(status=1, 143 | message='Success: Switching model file.') 144 | 145 | @error_handling(rekcurd_pb2.EvaluateModelResponse()) 146 | def EvaluateModel(self, 147 | request_iterator: Iterator[rekcurd_pb2.EvaluateModelRequest], 148 | context: ServicerContext 149 | ) -> rekcurd_pb2.EvaluateModelResponse: 150 | """ Evaluate your ML model and save result. 151 | 152 | :param request_iterator: 153 | :param context: 154 | :return: 155 | """ 156 | self.logger.info("Run EvaluateModel.") 157 | first_req = next(request_iterator) 158 | data_path = first_req.data_path 159 | result_path = first_req.result_path 160 | 161 | local_data_path = self.rekcurd_pack.app.data_server.get_evaluation_data_path(data_path) 162 | evaluate_result_gen = self.rekcurd_pack.app.evaluate(self.rekcurd_pack.predictor, local_data_path) 163 | result = self.rekcurd_pack.app.data_server.upload_evaluation_result(evaluate_result_gen, result_path) 164 | label_ios = [self.get_io_by_type(l) for l in result.label] 165 | metrics = rekcurd_pb2.EvaluationMetrics(num=result.num, 166 | accuracy=result.accuracy, 167 | precision=result.precision, 168 | recall=result.recall, 169 | fvalue=result.fvalue, 170 | option=result.option, 171 | label=label_ios) 172 | return rekcurd_pb2.EvaluateModelResponse(metrics=metrics) 173 | 174 | @error_handling(rekcurd_pb2.UploadEvaluationDataResponse(status=0, message='Error: Uploading evaluation data.')) 175 | def UploadEvaluationData(self, 176 | request_iterator: Iterator[rekcurd_pb2.UploadEvaluationDataRequest], 177 | context: ServicerContext 178 | ) -> rekcurd_pb2.UploadEvaluationDataResponse: 179 | """ Save evaluation data 180 | 181 | :param request_iterator: 182 | :param context: 183 | :return: 184 | """ 185 | self.logger.info("Run UploadEvaluationData.") 186 | self.rekcurd_pack.app.data_server.upload_evaluation_data(request_iterator) 187 | return rekcurd_pb2.UploadEvaluationDataResponse(status=1, 188 | message='Success: Uploading evaluation data.') 189 | 190 | @error_handling(rekcurd_pb2.EvaluationResultResponse()) 191 | def EvaluationResult(self, 192 | request: rekcurd_pb2.EvaluationResultRequest, 193 | context: ServicerContext 194 | ) -> Iterator[rekcurd_pb2.EvaluationResultResponse]: 195 | """ Return saved evaluation result 196 | """ 197 | self.logger.info("Run EvaluationResult.") 198 | data_path = request.data_path 199 | result_path = request.result_path 200 | local_data_path = self.rekcurd_pack.app.data_server.get_evaluation_data_path(data_path) 201 | local_result_detail_path = self.rekcurd_pack.app.data_server.get_eval_result_detail(result_path) 202 | 203 | # TODO: deprecated. remove metrics from response in the next gRPC spec 204 | metrics = rekcurd_pb2.EvaluationMetrics() 205 | 206 | detail_chunks = [] 207 | detail_chunk = [] 208 | metrics_size = sys.getsizeof(metrics) 209 | with open(local_result_detail_path, 'rb') as detail_file: 210 | def generate_result_detail(): 211 | try: 212 | while True: 213 | yield pickle.load(detail_file) 214 | except EOFError: 215 | pass 216 | for detail in self.rekcurd_pack.app.get_evaluate_detail(local_data_path, generate_result_detail()): 217 | detail_chunk.append(rekcurd_pb2.EvaluationResultResponse.Detail( 218 | input=self.get_io_by_type(detail.input), 219 | label=self.get_io_by_type(detail.label), 220 | output=self.get_io_by_type(detail.result.result.label), 221 | score=self.get_score_by_type(detail.result.result.score), 222 | is_correct=detail.result.is_correct 223 | )) 224 | if len(detail_chunk) == self.CHUNK_SIZE: 225 | if metrics_size + sys.getsizeof(detail_chunk + detail_chunks) < self.BYTE_LIMIT: 226 | detail_chunks.extend(detail_chunk) 227 | else: 228 | yield rekcurd_pb2.EvaluationResultResponse(metrics=metrics, detail=detail_chunks) 229 | detail_chunks = detail_chunk 230 | detail_chunk = [] 231 | 232 | if len(detail_chunks + detail_chunk) > 0: 233 | if metrics_size + sys.getsizeof(detail_chunk + detail_chunks) < self.BYTE_LIMIT: 234 | detail_chunks.extend(detail_chunk) 235 | yield rekcurd_pb2.EvaluationResultResponse(metrics=metrics, detail=detail_chunks) 236 | else: 237 | yield rekcurd_pb2.EvaluationResultResponse(metrics=metrics, detail=detail_chunks) 238 | yield rekcurd_pb2.EvaluationResultResponse(metrics=metrics, detail=detail_chunk) 239 | 240 | def get_io_by_type(self, io: Union[PredictInput, PredictLabel]) -> rekcurd_pb2.IO: 241 | if not isinstance(io, list): 242 | io = [io] 243 | 244 | if len(io) == 0: 245 | return rekcurd_pb2.IO(tensor=rekcurd_pb2.Tensor()) 246 | if isinstance(io[0], str): 247 | return rekcurd_pb2.IO(str=rekcurd_pb2.ArrString(val=io)) 248 | else: 249 | return rekcurd_pb2.IO(tensor=rekcurd_pb2.Tensor(shape=[1], val=io)) 250 | 251 | def get_score_by_type(self, score: PredictScore) -> List[float]: 252 | if isinstance(score, list): 253 | return score 254 | else: 255 | return [score] 256 | -------------------------------------------------------------------------------- /rekcurd/core/rekcurd_worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | from abc import ABCMeta, abstractmethod 6 | from enum import Enum 7 | from typing import Generator 8 | 9 | from rekcurd.utils import RekcurdConfig, PredictInput, PredictResult, EvaluateResult, EvaluateDetail, EvaluateResultDetail 10 | from rekcurd.logger import SystemLoggerInterface, ServiceLoggerInterface, JsonSystemLogger, JsonServiceLogger 11 | from rekcurd.data_servers import DataServer 12 | 13 | 14 | class Rekcurd(metaclass=ABCMeta): 15 | """ 16 | Rekcurd 17 | """ 18 | _system_logger: SystemLoggerInterface = None 19 | _service_logger: ServiceLoggerInterface = None 20 | config: RekcurdConfig = None 21 | data_server: DataServer = None 22 | 23 | @abstractmethod 24 | def load_model(self, filepath: str) -> object: 25 | """ 26 | load_model 27 | :param filepath: ML model file path. str 28 | :return predictor: Your ML predictor object. object 29 | """ 30 | raise NotImplemented() 31 | 32 | @abstractmethod 33 | def predict(self, predictor: object, idata: PredictInput, option: dict = None) -> PredictResult: 34 | """ 35 | predict 36 | :param predictor: Your ML predictor object. object 37 | :param idata: Input data. PredictInput, one of string/bytes/arr[int]/arr[float]/arr[string] 38 | :param option: Miscellaneous. dict 39 | :return result: Result. PredictResult 40 | result.label: Label. One of string/bytes/arr[int]/arr[float]/arr[string] 41 | result.score: Score. One of float/arr[float] 42 | result.option: Miscellaneous. dict 43 | """ 44 | raise NotImplemented() 45 | 46 | @abstractmethod 47 | def evaluate(self, predictor: object, filepath: str) -> Generator[EvaluateResultDetail, None, EvaluateResult]: 48 | """ 49 | evaluate 50 | :param predictor: Your ML predictor object. object 51 | :param filepath: Evaluation data file path. str 52 | :return result: Result. EvaluateResult 53 | result.num: Number of data. int 54 | result.accuracy: Accuracy. float 55 | result.precision: Precision. arr[float] 56 | result.recall: Recall. arr[float] 57 | result.fvalue: F1 value. arr[float] 58 | result.option: Optional metrics. dict[str, float] 59 | :generate detail[]: Detail result of each prediction. List[EvaluateResultDetail] 60 | detail[].result: Prediction result. PredictResult 61 | detail[].is_correct: Prediction result is correct or not. bool 62 | """ 63 | raise NotImplemented() 64 | 65 | @abstractmethod 66 | def get_evaluate_detail(self, filepath: str, details: Generator[EvaluateResultDetail, None, None]) -> Generator[EvaluateDetail, None, None]: 67 | """ 68 | get_evaluate_detail 69 | :param filepath: Evaluation data file path. str 70 | :param details: Detail result of each prediction. Generator[EvaluateResultDetail, None, None] 71 | :return rtn: Return results. Generator[EvaluateDetail, None, None] 72 | rtn.input: Input data. PredictInput, one of string/bytes/arr[int]/arr[float]/arr[string] 73 | rtn.label: Predict label. PredictLabel, one of string/bytes/arr[int]/arr[float]/arr[string] 74 | rtn.result: Predict detail. EvaluateResultDetail 75 | """ 76 | raise NotImplemented() 77 | 78 | @property 79 | def system_logger(self): 80 | return self._system_logger 81 | 82 | @system_logger.setter 83 | def system_logger(self, system_logger: SystemLoggerInterface): 84 | if isinstance(system_logger, SystemLoggerInterface): 85 | self._system_logger = system_logger 86 | else: 87 | raise TypeError("Invalid system_logger type.") 88 | 89 | @property 90 | def service_logger(self): 91 | return self._service_logger 92 | 93 | @service_logger.setter 94 | def service_logger(self, service_logger: ServiceLoggerInterface): 95 | if isinstance(service_logger, ServiceLoggerInterface): 96 | self._service_logger = service_logger 97 | else: 98 | raise TypeError("Invalid service_logger type.") 99 | 100 | def load_config_file(self, config_file: str): 101 | self.config = RekcurdConfig(config_file) 102 | 103 | def run(self, host: str = None, port: int = None, max_workers: int = None, **options): 104 | import grpc 105 | import os 106 | import time 107 | from concurrent import futures 108 | from rekcurd.protobuf import rekcurd_pb2_grpc 109 | from rekcurd import RekcurdDashboardServicer, RekcurdWorkerServicer 110 | 111 | if self.config is None: 112 | self.config = RekcurdConfig() 113 | if host and "service_insecure_host" in options: 114 | options["service_insecure_host"] = host 115 | if port and "service_insecure_port" in options: 116 | options["service_insecure_port"] = port 117 | self.config.set_configurations(**options) 118 | 119 | self.data_server = DataServer(self.config) 120 | if self._system_logger is None: 121 | self._system_logger = JsonSystemLogger(config=self.config) 122 | if self._service_logger is None: 123 | self._service_logger = JsonServiceLogger(config=self.config) 124 | self.system_logger.info("Service start.") 125 | _host = "127.0.0.1" 126 | _port = 5000 127 | _max_workers = 1 128 | host = host or self.config.SERVICE_INSECURE_HOST or _host 129 | port = int(port or self.config.SERVICE_INSECURE_PORT or _port) 130 | max_workers = int(max_workers or _max_workers) 131 | 132 | try: 133 | self.system_logger.info("Download model.") 134 | model_path = self.data_server.get_model_path() 135 | self.system_logger.info("Initialize predictor.") 136 | predictor = self.load_model(model_path) 137 | if predictor is None: 138 | raise Exception("Error: No predictor found. Need your \"Rekcurd\" implementation.") 139 | except Exception as e: 140 | self.system_logger.error(str(e)) 141 | print(str(e)) 142 | return 143 | 144 | rekcurd_pack = RekcurdPack(self, predictor) 145 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers)) 146 | rekcurd_pb2_grpc.add_RekcurdDashboardServicer_to_server( 147 | RekcurdDashboardServicer(rekcurd_pack), server) 148 | rekcurd_pb2_grpc.add_RekcurdWorkerServicer_to_server( 149 | RekcurdWorkerServicer(rekcurd_pack), server) 150 | server.add_insecure_port("{0}:{1}".format(host, port)) 151 | self.system_logger.info("Start rekcurd worker on {0}:{1}".format(host, port)) 152 | server.start() 153 | try: 154 | while os.getenv("REKCURD_UNITTEST", "False").lower() == 'false': 155 | time.sleep(86400) 156 | except KeyboardInterrupt: 157 | self.system_logger.info("Shutdown rekcurd worker.") 158 | finally: 159 | server.stop(0) 160 | 161 | # TODO: DEPRECATED BELOW 162 | __type_input = None 163 | __type_output = None 164 | 165 | def set_type(self, type_input: Enum, type_output: Enum) -> None: 166 | self.__type_input = type_input 167 | self.__type_output = type_output 168 | 169 | def get_type_input(self) -> Enum: 170 | return self.__type_input 171 | 172 | def get_type_output(self) -> Enum: 173 | return self.__type_output 174 | 175 | 176 | class RekcurdPack: 177 | def __init__(self, app: Rekcurd, predictor: object): 178 | self.app = app 179 | self.predictor = predictor 180 | -------------------------------------------------------------------------------- /rekcurd/core/rekcurd_worker_servicer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import json 6 | 7 | from enum import Enum 8 | from grpc import ServicerContext 9 | from typing import Iterator, Union 10 | 11 | from .rekcurd_worker import RekcurdPack 12 | from rekcurd.utils import PredictResult 13 | from rekcurd.protobuf import rekcurd_pb2, rekcurd_pb2_grpc 14 | 15 | 16 | RekcurdInput = Union[ 17 | rekcurd_pb2.StringInput, rekcurd_pb2.BytesInput, 18 | rekcurd_pb2.ArrIntInput, rekcurd_pb2.ArrFloatInput, rekcurd_pb2.ArrStringInput] 19 | RekcurdOutput = Union[ 20 | rekcurd_pb2.StringOutput, rekcurd_pb2.BytesOutput, 21 | rekcurd_pb2.ArrIntOutput, rekcurd_pb2.ArrFloatOutput, rekcurd_pb2.ArrStringOutput] 22 | 23 | 24 | class RekcurdWorkerServicer(rekcurd_pb2_grpc.RekcurdWorkerServicer): 25 | class Type(Enum): 26 | STRING = 1 27 | BYTES = 2 28 | ARRAY_INT = 3 29 | ARRAY_FLOAT = 4 30 | ARRAY_STRING = 5 31 | 32 | def __init__(self, rekcurd_pack: RekcurdPack): 33 | self.rekcurd_pack = rekcurd_pack 34 | self.system_logger = rekcurd_pack.app.system_logger 35 | self.service_logger = rekcurd_pack.app.service_logger 36 | 37 | def Process(self, 38 | request: RekcurdInput, 39 | context: ServicerContext, 40 | response: RekcurdOutput 41 | ) -> RekcurdOutput: 42 | 43 | input = request.input 44 | try: 45 | ioption = json.loads(request.option.val) 46 | except: 47 | ioption = {request.option.val: request.option.val} 48 | 49 | single_output = self.rekcurd_pack.app.get_type_output() in [self.Type.STRING, self.Type.BYTES] 50 | try: 51 | result = self.rekcurd_pack.app.predict(self.rekcurd_pack.predictor, input, ioption) 52 | except Exception as e: 53 | self.system_logger.error(str(e)) 54 | if single_output: 55 | if isinstance(response, rekcurd_pb2.StringOutput): 56 | label = "None" 57 | elif isinstance(response, rekcurd_pb2.BytesOutput): 58 | label = b'None' 59 | else: 60 | label = None 61 | result = PredictResult(label=label, score=0.0, option={}) 62 | else: 63 | if isinstance(response, rekcurd_pb2.ArrStringOutput): 64 | label = ["None"] 65 | elif isinstance(response, rekcurd_pb2.ArrIntOutput): 66 | label = [0] 67 | elif isinstance(response, rekcurd_pb2.ArrFloatOutput): 68 | label = [0.0] 69 | else: 70 | label = None 71 | result = PredictResult(label=label, score=[0.0], option={}) 72 | 73 | try: 74 | if single_output: 75 | response.output = result.label 76 | response.score = result.score 77 | else: 78 | response.output.extend(result.label) 79 | response.score.extend(result.score) 80 | response.option.val = result.option 81 | except Exception as e: 82 | self.system_logger.error(str(e)) 83 | 84 | self.service_logger.emit(request, response, ioption.get('suppress_log_inout', False)) 85 | return response 86 | 87 | def Predict_String_String(self, 88 | request: rekcurd_pb2.StringInput, 89 | context: ServicerContext 90 | ) -> rekcurd_pb2.StringOutput: 91 | response = rekcurd_pb2.StringOutput() 92 | self.rekcurd_pack.app.set_type(self.Type.STRING, self.Type.STRING) 93 | return self.Process(request, context, response) 94 | 95 | def Predict_String_Bytes(self, 96 | request: rekcurd_pb2.StringInput, 97 | context: ServicerContext 98 | ) -> rekcurd_pb2.BytesOutput: 99 | response = rekcurd_pb2.BytesOutput() 100 | self.rekcurd_pack.app.set_type(self.Type.STRING, self.Type.BYTES) 101 | yield self.Process(request, context, response) 102 | 103 | def Predict_String_ArrInt(self, 104 | request: rekcurd_pb2.StringInput, 105 | context: ServicerContext 106 | ) -> rekcurd_pb2.ArrIntOutput: 107 | response = rekcurd_pb2.ArrIntOutput() 108 | self.rekcurd_pack.app.set_type(self.Type.STRING, self.Type.ARRAY_INT) 109 | return self.Process(request, context, response) 110 | 111 | def Predict_String_ArrFloat(self, 112 | request: rekcurd_pb2.StringInput, 113 | context: ServicerContext 114 | ) -> rekcurd_pb2.ArrFloatOutput: 115 | response = rekcurd_pb2.ArrFloatOutput() 116 | self.rekcurd_pack.app.set_type(self.Type.STRING, self.Type.ARRAY_FLOAT) 117 | return self.Process(request, context, response) 118 | 119 | def Predict_String_ArrString(self, 120 | request: rekcurd_pb2.StringInput, 121 | context: ServicerContext 122 | ) -> rekcurd_pb2.ArrStringOutput: 123 | response = rekcurd_pb2.ArrStringOutput() 124 | self.rekcurd_pack.app.set_type(self.Type.STRING, self.Type.ARRAY_STRING) 125 | return self.Process(request, context, response) 126 | 127 | def Predict_Bytes_String(self, 128 | request_iterator: Iterator[rekcurd_pb2.BytesInput], 129 | context: ServicerContext 130 | ) -> rekcurd_pb2.StringOutput: 131 | for request in request_iterator: 132 | response = rekcurd_pb2.StringOutput() 133 | self.rekcurd_pack.app.set_type(self.Type.BYTES, self.Type.STRING) 134 | return self.Process(request, context, response) 135 | 136 | def Predict_Bytes_Bytes(self, 137 | request_iterator: Iterator[rekcurd_pb2.BytesInput], 138 | context: ServicerContext 139 | ) -> rekcurd_pb2.BytesOutput: 140 | for request in request_iterator: 141 | response = rekcurd_pb2.BytesOutput() 142 | self.rekcurd_pack.app.set_type(self.Type.BYTES, self.Type.BYTES) 143 | yield self.Process(request, context, response) 144 | 145 | def Predict_Bytes_ArrInt(self, 146 | request_iterator: Iterator[rekcurd_pb2.BytesInput], 147 | context: ServicerContext 148 | ) -> rekcurd_pb2.ArrIntOutput: 149 | for request in request_iterator: 150 | response = rekcurd_pb2.ArrIntOutput() 151 | self.rekcurd_pack.app.set_type(self.Type.BYTES, self.Type.ARRAY_INT) 152 | return self.Process(request, context, response) 153 | 154 | def Predict_Bytes_ArrFloat(self, 155 | request_iterator: Iterator[rekcurd_pb2.BytesInput], 156 | context: ServicerContext 157 | ) -> rekcurd_pb2.ArrFloatOutput: 158 | for request in request_iterator: 159 | response = rekcurd_pb2.ArrFloatOutput() 160 | self.rekcurd_pack.app.set_type(self.Type.BYTES, self.Type.ARRAY_FLOAT) 161 | return self.Process(request, context, response) 162 | 163 | def Predict_Bytes_ArrString(self, 164 | request_iterator: Iterator[rekcurd_pb2.BytesInput], 165 | context: ServicerContext 166 | ) -> rekcurd_pb2.ArrStringOutput: 167 | for request in request_iterator: 168 | response = rekcurd_pb2.ArrStringOutput() 169 | self.rekcurd_pack.app.set_type(self.Type.BYTES, self.Type.ARRAY_STRING) 170 | return self.Process(request, context, response) 171 | 172 | def Predict_ArrInt_String(self, 173 | request: rekcurd_pb2.ArrIntInput, 174 | context: ServicerContext 175 | ) -> rekcurd_pb2.StringOutput: 176 | response = rekcurd_pb2.StringOutput() 177 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_INT, self.Type.STRING) 178 | return self.Process(request, context, response) 179 | 180 | def Predict_ArrInt_Bytes(self, 181 | request: rekcurd_pb2.ArrIntInput, 182 | context: ServicerContext 183 | ) -> rekcurd_pb2.BytesOutput: 184 | response = rekcurd_pb2.BytesOutput() 185 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_INT, self.Type.BYTES) 186 | yield self.Process(request, context, response) 187 | 188 | def Predict_ArrInt_ArrInt(self, 189 | request: rekcurd_pb2.ArrIntInput, 190 | context: ServicerContext 191 | ) -> rekcurd_pb2.ArrIntOutput: 192 | response = rekcurd_pb2.ArrIntOutput() 193 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_INT, self.Type.ARRAY_INT) 194 | return self.Process(request, context, response) 195 | 196 | def Predict_ArrInt_ArrFloat(self, 197 | request: rekcurd_pb2.ArrIntInput, 198 | context: ServicerContext 199 | ) -> rekcurd_pb2.ArrFloatOutput: 200 | response = rekcurd_pb2.ArrFloatOutput() 201 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_INT, self.Type.ARRAY_FLOAT) 202 | return self.Process(request, context, response) 203 | 204 | def Predict_ArrInt_ArrString(self, 205 | request: rekcurd_pb2.ArrIntInput, 206 | context: ServicerContext 207 | ) -> rekcurd_pb2.ArrStringOutput: 208 | response = rekcurd_pb2.ArrStringOutput() 209 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_INT, self.Type.ARRAY_STRING) 210 | return self.Process(request, context, response) 211 | 212 | def Predict_ArrFloat_String(self, 213 | request: rekcurd_pb2.ArrFloatInput, 214 | context: ServicerContext 215 | ) -> rekcurd_pb2.StringOutput: 216 | response = rekcurd_pb2.StringOutput() 217 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_FLOAT, self.Type.STRING) 218 | return self.Process(request, context, response) 219 | 220 | def Predict_ArrFloat_Bytes(self, 221 | request: rekcurd_pb2.ArrFloatInput, 222 | context: ServicerContext 223 | ) -> rekcurd_pb2.BytesOutput: 224 | response = rekcurd_pb2.BytesOutput() 225 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_FLOAT, self.Type.BYTES) 226 | yield self.Process(request, context, response) 227 | 228 | def Predict_ArrFloat_ArrInt(self, 229 | request: rekcurd_pb2.ArrFloatInput, 230 | context: ServicerContext 231 | ) -> rekcurd_pb2.ArrIntOutput: 232 | response = rekcurd_pb2.ArrIntOutput() 233 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_FLOAT, self.Type.ARRAY_INT) 234 | return self.Process(request, context, response) 235 | 236 | def Predict_ArrFloat_ArrFloat(self, 237 | request: rekcurd_pb2.ArrFloatInput, 238 | context: ServicerContext 239 | ) -> rekcurd_pb2.ArrFloatOutput: 240 | response = rekcurd_pb2.ArrFloatOutput() 241 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_FLOAT, self.Type.ARRAY_FLOAT) 242 | return self.Process(request, context, response) 243 | 244 | def Predict_ArrFloat_ArrString(self, 245 | request: rekcurd_pb2.ArrFloatInput, 246 | context: ServicerContext 247 | ) -> rekcurd_pb2.ArrStringOutput: 248 | response = rekcurd_pb2.ArrStringOutput() 249 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_FLOAT, self.Type.ARRAY_STRING) 250 | return self.Process(request, context, response) 251 | 252 | def Predict_ArrString_String(self, 253 | request: rekcurd_pb2.ArrStringInput, 254 | context: ServicerContext 255 | ) -> rekcurd_pb2.StringOutput: 256 | response = rekcurd_pb2.StringOutput() 257 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_STRING, self.Type.STRING) 258 | return self.Process(request, context, response) 259 | 260 | def Predict_ArrString_Bytes(self, 261 | request: rekcurd_pb2.ArrStringInput, 262 | context: ServicerContext 263 | ) -> rekcurd_pb2.BytesOutput: 264 | response = rekcurd_pb2.BytesOutput() 265 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_STRING, self.Type.BYTES) 266 | yield self.Process(request, context, response) 267 | 268 | def Predict_ArrString_ArrInt(self, 269 | request: rekcurd_pb2.ArrStringInput, 270 | context: ServicerContext 271 | ) -> rekcurd_pb2.ArrIntOutput: 272 | response = rekcurd_pb2.ArrIntOutput() 273 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_STRING, self.Type.ARRAY_INT) 274 | return self.Process(request, context, response) 275 | 276 | def Predict_ArrString_ArrFloat(self, 277 | request: rekcurd_pb2.ArrStringInput, 278 | context: ServicerContext 279 | ) -> rekcurd_pb2.ArrFloatOutput: 280 | response = rekcurd_pb2.ArrFloatOutput() 281 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_STRING, self.Type.ARRAY_FLOAT) 282 | return self.Process(request, context, response) 283 | 284 | def Predict_ArrString_ArrString(self, 285 | request: rekcurd_pb2.ArrStringInput, 286 | context: ServicerContext 287 | ) -> rekcurd_pb2.ArrStringOutput: 288 | response = rekcurd_pb2.ArrStringOutput() 289 | self.rekcurd_pack.app.set_type(self.Type.ARRAY_STRING, self.Type.ARRAY_STRING) 290 | return self.Process(request, context, response) 291 | -------------------------------------------------------------------------------- /rekcurd/data_servers/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import pickle 5 | 6 | from pathlib import Path 7 | from typing import Iterator, Generator 8 | 9 | from rekcurd.protobuf import rekcurd_pb2 10 | from rekcurd.utils import RekcurdConfig, ModelModeEnum, EvaluateResultDetail, EvaluateResult 11 | from .data_handler import convert_to_valid_path 12 | from .local_handler import LocalHandler 13 | from .ceph_handler import CephHandler 14 | from .aws_s3_handler import AwsS3Handler 15 | from .gcs_handler import GcsHandler 16 | 17 | 18 | class DataServer(object): 19 | """DataServer 20 | Return file path. Download files if needed. 21 | """ 22 | 23 | def __init__(self, config: RekcurdConfig): 24 | self.config = config 25 | if config.MODEL_MODE_ENUM == ModelModeEnum.LOCAL: 26 | self._api_handler = LocalHandler(config) 27 | elif config.MODEL_MODE_ENUM == ModelModeEnum.CEPH_S3: 28 | self._api_handler = CephHandler(config) 29 | elif config.MODEL_MODE_ENUM == ModelModeEnum.AWS_S3: 30 | self._api_handler = AwsS3Handler(config) 31 | elif config.MODEL_MODE_ENUM == ModelModeEnum.GCS: 32 | self._api_handler = GcsHandler(config) 33 | else: 34 | raise ValueError("Invalid ModelModeEnum value.") 35 | 36 | def get_model_path(self) -> str: 37 | local_filepath = Path(self._api_handler.LOCAL_MODEL_DIR, self._api_handler.MODEL_FILE_NAME) 38 | if not local_filepath.exists(): 39 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 40 | self._api_handler.download(self.config.MODEL_FILE_PATH, str(local_filepath)) 41 | return str(local_filepath) 42 | 43 | def switch_model(self, filepath: str) -> str: 44 | valid_path = convert_to_valid_path(filepath) 45 | if filepath != str(valid_path): 46 | raise Exception(f'Error: Invalid file path specified -> {filepath}') 47 | 48 | local_filepath = Path(self._api_handler.LOCAL_MODEL_DIR, valid_path.name) 49 | if not local_filepath.exists(): 50 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 51 | self._api_handler.download(filepath, str(local_filepath)) 52 | self._api_handler.MODEL_FILE_NAME = valid_path.name 53 | return str(local_filepath) 54 | 55 | def upload_model(self, request_iterator: Iterator[rekcurd_pb2.UploadModelRequest]) -> str: 56 | first_req = next(request_iterator) 57 | filepath = first_req.path 58 | valid_path = convert_to_valid_path(filepath) 59 | if filepath != str(valid_path): 60 | raise Exception(f'Error: Invalid file path specified -> {filepath}') 61 | 62 | local_filepath = Path(self._api_handler.LOCAL_MODEL_DIR, valid_path.name) 63 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 64 | with local_filepath.open(mode='wb') as f: 65 | f.write(first_req.data) 66 | for request in request_iterator: 67 | f.write(request.data) 68 | del first_req 69 | self._api_handler.upload(filepath, str(local_filepath)) 70 | return str(local_filepath) 71 | 72 | def get_evaluation_data_path(self, filepath: str) -> str: 73 | valid_path = convert_to_valid_path(filepath) 74 | if filepath != str(valid_path): 75 | raise Exception(f'Error: Invalid file path specified -> {filepath}') 76 | 77 | local_filepath = Path(self._api_handler.LOCAL_EVAL_DIR, valid_path.name) 78 | if not local_filepath.exists(): 79 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 80 | self._api_handler.download(filepath, str(local_filepath)) 81 | return str(local_filepath) 82 | 83 | def get_eval_result_detail(self, filepath: str) -> str: 84 | valid_path = convert_to_valid_path(filepath) 85 | if filepath != str(valid_path): 86 | raise Exception(f'Error: Invalid file path specified -> {filepath}') 87 | 88 | local_filepath = Path(self._api_handler.LOCAL_EVAL_DIR, valid_path.name) 89 | if not local_filepath.exists(): 90 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 91 | self._api_handler.download(filepath, str(local_filepath)) 92 | return str(local_filepath) 93 | 94 | def upload_evaluation_data(self, request_iterator: Iterator[rekcurd_pb2.UploadEvaluationDataRequest]) -> str: 95 | first_req = next(request_iterator) 96 | filepath = first_req.data_path 97 | valid_path = convert_to_valid_path(filepath) 98 | if filepath != str(valid_path): 99 | raise Exception(f'Error: Invalid evaluation file path specified -> {filepath}') 100 | 101 | local_filepath = Path(self._api_handler.LOCAL_EVAL_DIR, valid_path.name) 102 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 103 | with local_filepath.open(mode='wb') as f: 104 | f.write(first_req.data) 105 | for request in request_iterator: 106 | f.write(request.data) 107 | del first_req 108 | self._api_handler.upload(filepath, str(local_filepath)) 109 | return str(local_filepath) 110 | 111 | def upload_evaluation_result(self, data_gen: Generator[EvaluateResultDetail, None, EvaluateResult], filepath: str) -> EvaluateResult: 112 | valid_path = convert_to_valid_path(filepath) 113 | if filepath != str(valid_path): 114 | raise Exception(f'Error: Invalid evaluation result file path specified -> {filepath}') 115 | 116 | local_filepath = Path(self._api_handler.LOCAL_EVAL_DIR, valid_path.name) 117 | local_filepath.parent.mkdir(parents=True, exist_ok=True) 118 | 119 | with local_filepath.open(mode='wb') as detail_file: 120 | try: 121 | while True: 122 | pickle.dump(next(data_gen), detail_file) 123 | except StopIteration as e: 124 | evaluate_result = e.value 125 | self._api_handler.upload(filepath, str(local_filepath)) 126 | 127 | return evaluate_result 128 | -------------------------------------------------------------------------------- /rekcurd/data_servers/aws_s3_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import boto3 5 | 6 | from .data_handler import DataHandler 7 | from rekcurd.utils import RekcurdConfig 8 | 9 | 10 | class AwsS3Handler(DataHandler): 11 | """AwsS3Handler 12 | """ 13 | def __init__(self, config: RekcurdConfig): 14 | super(AwsS3Handler, self).__init__(config) 15 | self._resource = boto3.resource( 16 | 's3', 17 | aws_access_key_id=config.AWS_ACCESS_KEY, 18 | aws_secret_access_key=config.AWS_SECRET_KEY, 19 | ) 20 | self._bucket = config.AWS_BUCKET_NAME 21 | 22 | def download(self, remote_filepath: str, local_filepath: str) -> None: 23 | self._resource.Bucket(self._bucket).download_file(remote_filepath, local_filepath) 24 | 25 | def upload(self, remote_filepath: str, local_filepath: str) -> None: 26 | self._resource.Bucket(self._bucket).upload_file(local_filepath, remote_filepath) 27 | -------------------------------------------------------------------------------- /rekcurd/data_servers/ceph_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import boto 5 | import boto.s3.connection 6 | 7 | from .data_handler import DataHandler 8 | from rekcurd.utils import RekcurdConfig 9 | 10 | 11 | class CephHandler(DataHandler): 12 | """CephHandler 13 | """ 14 | def __init__(self, config: RekcurdConfig): 15 | super(CephHandler, self).__init__(config) 16 | self._conn = boto.connect_s3( 17 | aws_access_key_id=config.CEPH_ACCESS_KEY, 18 | aws_secret_access_key=config.CEPH_SECRET_KEY, 19 | host=config.CEPH_HOST, 20 | port=config.CEPH_PORT, 21 | is_secure=config.CEPH_IS_SECURE, 22 | calling_format=boto.s3.connection.OrdinaryCallingFormat()) 23 | self._bucket = config.CEPH_BUCKET_NAME 24 | 25 | def download(self, remote_filepath: str, local_filepath: str) -> None: 26 | bucket = self._conn.get_bucket(self._bucket) 27 | key = bucket.get_key(remote_filepath) 28 | key.get_contents_to_filename(local_filepath) 29 | 30 | def upload(self, remote_filepath: str, local_filepath: str) -> None: 31 | bucket = self._conn.get_bucket(self._bucket) 32 | key = bucket.new_key(remote_filepath) 33 | key.set_contents_from_filename(local_filepath, replace=False) 34 | -------------------------------------------------------------------------------- /rekcurd/data_servers/data_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | from abc import ABCMeta, abstractmethod 5 | from pathlib import Path 6 | 7 | from rekcurd.utils import RekcurdConfig 8 | 9 | 10 | def convert_to_valid_path(filepath: str) -> Path: 11 | """ 12 | Remove a root path prefix "/", and a relative path "." and "..". 13 | :param filepath: 14 | :return: 15 | """ 16 | valid_factors = [factor for factor in filepath.split("/") if factor and factor != ".."] 17 | return Path(*valid_factors) 18 | 19 | 20 | class DataHandler(metaclass=ABCMeta): 21 | """Interface class. 22 | """ 23 | LOCAL_MODEL_DIR: str = None 24 | MODEL_FILE_NAME: str = None 25 | LOCAL_EVAL_DIR: str = None 26 | 27 | def __init__(self, config: RekcurdConfig): 28 | valid_model_path = convert_to_valid_path(config.MODEL_FILE_PATH) 29 | self.LOCAL_MODEL_DIR = str(Path("rekcurd-model", valid_model_path.parent)) 30 | self.MODEL_FILE_NAME = valid_model_path.name 31 | self.LOCAL_EVAL_DIR = "rekcurd-eval" 32 | 33 | @abstractmethod 34 | def download(self, remote_filepath: str, local_filepath: str) -> None: 35 | raise NotImplemented() 36 | 37 | @abstractmethod 38 | def upload(self, remote_filepath: str, local_filepath: str) -> None: 39 | raise NotImplemented() 40 | -------------------------------------------------------------------------------- /rekcurd/data_servers/gcs_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import boto3 5 | 6 | from .data_handler import DataHandler 7 | from rekcurd.utils import RekcurdConfig 8 | 9 | 10 | class GcsHandler(DataHandler): 11 | """GcsHandler 12 | """ 13 | def __init__(self, config: RekcurdConfig): 14 | super(GcsHandler, self).__init__(config) 15 | self._resource = boto3.resource( 16 | 's3', 17 | region_name="auto", 18 | endpoint_url="https://storage.googleapis.com", 19 | aws_access_key_id=config.GCS_ACCESS_KEY, 20 | aws_secret_access_key=config.GCS_SECRET_KEY, 21 | ) 22 | self._bucket = config.GCS_BUCKET_NAME 23 | 24 | def download(self, remote_filepath: str, local_filepath: str) -> None: 25 | self._resource.Bucket(self._bucket).download_file(remote_filepath, local_filepath) 26 | 27 | def upload(self, remote_filepath: str, local_filepath: str) -> None: 28 | self._resource.Bucket(self._bucket).upload_file(local_filepath, remote_filepath) 29 | -------------------------------------------------------------------------------- /rekcurd/data_servers/local_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | from pathlib import Path 5 | 6 | from .data_handler import DataHandler 7 | from rekcurd.utils import RekcurdConfig 8 | 9 | 10 | class LocalHandler(DataHandler): 11 | """LocalHandler 12 | """ 13 | def __init__(self, config: RekcurdConfig): 14 | super(LocalHandler, self).__init__(config) 15 | # Overwrite: No additional local directory is needed. 16 | self.LOCAL_MODEL_DIR = str(Path(config.MODEL_FILE_PATH).parent) 17 | 18 | def download(self, remote_filepath: str, local_filepath: str) -> None: 19 | raise Exception( 20 | "Error: No such a file. Local mode requires to store" 21 | " a ML model file on the directory you specified. " 22 | "-> \"{}\"".format(local_filepath)) 23 | 24 | def upload(self, remote_filepath: str, local_filepath: str) -> None: 25 | pass 26 | -------------------------------------------------------------------------------- /rekcurd/logger/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | from .logger_interface import SystemLoggerInterface, ServiceLoggerInterface 5 | from .logger_jsonlogger import JsonSystemLogger, JsonServiceLogger 6 | from .logger_fluent import FluentSystemLogger, FluentServiceLogger 7 | -------------------------------------------------------------------------------- /rekcurd/logger/logger_fluent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import logging 5 | import sys 6 | import time 7 | from socket import gethostname 8 | 9 | from fluent import handler, sender 10 | 11 | from rekcurd.utils import RekcurdConfig 12 | from .logger_interface import SystemLoggerInterface, ServiceLoggerInterface 13 | 14 | 15 | class FluentSystemLogger(SystemLoggerInterface): 16 | 17 | def __init__(self, 18 | logger_name: str = 'rekcurd.system', 19 | log_level: int = None, 20 | config: RekcurdConfig = RekcurdConfig()) -> None: 21 | """ 22 | Constructor 23 | :param logger_name: 24 | :param log_level: 25 | :param config: RekcurdConfig 26 | """ 27 | self.config = config 28 | log_level = int(log_level or logging.DEBUG if config.DEBUG_MODE else logging.NOTSET) 29 | application_name = config.APPLICATION_NAME 30 | service_level = config.SERVICE_LEVEL 31 | self.log = logging.getLogger(logger_name) 32 | self.log.setLevel(log_level) 33 | self.log.handlers = [] 34 | self.log.addHandler(self.__init_fluent_handler(application_name, service_level, log_level)) 35 | 36 | def __init_fluent_handler(self, application_name: str, service_level: str, log_level: int): 37 | custom_format = { 38 | 'host': gethostname(), 39 | 'short_message': '%(message)s', 40 | 'timestamp': '%(created)d.%(msecs)d', 41 | 'level': '%(loglevel)d', 42 | 'service': 'rekcurd', 43 | 'ml_service': application_name, 44 | 'service_level': service_level 45 | } 46 | fluent_handler = handler.FluentHandler('rekcurd') 47 | formatter = handler.FluentRecordFormatter(custom_format) 48 | fluent_handler.setFormatter(formatter) 49 | fluent_handler.setLevel(log_level) 50 | return fluent_handler 51 | 52 | def exception(self, message: str) -> None: 53 | """ 54 | emits exception to log 55 | :param message: error message 56 | """ 57 | self.log.error(message, exc_info=sys.exc_info(), stack_info=True, extra={'loglevel': 3}) 58 | 59 | def error(self, message: str) -> None: 60 | """ 61 | emits error log 62 | :param message: log 63 | """ 64 | self.log.error(message, extra={'loglevel': 3}) 65 | 66 | def debug(self, message: str) -> None: 67 | """ 68 | emits debug log 69 | :param message: log 70 | """ 71 | self.log.debug(message, extra={'loglevel': 7}) 72 | 73 | def info(self, message: str) -> None: 74 | """ 75 | emits info log 76 | :param message: log 77 | """ 78 | self.log.info(message, extra={'loglevel': 6}) 79 | 80 | def warn(self, message: str) -> None: 81 | """ 82 | emits warn log 83 | :param message: log 84 | """ 85 | self.log.warning(message, extra={'loglevel': 4}) 86 | 87 | 88 | class FluentServiceLogger(ServiceLoggerInterface): 89 | 90 | def __init__(self, 91 | logger_name: str = 'rekcurd.service', 92 | log_level: int = None, 93 | config: RekcurdConfig = RekcurdConfig()): 94 | """ 95 | Constructor 96 | :param logger_name: 97 | :param log_level: 98 | :param config: RekcurdConfig 99 | """ 100 | self.logger_name = logger_name 101 | self.log_level = int(log_level or logging.DEBUG) 102 | self.config = config 103 | self.ml_service = config.APPLICATION_NAME 104 | self.service_level = config.SERVICE_LEVEL 105 | self.logger = sender.FluentSender(logger_name) 106 | 107 | def emit(self, request, response, suppress_log_inout: bool = False) -> None: 108 | """ 109 | emits service log 110 | :param request: 111 | :param response: 112 | :param suppress_log_inout: 113 | :return: 114 | """ 115 | try: 116 | if suppress_log_inout: 117 | ml_input = '' 118 | ml_output = '' 119 | else: 120 | ml_input = super().to_str_from_request(request) 121 | ml_output = super().to_str_from_response(response) 122 | 123 | self.logger.emit(None, { 124 | 'host': gethostname(), 125 | 'short_message': 'prediction result.', 126 | 'timestamp': int(time.time() * 1000) / 1000, 127 | 'level': self.log_level, 128 | 'service': 'rekcurd', 129 | 'ml_service': self.ml_service, 130 | 'service_level': self.service_level, 131 | 'ml_input': ml_input, 132 | 'ml_output': ml_output 133 | }) 134 | except: 135 | try: 136 | FluentSystemLogger(self.logger_name, self.log_level, self.config).exception("can't write log") 137 | except: 138 | pass 139 | -------------------------------------------------------------------------------- /rekcurd/logger/logger_interface.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | from abc import ABCMeta, abstractmethod 6 | 7 | 8 | class SystemLoggerInterface(metaclass=ABCMeta): 9 | @abstractmethod 10 | def exception(self, message: str) -> None: 11 | raise NotImplemented() 12 | 13 | @abstractmethod 14 | def error(self, message: str) -> None: 15 | raise NotImplemented() 16 | 17 | @abstractmethod 18 | def debug(self, message: str) -> None: 19 | raise NotImplemented() 20 | 21 | @abstractmethod 22 | def info(self, message: str) -> None: 23 | raise NotImplemented() 24 | 25 | @abstractmethod 26 | def warn(self, message: str) -> None: 27 | raise NotImplemented() 28 | 29 | 30 | class ServiceLoggerInterface(metaclass=ABCMeta): 31 | @abstractmethod 32 | def emit(self, request, response, suppress_log_inout: bool = False) -> None: 33 | raise NotImplemented() 34 | 35 | # noinspection PyMethodMayBeStatic 36 | def to_str_from_request(self, request) -> str: 37 | tmp = {'option': request.option.val} 38 | if isinstance(request.input, (str, bytes)): 39 | tmp['input'] = str(request.input) 40 | else: 41 | tmp['input'] = list(request.input) 42 | return json.dumps(tmp) 43 | 44 | # noinspection PyMethodMayBeStatic 45 | def to_str_from_response(self, response) -> str: 46 | tmp = {'option': response.option.val} 47 | if isinstance(response.output, (str, bytes)): 48 | tmp['output'] = str(response.output) 49 | tmp['score'] = response.score 50 | else: 51 | tmp['output'] = list(response.output) 52 | tmp['score'] = list(response.score) 53 | return json.dumps(tmp) 54 | -------------------------------------------------------------------------------- /rekcurd/logger/logger_jsonlogger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import logging 5 | import sys 6 | import time 7 | from socket import gethostname 8 | 9 | from pythonjsonlogger import jsonlogger 10 | 11 | from rekcurd.utils import RekcurdConfig 12 | from .logger_interface import SystemLoggerInterface, ServiceLoggerInterface 13 | 14 | 15 | class JsonSystemLogger(SystemLoggerInterface): 16 | class JsonFormatter(jsonlogger.JsonFormatter): 17 | def parse(self): 18 | return [ 19 | 'host', 20 | 'short_message', 21 | 'timestamp', 22 | 'level', 23 | 'service', 24 | 'ml_service', 25 | 'service_level', 26 | ] 27 | 28 | def add_fields(self, log_record, record, message_dict): 29 | super().add_fields(log_record, record, message_dict) 30 | log_record['host'] = gethostname() 31 | log_record['timestamp'] = int(time.time() * 1000) / 1000 32 | log_record['service'] = 'rekcurd' 33 | 34 | def __init__(self, 35 | logger_name: str = 'rekcurd.system', 36 | log_level: int = None, 37 | config: RekcurdConfig = RekcurdConfig()) -> None: 38 | """ 39 | Constructor 40 | :param logger_name: 41 | :param log_level: 42 | :param config: RekcurdConfig 43 | """ 44 | self.config = config 45 | log_level = int(log_level or logging.DEBUG if config.DEBUG_MODE else logging.NOTSET) 46 | self.ml_service = config.APPLICATION_NAME 47 | self.service_level = config.SERVICE_LEVEL 48 | self.log = logging.getLogger(logger_name) 49 | handler = logging.StreamHandler() 50 | formatter = self.JsonFormatter() 51 | handler.setFormatter(formatter) 52 | handler.setLevel(log_level) 53 | self.log.handlers = [] 54 | self.log.addHandler(handler) 55 | self.log.setLevel(log_level) 56 | 57 | def exception(self, message: str) -> None: 58 | """ 59 | emits exception to log 60 | :param message: error message 61 | """ 62 | self.log.error(message, exc_info=sys.exc_info(), stack_info=True, 63 | extra={'short_message': message, 'level': 3, 64 | 'ml_service': self.ml_service, 65 | 'service_level': self.service_level}) 66 | 67 | def error(self, message: str) -> None: 68 | """ 69 | emits error log 70 | :param message: log 71 | """ 72 | self.log.error(message, extra={'short_message': message, 'level': 3, 73 | 'ml_service': self.ml_service, 74 | 'service_level': self.service_level}) 75 | 76 | def debug(self, message: str) -> None: 77 | """ 78 | emits debug log 79 | :param message: log 80 | """ 81 | self.log.debug(message, extra={'short_message': message, 'level': 7, 82 | 'ml_service': self.ml_service, 83 | 'service_level': self.service_level}) 84 | 85 | def info(self, message: str) -> None: 86 | """ 87 | emits info log 88 | :param message: log 89 | """ 90 | self.log.info(message, extra={'short_message': message, 'level': 6, 91 | 'ml_service': self.ml_service, 92 | 'service_level': self.service_level}) 93 | 94 | def warn(self, message: str) -> None: 95 | """ 96 | emits warn log 97 | :param message: log 98 | """ 99 | self.log.warning(message, extra={'short_message': message, 'level': 4, 100 | 'ml_service': self.ml_service, 101 | 'service_level': self.service_level}) 102 | 103 | 104 | class JsonServiceLogger(ServiceLoggerInterface): 105 | class JsonFormatter(jsonlogger.JsonFormatter): 106 | def parse(self): 107 | return [ 108 | 'host', 109 | 'short_message', 110 | 'timestamp', 111 | 'level', 112 | 'service', 113 | 'ml_service', 114 | 'service_level', 115 | 'ml_input', 116 | 'ml_output', 117 | ] 118 | 119 | def add_fields(self, log_record, record, message_dict): 120 | super().add_fields(log_record, record, message_dict) 121 | log_record['host'] = gethostname() 122 | log_record['timestamp'] = int(time.time() * 1000) / 1000 123 | log_record['service'] = 'rekcurd' 124 | 125 | def __init__(self, 126 | logger_name: str = 'rekcurd.service', 127 | log_level: int = None, 128 | config: RekcurdConfig = RekcurdConfig()): 129 | """ 130 | Constructor 131 | :param logger_name: 132 | :param log_level: 133 | :param config: RekcurdConfig 134 | """ 135 | self.logger_name = logger_name 136 | self.log_level = int(log_level or logging.DEBUG) 137 | self.config = config 138 | self.ml_service = config.APPLICATION_NAME 139 | self.service_level = config.SERVICE_LEVEL 140 | self.log = logging.getLogger(logger_name) 141 | handler = logging.StreamHandler() 142 | formatter = self.JsonFormatter() 143 | handler.setFormatter(formatter) 144 | handler.setLevel(self.log_level) 145 | self.log.addHandler(handler) 146 | self.log.setLevel(self.log_level) 147 | 148 | def emit(self, request, response, suppress_log_inout: bool = False) -> None: 149 | """ 150 | emits service log 151 | """ 152 | try: 153 | if suppress_log_inout: 154 | ml_input = '' 155 | ml_output = '' 156 | else: 157 | ml_input = super().to_str_from_request(request) 158 | ml_output = super().to_str_from_response(response) 159 | 160 | message = "prediction result." 161 | self.log.info(message, extra={'short_message': message, 162 | 'level': 6, 'ml_service': self.ml_service, 163 | 'service_level': self.service_level, 164 | 'ml_input': ml_input, 165 | 'ml_output': ml_output}) 166 | except Exception: 167 | try: 168 | JsonSystemLogger(self.logger_name, self.log_level, self.config).exception("can't write log") 169 | except: 170 | pass 171 | -------------------------------------------------------------------------------- /rekcurd/protobuf: -------------------------------------------------------------------------------- 1 | grpc/protobuf -------------------------------------------------------------------------------- /rekcurd/template/app.py-tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Example is available on https://github.com/rekcurd/rekcurd-example/tree/master/python/sklearn-digits 5 | 6 | 7 | from typing import Generator 8 | 9 | from rekcurd import Rekcurd 10 | from rekcurd.utils import PredictInput, PredictResult, EvaluateResult, EvaluateDetail, EvaluateResultDetail 11 | 12 | 13 | class RekcurdAppTemplateApp(Rekcurd): 14 | def load_model(self, filepath: str) -> object: 15 | """ override 16 | TODO: Implement "load_model" 17 | :param filepath: ML model file path. str 18 | :return predictor: Your ML predictor object. object 19 | """ 20 | pass 21 | 22 | def predict(self, predictor: object, idata: PredictInput, option: dict = None) -> PredictResult: 23 | """ override 24 | TODO: Implement "predict" 25 | :param predictor: Your ML predictor object. object 26 | :param idata: Input data. PredictInput, one of string/bytes/arr[int]/arr[float]/arr[string] 27 | :param option: Miscellaneous. dict 28 | :return result: Result. PredictResult 29 | result.label: Label. One of string/bytes/arr[int]/arr[float]/arr[string] 30 | result.score: Score. One of float/arr[float] 31 | result.option: Miscellaneous. dict 32 | """ 33 | pass 34 | 35 | def evaluate(self, predictor: object, filepath: str) -> Generator[EvaluateResultDetail, None, EvaluateResult]: 36 | """ override 37 | TODO: Implement "evaluate" 38 | :param predictor: Your ML predictor object. object 39 | :param filepath: Evaluation data file path. str 40 | :return result: Result. EvaluateResult 41 | result.num: Number of data. int 42 | result.accuracy: Accuracy. float 43 | result.precision: Precision. arr[float] 44 | result.recall: Recall. arr[float] 45 | result.fvalue: F1 value. arr[float] 46 | result.option: Optional metrics. dict[str, float] 47 | :return detail[]: Detail result of each prediction. List[EvaluateResultDetail] 48 | detail[].result: Prediction result. PredictResult 49 | detail[].is_correct: Prediction result is correct or not. bool 50 | """ 51 | pass 52 | 53 | def get_evaluate_detail(self, filepath: str, details: Generator[EvaluateResultDetail, None, None]) -> Generator[EvaluateDetail, None, None]: 54 | """ override 55 | TODO: Implement "get_evaluate_detail" 56 | :param filepath: Evaluation data file path. str 57 | :param details: Detail result of each prediction. List[EvaluateResultDetail] 58 | :return rtn: Return results. Generator[EvaluateDetail, None, None] 59 | rtn.input: Input data. PredictInput, one of string/bytes/arr[int]/arr[float]/arr[string] 60 | rtn.label: Predict label. PredictLabel, one of string/bytes/arr[int]/arr[float]/arr[string] 61 | rtn.result: Predict detail. EvaluateResultDetail 62 | """ 63 | pass 64 | 65 | 66 | if __name__ == '__main__': 67 | app = RekcurdAppTemplateApp() 68 | app.load_config_file("./settings.yml") 69 | app.run() 70 | -------------------------------------------------------------------------------- /rekcurd/template/requirements.txt-tpl: -------------------------------------------------------------------------------- 1 | rekcurd>=1.0.0,<1.1.0 # Apache-2.0 2 | -------------------------------------------------------------------------------- /rekcurd/template/settings.yml-tpl: -------------------------------------------------------------------------------- 1 | # For non-kubernetes users. If you deploy it from Rekcuerd-dashboard, all parameters are set by dashboard. 2 | 3 | ## Debug mode. 4 | debug: True 5 | 6 | ## Application parameters. 7 | app: 8 | name: RekcurdAppTemplate # This must be unique. 9 | host: 127.0.0.1 # Service insecure host. Default "127.0.0.1" 10 | port: 5000 # Service insecure port. Default "5000" 11 | service_level: development # Service level. One of [development/beta/staging/sandbox/production]. Default "development" 12 | 13 | ## Machine learning model access parameters. 14 | model: 15 | mode: local # ML model destination. One of [local/ceph_s3/aws_s3/gcs]. "ceph_s3" uses Ceph API that is compatible with AWS S3 API. "aws_s3" uses AWS S3 API. "gcs" uses GCS API. Default "local" 16 | local: # Local access parameters. Required for "local" mode. 17 | filepath: model/default.model # ML model file. Default "model/default.model" 18 | ceph_s3: # Ceph access parameters. Required for "ceph_s3" mode. 19 | filepath: model/default.model # ML model file. Default "model/default.model" 20 | access_key: xxxxx # Ceph access key. 21 | secret_key: xxxxx # Ceph secret key. 22 | host: xxxxx # Ceph host. 23 | port: 8773 # Ceph port. Default "8773" 24 | is_secure: False # Ceph secure access. Set "False" if you are not using ssl. 25 | bucket: xxxxx # Ceph bucket name. 26 | aws_s3: # AWS access parameters. Required for "aws_s3" mode. 27 | filepath: model/default.model # ML model file. Default "model/default.model" 28 | access_key: xxxxx # AWS access key. 29 | secret_key: xxxxx # AWS secret key. 30 | bucket: xxxxx # AWS bucket name. 31 | gcs: # GCS access parameters. Required for "gcs" mode. To generate keys, see "https://cloud.google.com/storage/docs/migrating#keys" 32 | filepath: model/default.model # ML model file. Default "model/default.model" 33 | access_key: xxxxx # GCS access key. 34 | secret_key: xxxxx # GCS secret key. 35 | bucket: xxxxx # GCS bucket name. 36 | -------------------------------------------------------------------------------- /rekcurd/template/start.sh-tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ECHO_PREFIX="[RekcurdAppTemplate application]: " 4 | 5 | set -e 6 | set -u 7 | 8 | echo "$ECHO_PREFIX start.." 9 | 10 | pip install -r requirements.txt 11 | python app.py 12 | -------------------------------------------------------------------------------- /rekcurd/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import json 5 | from typing import Union, List, Dict, NamedTuple 6 | 7 | from .rekcurd_config import RekcurdConfig, ModelModeEnum 8 | 9 | 10 | PredictInput = Union[str, bytes, List[str], List[int], List[float]] 11 | PredictLabel = Union[str, bytes, List[str], List[int], List[float]] 12 | PredictScore = Union[float, List[float]] 13 | 14 | 15 | class PredictResult: 16 | def __init__(self, label: PredictLabel, score: PredictScore, option: dict = None): 17 | self.label = label 18 | self.score = score 19 | self.option = json.dumps(option) if option is not None else '{}' 20 | 21 | 22 | class EvaluateResult: 23 | def __init__(self, num: int, accuracy: float, precision: List[float], 24 | recall: List[float], fvalue: List[float], label: List[PredictLabel], 25 | option: Dict[str, float] = {}): 26 | self.num = num 27 | self.accuracy = accuracy 28 | self.precision = precision 29 | self.recall = recall 30 | self.fvalue = fvalue 31 | self.label = label 32 | self.option = option 33 | 34 | 35 | class EvaluateResultDetail(NamedTuple): 36 | result: PredictResult 37 | is_correct: bool 38 | 39 | 40 | class EvaluateDetail(NamedTuple): 41 | input: PredictInput 42 | label: PredictLabel 43 | result: EvaluateResultDetail 44 | 45 | 46 | incoming_headers = [ 47 | 'x-request-id', 'x-b3-traceid', 'x-b3-spanid', 'x-b3-parentspanid', 48 | 'x-b3-sampled', 'x-b3-flags', 'x-ot-span-context'] 49 | 50 | def getForwardHeaders(incoming: list) -> list: 51 | headers = list() 52 | for k,v in incoming: 53 | if k in incoming_headers: 54 | headers.append((k, v)) 55 | return headers 56 | -------------------------------------------------------------------------------- /rekcurd/utils/rekcurd_config.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | 4 | import os 5 | import uuid 6 | import yaml 7 | 8 | from enum import Enum 9 | 10 | 11 | class ModelModeEnum(Enum): 12 | """ 13 | Rekcurd model storage options. 14 | """ 15 | LOCAL = 'local' 16 | CEPH_S3 = 'ceph_s3' 17 | AWS_S3 = 'aws_s3' 18 | GCS = 'gcs' 19 | 20 | @classmethod 21 | def to_Enum(cls, mode: str): 22 | if cls.LOCAL.value == mode: 23 | return cls.LOCAL 24 | elif cls.CEPH_S3.value == mode: 25 | return cls.CEPH_S3 26 | elif cls.AWS_S3.value == mode: 27 | return cls.AWS_S3 28 | elif cls.GCS.value == mode: 29 | return cls.GCS 30 | else: 31 | raise ValueError("'{}' is not supported as ModelModeEnum".format(mode)) 32 | 33 | 34 | class RekcurdConfig: 35 | """ 36 | Rekcurd configurations. 37 | """ 38 | __SERVICE_DEFAULT_HOST = "127.0.0.1" 39 | __SERVICE_DEFAULT_PORT = 5000 40 | __CEPH_DEFAULT_PORT = 8773 41 | KUBERNETES_MODE: str = None 42 | DEBUG_MODE: bool = None 43 | APPLICATION_NAME: str = None 44 | SERVICE_ID: str = None 45 | SERVICE_INSECURE_HOST: str = __SERVICE_DEFAULT_HOST 46 | SERVICE_INSECURE_PORT: int = __SERVICE_DEFAULT_PORT 47 | SERVICE_LEVEL: str = None 48 | MODEL_MODE_ENUM: ModelModeEnum = None 49 | MODEL_FILE_PATH: str = None 50 | CEPH_ACCESS_KEY: str = None 51 | CEPH_SECRET_KEY: str = None 52 | CEPH_HOST: str = None 53 | CEPH_PORT: int = __CEPH_DEFAULT_PORT 54 | CEPH_IS_SECURE: bool = None 55 | CEPH_BUCKET_NAME: str = None 56 | AWS_ACCESS_KEY: str = None 57 | AWS_SECRET_KEY: str = None 58 | AWS_BUCKET_NAME: str = None 59 | GCS_ACCESS_KEY: str = None 60 | GCS_SECRET_KEY: str = None 61 | GCS_BUCKET_NAME: str = None 62 | 63 | def __init__(self, config_file: str = None): 64 | self.KUBERNETES_MODE = os.getenv("REKCURD_KUBERNETES_MODE") 65 | if self.KUBERNETES_MODE is None: 66 | self.__load_from_file(config_file) 67 | else: 68 | self.__load_from_env() 69 | 70 | def set_configurations( 71 | self, debug_mode: bool = None, application_name: str = None, 72 | service_insecure_host: str = None, service_insecure_port: int = None, 73 | service_level: str = None, model_mode: str = None, 74 | model_filepath: str = None, ceph_access_key: str = None, 75 | ceph_secret_key: str = None, ceph_host: str = None, 76 | ceph_port: int = None, ceph_is_secure: bool = None, 77 | ceph_bucket_name: str = None, aws_access_key: str = None, 78 | aws_secret_key: str = None, aws_bucket_name: str = None, 79 | gcs_access_key: str = None, gcs_secret_key: str = None, gcs_bucket_name: str = None, 80 | **options): 81 | self.DEBUG_MODE = debug_mode if debug_mode is not None else self.DEBUG_MODE 82 | self.APPLICATION_NAME = application_name or self.APPLICATION_NAME 83 | self.SERVICE_INSECURE_HOST = service_insecure_host or self.SERVICE_INSECURE_HOST 84 | self.SERVICE_INSECURE_PORT = int(service_insecure_port or self.SERVICE_INSECURE_PORT) 85 | self.SERVICE_LEVEL = service_level or self.SERVICE_LEVEL 86 | if model_mode is not None: 87 | self.MODEL_MODE_ENUM = ModelModeEnum.to_Enum(model_mode) 88 | self.MODEL_FILE_PATH = model_filepath or self.MODEL_FILE_PATH 89 | self.CEPH_ACCESS_KEY = ceph_access_key or self.CEPH_ACCESS_KEY 90 | self.CEPH_SECRET_KEY = ceph_secret_key or self.CEPH_SECRET_KEY 91 | self.CEPH_HOST = ceph_host or self.CEPH_HOST 92 | self.CEPH_PORT = int(ceph_port or self.CEPH_PORT) 93 | self.CEPH_IS_SECURE = ceph_is_secure if ceph_is_secure is not None else self.CEPH_IS_SECURE 94 | self.CEPH_BUCKET_NAME = ceph_bucket_name or self.CEPH_BUCKET_NAME 95 | self.AWS_ACCESS_KEY = aws_access_key or self.AWS_ACCESS_KEY 96 | self.AWS_SECRET_KEY = aws_secret_key or self.AWS_SECRET_KEY 97 | self.AWS_BUCKET_NAME = aws_bucket_name or self.AWS_BUCKET_NAME 98 | self.GCS_ACCESS_KEY = gcs_access_key or self.GCS_ACCESS_KEY 99 | self.GCS_SECRET_KEY = gcs_secret_key or self.GCS_SECRET_KEY 100 | self.GCS_BUCKET_NAME = gcs_bucket_name or self.GCS_BUCKET_NAME 101 | 102 | def __load_from_file(self, config_file: str): 103 | if config_file is not None: 104 | with open(config_file, 'r') as f: 105 | config = yaml.load(f) 106 | else: 107 | config = dict() 108 | self.DEBUG_MODE = config.get("debug", True) 109 | config_app = config.get("app", dict()) 110 | self.APPLICATION_NAME = config_app.get("name", "sample") 111 | self.SERVICE_ID = uuid.uuid4().hex 112 | self.SERVICE_INSECURE_HOST = config_app.get("host", self.__SERVICE_DEFAULT_HOST) 113 | self.SERVICE_INSECURE_PORT = config_app.get("port", self.__SERVICE_DEFAULT_PORT) 114 | self.SERVICE_LEVEL = config_app.get("service_level", "development") 115 | config_model = config.get("model", dict()) 116 | model_mode = config_model.get("mode", "local") 117 | self.MODEL_MODE_ENUM = ModelModeEnum.to_Enum(model_mode) 118 | if self.MODEL_MODE_ENUM == ModelModeEnum.LOCAL: 119 | config_model_mode = config_model.get(model_mode, dict()) 120 | self.MODEL_FILE_PATH = config_model_mode.get("filepath", "model/default.model") 121 | elif self.MODEL_MODE_ENUM == ModelModeEnum.CEPH_S3: 122 | config_model_mode = config_model.get(model_mode, dict()) 123 | self.MODEL_FILE_PATH = config_model_mode.get("filepath", "model/default.model") 124 | self.CEPH_ACCESS_KEY = config_model_mode.get("access_key") 125 | self.CEPH_SECRET_KEY = config_model_mode.get("secret_key") 126 | self.CEPH_HOST = config_model_mode.get("host") 127 | self.CEPH_PORT = config_model_mode.get("port", self.__CEPH_DEFAULT_PORT) 128 | self.CEPH_IS_SECURE = config_model_mode.get("is_secure", False) 129 | self.CEPH_BUCKET_NAME = config_model_mode.get("bucket") 130 | elif self.MODEL_MODE_ENUM == ModelModeEnum.AWS_S3: 131 | config_model_mode = config_model.get(model_mode, dict()) 132 | self.MODEL_FILE_PATH = config_model_mode.get("filepath", "model/default.model") 133 | self.AWS_ACCESS_KEY = config_model_mode.get("access_key") 134 | self.AWS_SECRET_KEY = config_model_mode.get("secret_key") 135 | self.AWS_BUCKET_NAME = config_model_mode.get("bucket") 136 | elif self.MODEL_MODE_ENUM == ModelModeEnum.GCS: 137 | config_model_mode = config_model.get(model_mode, dict()) 138 | self.MODEL_FILE_PATH = config_model_mode.get("filepath", "model/default.model") 139 | self.GCS_ACCESS_KEY = config_model_mode.get("access_key") 140 | self.GCS_SECRET_KEY = config_model_mode.get("secret_key") 141 | self.GCS_BUCKET_NAME = config_model_mode.get("bucket") 142 | else: 143 | raise ValueError("'{}' is not supported as ModelModeEnum".format(model_mode)) 144 | 145 | def __load_from_env(self): 146 | self.DEBUG_MODE = os.getenv("REKCURD_DEBUG_MODE", "True").lower() == 'true' 147 | self.APPLICATION_NAME = os.getenv("REKCURD_APPLICATION_NAME") 148 | self.SERVICE_ID = os.getenv("REKCURD_SERVICE_ID") 149 | self.SERVICE_INSECURE_HOST = os.getenv("REKCURD_SERVICE_INSECURE_HOST", self.__SERVICE_DEFAULT_HOST) 150 | self.SERVICE_INSECURE_PORT = int(os.getenv("REKCURD_SERVICE_INSECURE_PORT", self.__SERVICE_DEFAULT_PORT)) 151 | self.SERVICE_LEVEL = os.getenv("REKCURD_SERVICE_LEVEL") 152 | model_mode = os.getenv("REKCURD_MODEL_MODE") 153 | self.MODEL_MODE_ENUM = ModelModeEnum.to_Enum(model_mode) 154 | self.MODEL_FILE_PATH = os.getenv("REKCURD_MODEL_FILE_PATH") 155 | self.CEPH_ACCESS_KEY = os.getenv("REKCURD_CEPH_ACCESS_KEY") 156 | self.CEPH_SECRET_KEY = os.getenv("REKCURD_CEPH_SECRET_KEY") 157 | self.CEPH_HOST = os.getenv("REKCURD_CEPH_HOST") 158 | self.CEPH_PORT = int(os.getenv("REKCURD_CEPH_PORT", str(self.__CEPH_DEFAULT_PORT))) 159 | self.CEPH_IS_SECURE = os.getenv("REKCURD_CEPH_IS_SECURE", "False").lower() == 'true' 160 | self.CEPH_BUCKET_NAME = os.getenv("REKCURD_CEPH_BUCKET_NAME") 161 | self.AWS_ACCESS_KEY = os.getenv("REKCURD_AWS_ACCESS_KEY") 162 | self.AWS_SECRET_KEY = os.getenv("REKCURD_AWS_SECRET_KEY") 163 | self.AWS_BUCKET_NAME = os.getenv("REKCURD_AWS_BUCKET_NAME") 164 | self.GCS_ACCESS_KEY = os.getenv("REKCURD_GCS_ACCESS_KEY") 165 | self.GCS_SECRET_KEY = os.getenv("REKCURD_GCS_SECRET_KEY") 166 | self.GCS_BUCKET_NAME = os.getenv("REKCURD_GCS_BUCKET_NAME") 167 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fluent-logger>=0.9.3 # Apache-2.0 2 | python-json-logger>=0.1.9 # BSD 3 | grpcio>=1.22.0 # Apache-2.0 4 | grpcio-tools>=1.22.0 # Apache-2.0 5 | PyYAML>=3.12 # MIT 6 | boto>=2.49.0 # MIT 7 | boto3>=1.9.38 # Apache-2.0 8 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo 'git clean -xdf' 4 | echo 'python setup.py sdist' 5 | echo 'python setup.py bdist_wheel --universal' 6 | echo 'twine upload dist/* -r https://upload.pypi.org/legacy/ -u rekcurd' 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Rekcurd Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | from setuptools import setup 17 | 18 | 19 | here = os.path.abspath(os.path.dirname(__file__)) 20 | exec(open(os.path.join(here, 'rekcurd', '_project.py')).read()) 21 | exec(open(os.path.join(here, 'rekcurd', '_version.py')).read()) 22 | PACKAGE_NAME = __project__ # NOQA 23 | VERSION = __version__ # NOQA 24 | DEVELOPMENT_STATUS = "3 - Alpha" 25 | 26 | # To install the library, run the following 27 | # 28 | # python setup.py install 29 | # 30 | # prerequisite: setuptools 31 | # http://pypi.python.org/pypi/setuptools 32 | 33 | EXTRAS = {} 34 | REQUIRES = [] 35 | with open('requirements.txt') as f: 36 | for line in f: 37 | line, _, _ = line.partition('#') 38 | line = line.strip() 39 | if ';' in line: 40 | requirement, _, specifier = line.partition(';') 41 | for_specifier = EXTRAS.setdefault(':{}'.format(specifier), []) 42 | for_specifier.append(requirement) 43 | else: 44 | REQUIRES.append(line) 45 | 46 | with open('test-requirements.txt') as f: 47 | TESTS_REQUIRES = f.readlines() 48 | 49 | with open("README.md", "r") as fh: 50 | LONG_DESCRIPTION = fh.read() 51 | 52 | setup( 53 | name=PACKAGE_NAME, 54 | version=VERSION, 55 | description="A Python gRPC framework for serving a machine learning module written in Python.", 56 | long_description_content_type='text/markdown', 57 | author_email="", 58 | author="Rekcurd", 59 | license="Apache License Version 2.0", 60 | url="https://github.com/rekcurd/rekcurd-python", 61 | keywords=["Rekcurd", "Kubernetes"], 62 | install_requires=REQUIRES, 63 | tests_require=TESTS_REQUIRES, 64 | extras_require=EXTRAS, 65 | packages=['rekcurd', 'rekcurd.console_scripts', 'rekcurd.core', 66 | 'rekcurd.data_servers', 'rekcurd.logger', 'rekcurd.protobuf', 67 | 'rekcurd.template', 'rekcurd.utils'], 68 | package_data={ 69 | 'rekcurd': [ 70 | 'template/*' 71 | ], 72 | }, 73 | include_package_data=True, 74 | long_description=LONG_DESCRIPTION, 75 | entry_points={ 76 | 'console_scripts': [ 77 | 'rekcurd=rekcurd.console_scripts:main', 78 | ], 79 | }, 80 | classifiers=[ 81 | "Development Status :: %s" % DEVELOPMENT_STATUS, 82 | "Environment :: Web Environment", 83 | "Topic :: Software Development :: Libraries :: Application Frameworks", 84 | "Intended Audience :: Developers", 85 | "Intended Audience :: Information Technology", 86 | "License :: OSI Approved :: Apache Software License", 87 | "Operating System :: OS Independent", 88 | "Programming Language :: Python", 89 | "Programming Language :: Python :: 3", 90 | "Programming Language :: Python :: 3.6", 91 | "Programming Language :: Python :: 3.7", 92 | ], 93 | ) 94 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage>=4.0.3 2 | nose>=1.3.7 3 | pytest 4 | py>=1.4.31 5 | codecov>=1.4.0 6 | grpcio-testing>=1.22.0 7 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | from test.test_app import RekcurdAppTemplateApp 2 | from rekcurd import RekcurdWorkerServicer 3 | 4 | 5 | app = RekcurdAppTemplateApp() 6 | Type = RekcurdWorkerServicer.Type 7 | -------------------------------------------------------------------------------- /test/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rekcurd/rekcurd-python/71e1ecd1f7bf8394563e682308fa7b404492e6da/test/core/__init__.py -------------------------------------------------------------------------------- /test/core/test_dashboard_servicer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import time 3 | from functools import wraps 4 | from unittest.mock import patch, Mock, mock_open 5 | import grpc_testing 6 | from grpc import StatusCode 7 | 8 | from rekcurd import RekcurdPack, RekcurdDashboardServicer 9 | from rekcurd.protobuf import rekcurd_pb2 10 | from rekcurd.data_servers import DataServer 11 | from rekcurd.logger import JsonSystemLogger, JsonServiceLogger 12 | from rekcurd.utils import EvaluateResult, EvaluateResultDetail, PredictResult, EvaluateDetail 13 | from test import app 14 | 15 | 16 | target_service = rekcurd_pb2.DESCRIPTOR.services_by_name['RekcurdDashboard'] 17 | eval_result = EvaluateResult(1, 0.8, [0.7], [0.6], [0.5], ['label1'], {'dummy': 0.4}) 18 | eval_result_details = [EvaluateResultDetail(PredictResult('pre_label', 0.9), False)] 19 | eval_detail = EvaluateDetail('input', 'label', eval_result_details[0]) 20 | 21 | # Overwrite CHUNK_SIZE and BYTE_LIMIT to smaller values for testing 22 | chunk_size = 3 23 | RekcurdDashboardServicer.CHUNK_SIZE = chunk_size 24 | # response byte size will be greater than BYTE_LIMIT in 2nd loop 25 | RekcurdDashboardServicer.BYTE_LIMIT = 190 26 | 27 | 28 | def patch_predictor(): 29 | """Decorator to mock for dashboard. 30 | """ 31 | 32 | def test_method(func): 33 | @wraps(func) 34 | def inner_method(*args, **kwargs): 35 | with patch('rekcurd.data_servers.LocalHandler.download', 36 | new=Mock(return_value=True)) as _, \ 37 | patch('rekcurd.data_servers.LocalHandler.upload', 38 | new=Mock(return_value=True)) as _, \ 39 | patch('rekcurd.data_servers.CephHandler.download', 40 | new=Mock(return_value=True)) as _, \ 41 | patch('rekcurd.data_servers.CephHandler.upload', 42 | new=Mock(return_value=True)) as _, \ 43 | patch('rekcurd.core.rekcurd_dashboard_servicer.pickle', 44 | new=Mock()) as _, \ 45 | patch('rekcurd.core.rekcurd_dashboard_servicer.pickle.load', 46 | new=Mock(return_value=eval_result)) as _, \ 47 | patch('builtins.open', new_callable=mock_open) as _: 48 | return func(*args, **kwargs) 49 | return inner_method 50 | return test_method 51 | 52 | 53 | class RekcurdWorkerServicerTest(unittest.TestCase): 54 | """Tests for RekcurdDashboardServicer. 55 | """ 56 | 57 | def setUp(self): 58 | def gen_eval(): 59 | for d in eval_result_details: 60 | yield d 61 | return eval_result 62 | app.load_config_file("./test/test-settings.yml") 63 | app.data_server = DataServer(app.config) 64 | app.system_logger = JsonSystemLogger(config=app.config) 65 | app.service_logger = JsonServiceLogger(config=app.config) 66 | app.evaluate = Mock(return_value=gen_eval()) 67 | self._real_time = grpc_testing.strict_real_time() 68 | self._fake_time = grpc_testing.strict_fake_time(time.time()) 69 | servicer = RekcurdDashboardServicer(RekcurdPack(app, None)) 70 | descriptors_to_services = { 71 | target_service: servicer 72 | } 73 | self._real_time_server = grpc_testing.server_from_dictionary( 74 | descriptors_to_services, self._real_time) 75 | self._fake_time_server = grpc_testing.server_from_dictionary( 76 | descriptors_to_services, self._fake_time) 77 | 78 | def test_ServiceInfo(self): 79 | rpc = self._real_time_server.invoke_unary_unary( 80 | target_service.methods_by_name['ServiceInfo'], (), 81 | rekcurd_pb2.ServiceInfoRequest(), None) 82 | response, trailing_metadata, code, details = rpc.termination() 83 | self.assertIs(code, StatusCode.OK) 84 | self.assertEqual(response.application_name, 'test') 85 | self.assertEqual(response.service_level, 'development') 86 | 87 | @patch_predictor() 88 | def test_UploadModel(self): 89 | request = rekcurd_pb2.UploadModelRequest(path='my_path', data=b'data') 90 | rpc = self._real_time_server.invoke_stream_unary( 91 | target_service.methods_by_name['UploadModel'], (), None) 92 | rpc.send_request(request) 93 | rpc.send_request(request) 94 | rpc.send_request(request) 95 | rpc.requests_closed() 96 | response, trailing_metadata, code, details = rpc.termination() 97 | self.assertIs(code, StatusCode.OK) 98 | self.assertEqual(response.status, 1) 99 | 100 | @patch_predictor() 101 | def test_InvalidUploadModel(self): 102 | request = rekcurd_pb2.UploadModelRequest(path='../../../my_path', data=b'data') 103 | rpc = self._real_time_server.invoke_stream_unary( 104 | target_service.methods_by_name['UploadModel'], (), None) 105 | rpc.send_request(request) 106 | rpc.send_request(request) 107 | rpc.send_request(request) 108 | rpc.requests_closed() 109 | response, trailing_metadata, code, details = rpc.termination() 110 | self.assertIs(code, StatusCode.UNKNOWN) 111 | self.assertEqual(response.status, 0) 112 | 113 | @patch_predictor() 114 | def test_SwitchModel(self): 115 | rpc = self._real_time_server.invoke_unary_unary( 116 | target_service.methods_by_name['SwitchModel'], (), 117 | rekcurd_pb2.SwitchModelRequest(path='my_path'), None) 118 | response, trailing_metadata, code, details = rpc.termination() 119 | self.assertIs(code, StatusCode.OK) 120 | self.assertEqual(response.status, 1) 121 | 122 | @patch_predictor() 123 | def test_InvalidSwitchModel(self): 124 | rpc = self._real_time_server.invoke_unary_unary( 125 | target_service.methods_by_name['SwitchModel'], (), 126 | rekcurd_pb2.SwitchModelRequest(path='../../my_path'), None) 127 | response, trailing_metadata, code, details = rpc.termination() 128 | self.assertIs(code, StatusCode.UNKNOWN) 129 | self.assertEqual(response.status, 0) 130 | 131 | @patch_predictor() 132 | def test_UploadEvaluationData(self): 133 | request = rekcurd_pb2.UploadEvaluationDataRequest(data_path='my_path', data=b'data_') 134 | rpc = self._real_time_server.invoke_stream_unary( 135 | target_service.methods_by_name['UploadEvaluationData'], (), None) 136 | rpc.send_request(request) 137 | rpc.send_request(request) 138 | rpc.send_request(request) 139 | rpc.requests_closed() 140 | response, trailing_metadata, code, details = rpc.termination() 141 | self.assertEqual(response.status, 1) 142 | 143 | @patch_predictor() 144 | def test_InvalidEvaluationData(self): 145 | request = rekcurd_pb2.UploadEvaluationDataRequest(data_path='../../my_path', data=b'data_') 146 | rpc = self._real_time_server.invoke_stream_unary( 147 | target_service.methods_by_name['UploadEvaluationData'], (), None) 148 | rpc.send_request(request) 149 | rpc.send_request(request) 150 | rpc.send_request(request) 151 | rpc.requests_closed() 152 | response, trailing_metadata, code, details = rpc.termination() 153 | self.assertEqual(response.status, 0) 154 | 155 | @patch_predictor() 156 | def test_EvaluateModel(self): 157 | request = rekcurd_pb2.EvaluateModelRequest(data_path='my_path', result_path='my_path') 158 | rpc = self._real_time_server.invoke_stream_unary( 159 | target_service.methods_by_name['EvaluateModel'], (), None) 160 | rpc.send_request(request) 161 | rpc.send_request(request) 162 | rpc.send_request(request) 163 | rpc.requests_closed() 164 | response, trailing_metadata, code, details = rpc.termination() 165 | self.assertIs(code, StatusCode.OK) 166 | self.assertEqual(round(response.metrics.num, 3), eval_result.num) 167 | self.assertEqual(round(response.metrics.accuracy, 3), eval_result.accuracy) 168 | self.assertEqual([round(p, 3) for p in response.metrics.precision], eval_result.precision) 169 | self.assertEqual([round(r, 3) for r in response.metrics.recall], eval_result.recall) 170 | self.assertEqual([round(f, 3) for f in response.metrics.fvalue], eval_result.fvalue) 171 | self.assertEqual(round(response.metrics.option['dummy'], 3), eval_result.option['dummy']) 172 | self.assertEqual([l.str.val[0] for l in response.metrics.label], eval_result.label) 173 | 174 | @patch_predictor() 175 | def test_InvalidEvalauteModel(self): 176 | request = rekcurd_pb2.EvaluateModelRequest(data_path='../../my_path', result_path='my_res_path') 177 | rpc = self._real_time_server.invoke_stream_unary( 178 | target_service.methods_by_name['EvaluateModel'], (), None) 179 | rpc.send_request(request) 180 | rpc.requests_closed() 181 | response, trailing_metadata, code, details = rpc.termination() 182 | self.assertIs(code, StatusCode.UNKNOWN) 183 | self.assertEqual(response.metrics.num, 0) 184 | 185 | request = rekcurd_pb2.EvaluateModelRequest(data_path='my_path', result_path='../my_res_path') 186 | rpc = self._real_time_server.invoke_stream_unary( 187 | target_service.methods_by_name['EvaluateModel'], (), None) 188 | rpc.send_request(request) 189 | rpc.requests_closed() 190 | response, trailing_metadata, code, details = rpc.termination() 191 | self.assertIs(code, StatusCode.UNKNOWN) 192 | self.assertEqual(response.metrics.num, 0) 193 | 194 | def __send_eval_result(self, size, take_times): 195 | app.get_evaluate_detail = Mock(return_value=iter(eval_detail for _ in range(size))) 196 | rpc = self._real_time_server.invoke_unary_stream( 197 | target_service.methods_by_name['EvaluationResult'], (), 198 | rekcurd_pb2.EvaluationResultRequest(data_path='my_path', result_path='my_path'), None) 199 | responses = [rpc.take_response() for _ in range(take_times)] 200 | return rpc, responses 201 | 202 | @patch_predictor() 203 | def test_EvalautionResult(self): 204 | rpc, responses = self.__send_eval_result(1, 1) 205 | response = responses[0] 206 | 207 | self.assertEqual(len(response.detail), 1) 208 | detail = response.detail[0] 209 | self.assertEqual(detail.input.str.val, [eval_detail.input]) 210 | self.assertEqual(detail.label.str.val, [eval_detail.label]) 211 | self.assertEqual(detail.output.str.val, [eval_result_details[0].result.label]) 212 | self.assertEqual([round(s, 3) for s in detail.score], [eval_result_details[0].result.score]) 213 | self.assertEqual(detail.is_correct, eval_result_details[0].is_correct) 214 | 215 | with self.assertRaises(ValueError): 216 | rpc.take_response() 217 | 218 | trailing_metadata, code, details = rpc.termination() 219 | self.assertIs(code, StatusCode.OK) 220 | 221 | @patch_predictor() 222 | def test_EvalautionResult_multi_response(self): 223 | # chunk_size * 3 224 | rpc, responses = self.__send_eval_result(chunk_size * 3, 3) 225 | for r in responses: 226 | self.assertEqual(len(r.detail), chunk_size) 227 | with self.assertRaises(ValueError): 228 | rpc.take_response() 229 | rpc.termination() 230 | 231 | # chunk_size * 2 + 1 232 | rpc, responses = self.__send_eval_result(chunk_size * 2 + 1, 2) 233 | self.assertEqual(len(responses[0].detail), chunk_size) 234 | self.assertEqual(len(responses[1].detail), chunk_size + 1) 235 | with self.assertRaises(ValueError): 236 | rpc.take_response() 237 | rpc.termination() 238 | 239 | def test_get_io_by_type(self): 240 | servicer = RekcurdDashboardServicer(RekcurdPack(app, None)) 241 | self.assertEqual(servicer.get_io_by_type('test').str.val, ['test']) 242 | self.assertEqual(servicer.get_io_by_type(['test', 'test2']).str.val, ['test', 'test2']) 243 | self.assertEqual(servicer.get_io_by_type(2).tensor.val, [2]) 244 | self.assertEqual(servicer.get_io_by_type([2, 3]).tensor.val, [2, 3]) 245 | 246 | def test_get_score_by_type(self): 247 | servicer = RekcurdDashboardServicer(RekcurdPack(app, None)) 248 | score = 4.5 249 | self.assertEqual(servicer.get_score_by_type(score), [score]) 250 | self.assertEqual(servicer.get_score_by_type([score]), [score]) 251 | -------------------------------------------------------------------------------- /test/core/test_rekcurd_worker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from functools import wraps 4 | from unittest.mock import Mock, patch 5 | 6 | from rekcurd.data_servers import DataServer 7 | from rekcurd.logger import JsonSystemLogger, JsonServiceLogger 8 | from test import app, Type 9 | 10 | 11 | def patch_predictor(): 12 | """Decorator to mock for dashboard. 13 | """ 14 | 15 | def test_method(func): 16 | @wraps(func) 17 | def inner_method(*args, **kwargs): 18 | with patch('rekcurd.data_servers.LocalHandler.download', 19 | new=Mock(return_value=True)) as _, \ 20 | patch('rekcurd.data_servers.LocalHandler.upload', 21 | new=Mock(return_value=True)) as _, \ 22 | patch('rekcurd.data_servers.CephHandler.download', 23 | new=Mock(return_value=True)) as _, \ 24 | patch('rekcurd.data_servers.CephHandler.upload', 25 | new=Mock(return_value=True)) as _: 26 | return func(*args, **kwargs) 27 | return inner_method 28 | return test_method 29 | 30 | 31 | class RekcurdWorkerTest(unittest.TestCase): 32 | """Tests for Rekcurd. 33 | """ 34 | 35 | def setUp(self): 36 | os.environ["REKCURD_UNITTEST"] = "True" 37 | app.load_config_file("./test/test-settings.yml") 38 | app.data_server = DataServer(app.config) 39 | app.system_logger = JsonSystemLogger(config=app.config) 40 | app.service_logger = JsonServiceLogger(config=app.config) 41 | 42 | def tearDown(self): 43 | del os.environ["REKCURD_UNITTEST"] 44 | 45 | @patch_predictor() 46 | def test_run(self): 47 | self.assertIsNone(app.run()) 48 | 49 | def test_type(self): 50 | # TODO: DEPRECATED 51 | self.assertIsNone(app.set_type(Type.STRING, Type.ARRAY_INT)) 52 | self.assertEqual(app.get_type_input(), Type.STRING) 53 | self.assertEqual(app.get_type_output(), Type.ARRAY_INT) 54 | -------------------------------------------------------------------------------- /test/core/test_worker_servicer.py: -------------------------------------------------------------------------------- 1 | from test import app, Type 2 | 3 | import unittest 4 | import time 5 | from functools import wraps 6 | from unittest.mock import patch, Mock 7 | import grpc_testing 8 | from grpc import StatusCode 9 | 10 | from rekcurd.protobuf import rekcurd_pb2 11 | from rekcurd import RekcurdPack, RekcurdWorkerServicer 12 | from rekcurd.data_servers import DataServer 13 | from rekcurd.logger import JsonSystemLogger, JsonServiceLogger 14 | from rekcurd.utils import PredictResult 15 | 16 | 17 | target_service = rekcurd_pb2.DESCRIPTOR.services_by_name['RekcurdWorker'] 18 | 19 | 20 | def patch_predictor(input_type, output_type): 21 | """Decorator to mock the predictor. 22 | Patch the several methods of the Predict class to make a fake predictor. 23 | """ 24 | 25 | _prediction_value_map = { 26 | Type.STRING: PredictResult('Rekcurd', 1.0, option={}), 27 | Type.BYTES: PredictResult(b'\x8f\xfa;\xc8a\xa3T%', 1.0, option={}), 28 | Type.ARRAY_INT: PredictResult([2, 3, 5, 7], [1.0, 1.0, 1.0, 1.0], option={}), 29 | Type.ARRAY_FLOAT: PredictResult([0.78341155, 0.03166816, 0.92745938], [1.0, 1.0, 1.0], option={}), 30 | Type.ARRAY_STRING: PredictResult(['Rekcurd', 'is', 'awesome'], [1.0, 1.0, 1.0], option={}), 31 | } 32 | 33 | def test_method(func): 34 | @wraps(func) 35 | def inner_method(*args, **kwargs): 36 | with patch('test.RekcurdAppTemplateApp.get_type_input', 37 | new=Mock(return_value=input_type)) as _, \ 38 | patch('test.RekcurdAppTemplateApp.get_type_output', 39 | new=Mock(return_value=output_type)) as _, \ 40 | patch('test.RekcurdAppTemplateApp.load_model') as _, \ 41 | patch('test.RekcurdAppTemplateApp.predict', 42 | new=Mock(return_value=_prediction_value_map[output_type])) as _: 43 | return func(*args, **kwargs) 44 | return inner_method 45 | return test_method 46 | 47 | 48 | class RekcurdWorkerServicerTest(unittest.TestCase): 49 | """Tests for RekcurdWorkerServicer.""" 50 | 51 | def fake_string_request(self): 52 | request = rekcurd_pb2.StringInput() 53 | request.input = 'Rekcurd' 54 | request.option.val = '{}' 55 | return request 56 | 57 | def fake_bytes_request(self): 58 | request = rekcurd_pb2.BytesInput() 59 | request.input = b'\x9cT\xee\xca\x19\xbb\xa44\xfcS' 60 | request.option.val = '{}' 61 | return request 62 | 63 | def fake_arrint_request(self): 64 | request = rekcurd_pb2.ArrIntInput() 65 | request.input.extend([218, 81, 2, 215, 28]) 66 | request.option.val = '{}' 67 | return request 68 | 69 | def fake_arrfloat_request(self): 70 | request = rekcurd_pb2.ArrFloatInput() 71 | request.input.extend([0.22861859, 0.90036856, 0.03665003, 0.69281863, 0.23225956]) 72 | request.option.val = '{}' 73 | return request 74 | 75 | def fake_arrstring_request(self): 76 | request = rekcurd_pb2.ArrStringInput() 77 | request.input.extend(['Rekcurd', 'Docker', 'Rekcurd', 'Rekcod']) 78 | request.option.val = '{}' 79 | return request 80 | 81 | def assertStringResponse(self, response): 82 | self.assertIsInstance(response, rekcurd_pb2.StringOutput) 83 | 84 | def assertBytesResponse(self, response): 85 | self.assertIsInstance(response, rekcurd_pb2.BytesOutput) 86 | 87 | def assertArrIntResponse(self, response): 88 | self.assertIsInstance(response, rekcurd_pb2.ArrIntOutput) 89 | 90 | def assertArrFloatResponse(self, response): 91 | self.assertIsInstance(response, rekcurd_pb2.ArrFloatOutput) 92 | 93 | def assertArrStringResponse(self, response): 94 | self.assertIsInstance(response, rekcurd_pb2.ArrStringOutput) 95 | 96 | def setUp(self): 97 | app.load_config_file("./test/test-settings.yml") 98 | app.data_server = DataServer(app.config) 99 | app.system_logger = JsonSystemLogger(config=app.config) 100 | app.service_logger = JsonServiceLogger(config=app.config) 101 | self._real_time = grpc_testing.strict_real_time() 102 | self._fake_time = grpc_testing.strict_fake_time(time.time()) 103 | servicer = RekcurdWorkerServicer(RekcurdPack(app, None)) 104 | descriptors_to_services = { 105 | target_service: servicer 106 | } 107 | self._real_time_server = grpc_testing.server_from_dictionary( 108 | descriptors_to_services, self._real_time) 109 | self._fake_time_server = grpc_testing.server_from_dictionary( 110 | descriptors_to_services, self._fake_time) 111 | 112 | @patch_predictor(Type.STRING, Type.STRING) 113 | def test_String_String(self): 114 | rpc = self._real_time_server.invoke_unary_unary( 115 | target_service.methods_by_name['Predict_String_String'], (), 116 | self.fake_string_request(), None) 117 | initial_metadata = rpc.initial_metadata() 118 | response, trailing_metadata, code, details = rpc.termination() 119 | self.assertIs(code, StatusCode.OK) 120 | self.assertStringResponse(response) 121 | 122 | @patch_predictor(Type.STRING, Type.STRING) 123 | def test_metadata(self): 124 | metadata = [('x-request-id', 'test'), ('dummy', 'dummy')] 125 | rpc = self._real_time_server.invoke_unary_unary( 126 | target_service.methods_by_name['Predict_String_String'], metadata, 127 | self.fake_string_request(), None) 128 | initial_metadata = rpc.initial_metadata() 129 | response, trailing_metadata, code, details = rpc.termination() 130 | self.assertIs(code, StatusCode.OK) 131 | self.assertStringResponse(response) 132 | 133 | @patch_predictor(Type.STRING, Type.BYTES) 134 | def test_String_Bytes(self): 135 | rpc = self._real_time_server.invoke_unary_stream( 136 | target_service.methods_by_name['Predict_String_Bytes'], (), 137 | self.fake_string_request(), None) 138 | initial_metadata = rpc.initial_metadata() 139 | trailing_metadata, code, details = rpc.termination() 140 | self.assertIs(code, StatusCode.OK) 141 | 142 | @patch_predictor(Type.STRING, Type.ARRAY_INT) 143 | def test_String_ArrInt(self): 144 | rpc = self._real_time_server.invoke_unary_unary( 145 | target_service.methods_by_name['Predict_String_ArrInt'], (), 146 | self.fake_string_request(), None) 147 | initial_metadata = rpc.initial_metadata() 148 | response, trailing_metadata, code, details = rpc.termination() 149 | self.assertIs(code, StatusCode.OK) 150 | self.assertArrIntResponse(response) 151 | 152 | @patch_predictor(Type.STRING, Type.ARRAY_FLOAT) 153 | def test_String_ArrFloat(self): 154 | rpc = self._real_time_server.invoke_unary_unary( 155 | target_service.methods_by_name['Predict_String_ArrFloat'], (), 156 | self.fake_string_request(), None) 157 | initial_metadata = rpc.initial_metadata() 158 | response, trailing_metadata, code, details = rpc.termination() 159 | self.assertIs(code, StatusCode.OK) 160 | self.assertArrFloatResponse(response) 161 | 162 | @patch_predictor(Type.STRING, Type.ARRAY_STRING) 163 | def test_String_ArrString(self): 164 | rpc = self._real_time_server.invoke_unary_unary( 165 | target_service.methods_by_name['Predict_String_ArrString'], (), 166 | self.fake_string_request(), None) 167 | initial_metadata = rpc.initial_metadata() 168 | response, trailing_metadata, code, details = rpc.termination() 169 | self.assertIs(code, StatusCode.OK) 170 | self.assertArrStringResponse(response) 171 | 172 | @patch_predictor(Type.BYTES, Type.STRING) 173 | def test_Bytes_String(self): 174 | rpc = self._real_time_server.invoke_stream_unary( 175 | target_service.methods_by_name['Predict_Bytes_String'], (), None) 176 | rpc.send_request(self.fake_bytes_request()) 177 | rpc.send_request(self.fake_bytes_request()) 178 | rpc.send_request(self.fake_bytes_request()) 179 | rpc.requests_closed() 180 | initial_metadata = rpc.initial_metadata() 181 | response, trailing_metadata, code, details = rpc.termination() 182 | self.assertIs(code, StatusCode.OK) 183 | self.assertStringResponse(response) 184 | 185 | @patch_predictor(Type.BYTES, Type.BYTES) 186 | def test_Bytes_Bytes(self): 187 | rpc = self._real_time_server.invoke_stream_stream( 188 | target_service.methods_by_name['Predict_Bytes_Bytes'], (), None) 189 | rpc.send_request(self.fake_bytes_request()) 190 | initial_metadata = rpc.initial_metadata() 191 | responses = [ 192 | rpc.take_response(), 193 | ] 194 | rpc.send_request(self.fake_bytes_request()) 195 | rpc.send_request(self.fake_bytes_request()) 196 | responses.extend([ 197 | rpc.take_response(), 198 | rpc.take_response(), 199 | ]) 200 | rpc.requests_closed() 201 | trailing_metadata, code, details = rpc.termination() 202 | self.assertIs(code, StatusCode.OK) 203 | for response in responses: 204 | self.assertBytesResponse(response) 205 | 206 | @patch_predictor(Type.BYTES, Type.ARRAY_INT) 207 | def test_Bytes_ArrInt(self): 208 | rpc = self._real_time_server.invoke_stream_unary( 209 | target_service.methods_by_name['Predict_Bytes_ArrInt'], (), None) 210 | rpc.send_request(self.fake_bytes_request()) 211 | rpc.send_request(self.fake_bytes_request()) 212 | rpc.send_request(self.fake_bytes_request()) 213 | rpc.requests_closed() 214 | initial_metadata = rpc.initial_metadata() 215 | response, trailing_metadata, code, details = rpc.termination() 216 | self.assertIs(code, StatusCode.OK) 217 | self.assertArrIntResponse(response) 218 | 219 | @patch_predictor(Type.BYTES, Type.ARRAY_FLOAT) 220 | def test_Bytes_ArrFloat(self): 221 | rpc = self._real_time_server.invoke_stream_unary( 222 | target_service.methods_by_name['Predict_Bytes_ArrFloat'], (), None) 223 | rpc.send_request(self.fake_bytes_request()) 224 | rpc.send_request(self.fake_bytes_request()) 225 | rpc.send_request(self.fake_bytes_request()) 226 | rpc.requests_closed() 227 | initial_metadata = rpc.initial_metadata() 228 | response, trailing_metadata, code, details = rpc.termination() 229 | self.assertIs(code, StatusCode.OK) 230 | self.assertArrFloatResponse(response) 231 | 232 | @patch_predictor(Type.BYTES, Type.ARRAY_STRING) 233 | def test_Bytes_ArrString(self): 234 | rpc = self._real_time_server.invoke_stream_unary( 235 | target_service.methods_by_name['Predict_Bytes_ArrString'], (), None) 236 | rpc.send_request(self.fake_bytes_request()) 237 | rpc.send_request(self.fake_bytes_request()) 238 | rpc.send_request(self.fake_bytes_request()) 239 | rpc.requests_closed() 240 | initial_metadata = rpc.initial_metadata() 241 | response, trailing_metadata, code, details = rpc.termination() 242 | self.assertIs(code, StatusCode.OK) 243 | self.assertArrStringResponse(response) 244 | 245 | @patch_predictor(Type.ARRAY_INT, Type.STRING) 246 | def test_ArrInt_String(self): 247 | rpc = self._real_time_server.invoke_unary_unary( 248 | target_service.methods_by_name['Predict_ArrInt_String'], (), 249 | self.fake_arrint_request(), None) 250 | initial_metadata = rpc.initial_metadata() 251 | response, trailing_metadata, code, details = rpc.termination() 252 | self.assertIs(code, StatusCode.OK) 253 | self.assertStringResponse(response) 254 | 255 | @patch_predictor(Type.ARRAY_INT, Type.BYTES) 256 | def test_ArrInt_Bytes(self): 257 | rpc = self._real_time_server.invoke_unary_stream( 258 | target_service.methods_by_name['Predict_ArrInt_Bytes'], (), 259 | self.fake_arrint_request(), None) 260 | initial_metadata = rpc.initial_metadata() 261 | trailing_metadata, code, details = rpc.termination() 262 | self.assertIs(code, StatusCode.OK) 263 | 264 | @patch_predictor(Type.ARRAY_INT, Type.ARRAY_INT) 265 | def test_ArrInt_ArrInt(self): 266 | rpc = self._real_time_server.invoke_unary_unary( 267 | target_service.methods_by_name['Predict_ArrInt_ArrInt'], (), 268 | self.fake_arrint_request(), None) 269 | initial_metadata = rpc.initial_metadata() 270 | response, trailing_metadata, code, details = rpc.termination() 271 | self.assertIs(code, StatusCode.OK) 272 | self.assertArrIntResponse(response) 273 | 274 | @patch_predictor(Type.ARRAY_INT, Type.ARRAY_FLOAT) 275 | def test_ArrInt_ArrFloat(self): 276 | rpc = self._real_time_server.invoke_unary_unary( 277 | target_service.methods_by_name['Predict_ArrInt_ArrFloat'], (), 278 | self.fake_arrint_request(), None) 279 | initial_metadata = rpc.initial_metadata() 280 | response, trailing_metadata, code, details = rpc.termination() 281 | self.assertIs(code, StatusCode.OK) 282 | self.assertArrFloatResponse(response) 283 | 284 | @patch_predictor(Type.ARRAY_INT, Type.ARRAY_STRING) 285 | def test_ArrInt_ArrString(self): 286 | rpc = self._real_time_server.invoke_unary_unary( 287 | target_service.methods_by_name['Predict_ArrInt_ArrString'], (), 288 | self.fake_arrint_request(), None) 289 | initial_metadata = rpc.initial_metadata() 290 | response, trailing_metadata, code, details = rpc.termination() 291 | self.assertIs(code, StatusCode.OK) 292 | self.assertArrStringResponse(response) 293 | 294 | @patch_predictor(Type.ARRAY_FLOAT, Type.STRING) 295 | def test_ArrFloat_String(self): 296 | rpc = self._real_time_server.invoke_unary_unary( 297 | target_service.methods_by_name['Predict_ArrFloat_String'], (), 298 | self.fake_arrfloat_request(), None) 299 | initial_metadata = rpc.initial_metadata() 300 | response, trailing_metadata, code, details = rpc.termination() 301 | self.assertIs(code, StatusCode.OK) 302 | self.assertStringResponse(response) 303 | 304 | @patch_predictor(Type.ARRAY_FLOAT, Type.BYTES) 305 | def test_ArrFloat_Bytes(self): 306 | rpc = self._real_time_server.invoke_unary_stream( 307 | target_service.methods_by_name['Predict_ArrFloat_Bytes'], (), 308 | self.fake_arrfloat_request(), None) 309 | initial_metadata = rpc.initial_metadata() 310 | trailing_metadata, code, details = rpc.termination() 311 | self.assertIs(code, StatusCode.OK) 312 | 313 | @patch_predictor(Type.ARRAY_FLOAT, Type.ARRAY_INT) 314 | def test_ArrFloat_ArrInt(self): 315 | rpc = self._real_time_server.invoke_unary_unary( 316 | target_service.methods_by_name['Predict_ArrFloat_ArrInt'], (), 317 | self.fake_arrfloat_request(), None) 318 | initial_metadata = rpc.initial_metadata() 319 | response, trailing_metadata, code, details = rpc.termination() 320 | self.assertIs(code, StatusCode.OK) 321 | self.assertArrIntResponse(response) 322 | 323 | @patch_predictor(Type.ARRAY_FLOAT, Type.ARRAY_FLOAT) 324 | def test_ArrFloat_ArrFloat(self): 325 | rpc = self._real_time_server.invoke_unary_unary( 326 | target_service.methods_by_name['Predict_ArrFloat_ArrFloat'], (), 327 | self.fake_arrfloat_request(), None) 328 | initial_metadata = rpc.initial_metadata() 329 | response, trailing_metadata, code, details = rpc.termination() 330 | self.assertIs(code, StatusCode.OK) 331 | self.assertArrFloatResponse(response) 332 | 333 | @patch_predictor(Type.ARRAY_FLOAT, Type.ARRAY_STRING) 334 | def test_ArrFloat_ArrString(self): 335 | rpc = self._real_time_server.invoke_unary_unary( 336 | target_service.methods_by_name['Predict_ArrFloat_ArrString'], (), 337 | self.fake_arrfloat_request(), None) 338 | initial_metadata = rpc.initial_metadata() 339 | response, trailing_metadata, code, details = rpc.termination() 340 | self.assertIs(code, StatusCode.OK) 341 | self.assertArrStringResponse(response) 342 | 343 | @patch_predictor(Type.ARRAY_STRING, Type.STRING) 344 | def test_ArrString_String(self): 345 | rpc = self._real_time_server.invoke_unary_unary( 346 | target_service.methods_by_name['Predict_ArrString_String'], (), 347 | self.fake_arrstring_request(), None) 348 | initial_metadata = rpc.initial_metadata() 349 | response, trailing_metadata, code, details = rpc.termination() 350 | self.assertIs(code, StatusCode.OK) 351 | self.assertStringResponse(response) 352 | 353 | @patch_predictor(Type.ARRAY_STRING, Type.BYTES) 354 | def test_ArrString_Bytes(self): 355 | rpc = self._real_time_server.invoke_unary_stream( 356 | target_service.methods_by_name['Predict_ArrString_Bytes'], (), 357 | self.fake_arrstring_request(), None) 358 | initial_metadata = rpc.initial_metadata() 359 | trailing_metadata, code, details = rpc.termination() 360 | self.assertIs(code, StatusCode.OK) 361 | 362 | @patch_predictor(Type.ARRAY_STRING, Type.ARRAY_INT) 363 | def test_ArrString_ArrInt(self): 364 | rpc = self._real_time_server.invoke_unary_unary( 365 | target_service.methods_by_name['Predict_ArrString_ArrInt'], (), 366 | self.fake_arrstring_request(), None) 367 | initial_metadata = rpc.initial_metadata() 368 | response, trailing_metadata, code, details = rpc.termination() 369 | self.assertIs(code, StatusCode.OK) 370 | self.assertArrIntResponse(response) 371 | 372 | @patch_predictor(Type.ARRAY_STRING, Type.ARRAY_FLOAT) 373 | def test_ArrString_ArrFloat(self): 374 | rpc = self._real_time_server.invoke_unary_unary( 375 | target_service.methods_by_name['Predict_ArrString_ArrFloat'], (), 376 | self.fake_arrstring_request(), None) 377 | initial_metadata = rpc.initial_metadata() 378 | response, trailing_metadata, code, details = rpc.termination() 379 | self.assertIs(code, StatusCode.OK) 380 | self.assertArrFloatResponse(response) 381 | 382 | @patch_predictor(Type.ARRAY_STRING, Type.ARRAY_STRING) 383 | def test_ArrString_ArrString(self): 384 | rpc = self._real_time_server.invoke_unary_unary( 385 | target_service.methods_by_name['Predict_ArrString_ArrString'], (), 386 | self.fake_arrstring_request(), None) 387 | initial_metadata = rpc.initial_metadata() 388 | response, trailing_metadata, code, details = rpc.termination() 389 | self.assertIs(code, StatusCode.OK) 390 | self.assertArrStringResponse(response) 391 | -------------------------------------------------------------------------------- /test/data_servers/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from unittest.mock import Mock, patch, mock_open 3 | 4 | 5 | def patch_predictor(): 6 | def test_method(func): 7 | @wraps(func) 8 | def inner_method(*args, **kwargs): 9 | with patch('rekcurd.data_servers.LocalHandler.upload', 10 | new=Mock(return_value=None)) as _, \ 11 | patch('rekcurd.data_servers.CephHandler.download', 12 | new=Mock(return_value=None)) as _, \ 13 | patch('rekcurd.data_servers.CephHandler.upload', 14 | new=Mock(return_value=None)) as _, \ 15 | patch('rekcurd.data_servers.AwsS3Handler.download', 16 | new=Mock(return_value=None)) as _, \ 17 | patch('rekcurd.data_servers.AwsS3Handler.upload', 18 | new=Mock(return_value=None)) as _, \ 19 | patch('rekcurd.data_servers.GcsHandler.download', 20 | new=Mock(return_value=None)) as _, \ 21 | patch('rekcurd.data_servers.GcsHandler.upload', 22 | new=Mock(return_value=None)) as _, \ 23 | patch('builtins.open', new_callable=mock_open) as _: 24 | return func(*args, **kwargs) 25 | return inner_method 26 | return test_method 27 | -------------------------------------------------------------------------------- /test/data_servers/test_aws_s3_handler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rekcurd.utils import RekcurdConfig, ModelModeEnum 4 | from rekcurd.data_servers import AwsS3Handler 5 | 6 | from . import patch_predictor 7 | 8 | 9 | class AwsS3HandlerTest(unittest.TestCase): 10 | """Tests for AwsS3HandlerTest. 11 | """ 12 | 13 | def setUp(self): 14 | config = RekcurdConfig("./test/test-settings.yml") 15 | config.set_configurations( 16 | model_mode=ModelModeEnum.AWS_S3.value, aws_access_key="xxx", 17 | aws_secret_key="xxx", aws_bucket_name="xxx") 18 | self.handler = AwsS3Handler(config) 19 | 20 | @patch_predictor() 21 | def test_download(self): 22 | self.assertIsNone(self.handler.download("remote","local")) 23 | 24 | @patch_predictor() 25 | def test_upload(self): 26 | self.assertIsNone(self.handler.upload("remote","local")) 27 | -------------------------------------------------------------------------------- /test/data_servers/test_ceph_handler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rekcurd.utils import RekcurdConfig, ModelModeEnum 4 | from rekcurd.data_servers import CephHandler 5 | 6 | from . import patch_predictor 7 | 8 | 9 | class CephHandlerTest(unittest.TestCase): 10 | """Tests for CephHandlerTest. 11 | """ 12 | 13 | def setUp(self): 14 | config = RekcurdConfig("./test/test-settings.yml") 15 | config.set_configurations( 16 | model_mode=ModelModeEnum.CEPH_S3.value, ceph_access_key="xxx", 17 | ceph_secret_key="xxx", ceph_host="127.0.0.1", ceph_port=443, 18 | ceph_is_secure=True, ceph_bucket_name="xxx") 19 | self.handler = CephHandler(config) 20 | 21 | @patch_predictor() 22 | def test_download(self): 23 | self.assertIsNone(self.handler.download("remote","local")) 24 | 25 | @patch_predictor() 26 | def test_upload(self): 27 | self.assertIsNone(self.handler.upload("remote","local")) 28 | -------------------------------------------------------------------------------- /test/data_servers/test_data_server.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rekcurd.protobuf import rekcurd_pb2 4 | from rekcurd.utils import RekcurdConfig, ModelModeEnum, EvaluateResultDetail, EvaluateResult, PredictResult 5 | from rekcurd.data_servers import DataServer 6 | 7 | from . import patch_predictor 8 | 9 | 10 | class DataServerTest(unittest.TestCase): 11 | """Tests for DataServer. 12 | """ 13 | 14 | def setUp(self): 15 | config = RekcurdConfig("./test/test-settings.yml") 16 | config.set_configurations( 17 | model_mode=ModelModeEnum.CEPH_S3.value, ceph_access_key="xxx", 18 | ceph_secret_key="xxx", ceph_host="127.0.0.1", ceph_port=443, 19 | ceph_is_secure=True, ceph_bucket_name="xxx") 20 | self.data_server = DataServer(config) 21 | 22 | @patch_predictor() 23 | def test_get_model_path(self): 24 | self.assertEqual(self.data_server.get_model_path(), "rekcurd-model/test/model/default.model") 25 | 26 | @patch_predictor() 27 | def test_get_model_path(self): 28 | self.assertEqual(self.data_server.switch_model("test/model/switch.model"), "rekcurd-model/test/model/switch.model") 29 | 30 | def __get_UploadModelRequest(self, path: str): 31 | yield rekcurd_pb2.UploadModelRequest(path=path, data=b'data') 32 | 33 | @patch_predictor() 34 | def test_upload_model(self): 35 | request_iterator = self.__get_UploadModelRequest('my_path') 36 | self.assertEqual(self.data_server.upload_model(request_iterator), 'rekcurd-model/test/model/my_path') 37 | 38 | @patch_predictor() 39 | def test_upload_model_invalid_path(self): 40 | request_iterator = self.__get_UploadModelRequest('../my_path') 41 | with self.assertRaises(Exception): 42 | self.data_server.upload_model(request_iterator) 43 | 44 | @patch_predictor() 45 | def test_get_evaluation_data_path(self): 46 | self.assertEqual(self.data_server.get_evaluation_data_path("test/eval/data"), "rekcurd-eval/data") 47 | 48 | @patch_predictor() 49 | def test_get_eval_result_detail(self): 50 | self.assertEqual(self.data_server.get_eval_result_detail("test/eval/detail.pkl"), "rekcurd-eval/detail.pkl") 51 | 52 | def __get_UploadEvaluationDataRequest(self, data_path: str): 53 | yield rekcurd_pb2.UploadEvaluationDataRequest(data_path=data_path, data=b'data') 54 | 55 | @patch_predictor() 56 | def test_upload_evaluation_data(self): 57 | request_iterator = self.__get_UploadEvaluationDataRequest('my_path') 58 | self.assertEqual(self.data_server.upload_evaluation_data(request_iterator), "rekcurd-eval/my_path") 59 | 60 | @patch_predictor() 61 | def test_upload_evaluation_data_invalid_path(self): 62 | request_iterator = self.__get_UploadEvaluationDataRequest('../my_path') 63 | with self.assertRaises(Exception): 64 | self.data_server.upload_evaluation_data(request_iterator) 65 | 66 | @patch_predictor() 67 | def test_upload_evaluation_result(self): 68 | eval_res = EvaluateResult(1, 0.4, [0.5], [0.5], [0.5], [0.5], ['label']) 69 | 70 | def generate_eval(): 71 | for _ in range(2): 72 | yield EvaluateResultDetail(result=PredictResult('label', 0.5), 73 | is_correct=True) 74 | return eval_res 75 | self.assertEqual(self.data_server.upload_evaluation_result(generate_eval(), "test/eval/data"), eval_res) 76 | 77 | @patch_predictor() 78 | def test_upload_evaluation_result_invalid_path(self): 79 | with self.assertRaises(Exception): 80 | self.data_server.upload_evaluation_result_summary(b'hoge', "test/../data") 81 | -------------------------------------------------------------------------------- /test/data_servers/test_gcs_handler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rekcurd.utils import RekcurdConfig, ModelModeEnum 4 | from rekcurd.data_servers import GcsHandler 5 | 6 | from . import patch_predictor 7 | 8 | 9 | class GcsHandlerTest(unittest.TestCase): 10 | """Tests for GcsHandlerTest. 11 | """ 12 | 13 | def setUp(self): 14 | config = RekcurdConfig("./test/test-settings.yml") 15 | config.set_configurations( 16 | model_mode=ModelModeEnum.GCS.value, gcs_access_key="xxx", 17 | gcs_secret_key="xxx", gcs_bucket_name="xxx") 18 | self.handler = GcsHandler(config) 19 | 20 | @patch_predictor() 21 | def test_download(self): 22 | self.assertIsNone(self.handler.download("remote","local")) 23 | 24 | @patch_predictor() 25 | def test_upload(self): 26 | self.assertIsNone(self.handler.upload("remote","local")) 27 | -------------------------------------------------------------------------------- /test/data_servers/test_local_handler.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from rekcurd.utils import RekcurdConfig 4 | from rekcurd.data_servers import LocalHandler 5 | 6 | from . import patch_predictor 7 | 8 | 9 | class LocalHandlerTest(unittest.TestCase): 10 | """Tests for LocalHandler. 11 | """ 12 | 13 | def setUp(self): 14 | config = RekcurdConfig("./test/test-settings.yml") 15 | self.handler = LocalHandler(config) 16 | 17 | @patch_predictor() 18 | def test_download(self): 19 | with self.assertRaises(Exception): 20 | self.handler.download("remote","local") 21 | 22 | @patch_predictor() 23 | def test_upload(self): 24 | self.assertIsNone(self.handler.upload("remote","local")) 25 | -------------------------------------------------------------------------------- /test/logger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rekcurd/rekcurd-python/71e1ecd1f7bf8394563e682308fa7b404492e6da/test/logger/__init__.py -------------------------------------------------------------------------------- /test/logger/test_logger_fluent.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import Mock 3 | 4 | from rekcurd.utils import RekcurdConfig 5 | from rekcurd.logger import FluentSystemLogger, FluentServiceLogger 6 | 7 | 8 | class FluentSystemLoggerTest(unittest.TestCase): 9 | """Tests for FluentSystemLogger. 10 | """ 11 | 12 | def setUp(self): 13 | self.logger = FluentSystemLogger(config=RekcurdConfig()) 14 | 15 | def test_exception(self): 16 | self.assertIsNone(self.logger.exception("Exception")) 17 | 18 | def test_error(self): 19 | self.assertIsNone(self.logger.error("Error")) 20 | 21 | def test_debug(self): 22 | self.assertIsNone(self.logger.debug("Debug")) 23 | 24 | def test_info(self): 25 | self.assertIsNone(self.logger.info("Info")) 26 | 27 | def test_warn(self): 28 | self.assertIsNone(self.logger.warn("Warn")) 29 | 30 | 31 | class FluentServiceLoggerTest(unittest.TestCase): 32 | """Tests for FluentServiceLogger. 33 | """ 34 | 35 | def setUp(self): 36 | self.request = Mock() 37 | self.request.input = "input" 38 | self.request.option = Mock() 39 | self.request.option.val = "request-option" 40 | self.response = Mock() 41 | self.response.output = "output" 42 | self.response.score = 0.0 43 | self.response.option = Mock() 44 | self.response.option.val = "response-option" 45 | self.logger = FluentServiceLogger(config=RekcurdConfig()) 46 | 47 | def test_emit(self): 48 | self.assertIsNone(self.logger.emit(self.request, self.response)) 49 | 50 | def test_emit_suppressed(self): 51 | self.assertIsNone(self.logger.emit(self.request, self.response, suppress_log_inout=True)) 52 | -------------------------------------------------------------------------------- /test/logger/test_logger_jsonlogger.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import Mock 3 | 4 | from rekcurd.utils import RekcurdConfig 5 | from rekcurd.logger import JsonSystemLogger, JsonServiceLogger 6 | 7 | 8 | class JsonSystemLoggerTest(unittest.TestCase): 9 | """Tests for JsonSystemLogger. 10 | """ 11 | 12 | def setUp(self): 13 | self.logger = JsonSystemLogger(config=RekcurdConfig()) 14 | 15 | def test_exception(self): 16 | self.assertIsNone(self.logger.exception("Exception")) 17 | 18 | def test_error(self): 19 | self.assertIsNone(self.logger.error("Error")) 20 | 21 | def test_debug(self): 22 | self.assertIsNone(self.logger.debug("Debug")) 23 | 24 | def test_info(self): 25 | self.assertIsNone(self.logger.info("Info")) 26 | 27 | def test_warn(self): 28 | self.assertIsNone(self.logger.warn("Warn")) 29 | 30 | 31 | class JsonServiceLoggerTest(unittest.TestCase): 32 | """Tests for JsonServiceLogger. 33 | """ 34 | 35 | def setUp(self): 36 | self.request = Mock() 37 | self.request.input = "input" 38 | self.request.option = Mock() 39 | self.request.option.val = "request-option" 40 | self.response = Mock() 41 | self.response.output = "output" 42 | self.response.score = 0.0 43 | self.response.option = Mock() 44 | self.response.option.val = "response-option" 45 | self.logger = JsonServiceLogger(config=RekcurdConfig()) 46 | 47 | def test_emit(self): 48 | self.assertIsNone(self.logger.emit(self.request, self.response)) 49 | 50 | def test_emit_suppressed(self): 51 | self.assertIsNone(self.logger.emit(self.request, self.response, suppress_log_inout=True)) 52 | -------------------------------------------------------------------------------- /test/test-settings.yml: -------------------------------------------------------------------------------- 1 | debug: True 2 | app: 3 | name: test 4 | port: 5000 5 | service_level: development 6 | model: 7 | mode: local 8 | local: 9 | filepath: test/model/default.model 10 | -------------------------------------------------------------------------------- /test/test_app.py: -------------------------------------------------------------------------------- 1 | ../rekcurd/template/app.py-tpl -------------------------------------------------------------------------------- /test/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rekcurd/rekcurd-python/71e1ecd1f7bf8394563e682308fa7b404492e6da/test/utils/__init__.py -------------------------------------------------------------------------------- /test/utils/test_rekcurd_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | from rekcurd.utils import RekcurdConfig, ModelModeEnum 5 | 6 | 7 | class RekcurdConfigTest(unittest.TestCase): 8 | """Tests for RekcurdConfig. 9 | """ 10 | 11 | def test_load_from_file(self): 12 | config = RekcurdConfig("./test/test-settings.yml") 13 | self.assertEqual(config.DEBUG_MODE, True) 14 | self.assertEqual(config.APPLICATION_NAME, "test") 15 | self.assertEqual(config.MODEL_MODE_ENUM, ModelModeEnum.LOCAL) 16 | 17 | def test_load_from_env(self): 18 | os.environ["REKCURD_KUBERNETES_MODE"] = "True" 19 | os.environ["REKCURD_DEBUG_MODE"] = "False" 20 | os.environ["REKCURD_APPLICATION_NAME"] = "test2" 21 | os.environ["REKCURD_MODEL_MODE"] = ModelModeEnum.CEPH_S3.value 22 | config = RekcurdConfig("./test/test-settings.yml") 23 | self.assertEqual(config.DEBUG_MODE, False) 24 | self.assertEqual(config.APPLICATION_NAME, "test2") 25 | self.assertEqual(config.MODEL_MODE_ENUM, ModelModeEnum.CEPH_S3) 26 | del os.environ["REKCURD_KUBERNETES_MODE"] 27 | del os.environ["REKCURD_DEBUG_MODE"] 28 | del os.environ["REKCURD_APPLICATION_NAME"] 29 | del os.environ["REKCURD_MODEL_MODE"] 30 | 31 | def test_set_configurations(self): 32 | config = RekcurdConfig("./test/test-settings.yml") 33 | config.set_configurations(debug_mode=False, application_name="test3", model_mode=ModelModeEnum.AWS_S3.value) 34 | self.assertEqual(config.DEBUG_MODE, False) 35 | self.assertEqual(config.APPLICATION_NAME, "test3") 36 | self.assertEqual(config.MODEL_MODE_ENUM, ModelModeEnum.AWS_S3) 37 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36, py37 3 | 4 | [testenv] 5 | passenv = TOXENV CI TRAVIS TRAVIS_* 6 | usedevelop = True 7 | install_command = pip install -U {opts} {packages} 8 | deps = -r{toxinidir}/test-requirements.txt 9 | -r{toxinidir}/requirements.txt 10 | commands = 11 | python -V 12 | py.test -vvv -s 13 | # Patch for https://github.com/travis-ci/travis-ci/issues/7940 14 | setenv = 15 | BOTO_CONFIG = /dev/null 16 | 17 | [testenv:coverage] 18 | commands = 19 | python -V 20 | nosetests --with-coverage --cover-package=rekcurd.core,rekcurd.data_servers,rekcurd.logger,rekcurd.utils --cover-tests 21 | 22 | [testenv:codecov] 23 | commands = 24 | codecov 25 | --------------------------------------------------------------------------------