├── .github ├── ISSUE_TEMPLATE │ └── tabpy-issue-report.md ├── PULL_REQUEST_TEMPLATE │ └── pull_request_template.md └── workflows │ ├── coverage.yml │ ├── docker-publish.yml │ ├── lint.yml │ ├── pull_request.yml │ └── push.yml ├── .gitignore ├── .pep8speaks.yml ├── .scrutinizer.yml ├── CHANGELOG ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── Procfile ├── README.md ├── _config.yml ├── app.json ├── codeql-analysis.yml ├── docs ├── TableauConfiguration.md ├── about.md ├── deploy-to-heroku.md ├── img │ ├── Example1-SimpleFunctionCall.png │ ├── Example2-MultipleFunctionCalls.png │ └── python-calculated-field.png ├── security.md ├── server-config.md ├── server-install.md ├── server-rest.md ├── tabpy-tools.md └── tabpy-virtualenv.md ├── misc ├── TabPy.postman_collection.json └── TabPy.yml ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── start.sh ├── tabpy ├── VERSION ├── __init__.py ├── models │ ├── __init__.py │ ├── deploy_models.py │ ├── scripts │ │ ├── ANOVA.py │ │ ├── PCA.py │ │ ├── SentimentAnalysis.py │ │ ├── __init__.py │ │ └── tTest.py │ └── utils │ │ ├── __init__.py │ │ └── setup_utils.py ├── tabpy.py ├── tabpy_server │ ├── __init__.py │ ├── app │ │ ├── __init__.py │ │ ├── app.py │ │ ├── app_parameters.py │ │ ├── arrow_server.py │ │ └── util.py │ ├── common │ │ ├── __init__.py │ │ ├── default.conf │ │ ├── endpoint_file_mgr.py │ │ ├── messages.py │ │ └── util.py │ ├── handlers │ │ ├── __init__.py │ │ ├── base_handler.py │ │ ├── basic_auth_server_middleware_factory.py │ │ ├── endpoint_handler.py │ │ ├── endpoints_handler.py │ │ ├── evaluation_plane_handler.py │ │ ├── management_handler.py │ │ ├── no_op_auth_handler.py │ │ ├── query_plane_handler.py │ │ ├── service_info_handler.py │ │ ├── status_handler.py │ │ ├── upload_destination_handler.py │ │ └── util.py │ ├── management │ │ ├── __init__.py │ │ ├── state.py │ │ └── util.py │ ├── psws │ │ ├── __init__.py │ │ ├── callbacks.py │ │ └── python_service.py │ ├── state.ini.template │ └── static │ │ ├── TabPy_logo.png │ │ ├── index.html │ │ └── tableau.png ├── tabpy_tools │ ├── __init__.py │ ├── client.py │ ├── custom_query_object.py │ ├── query_object.py │ ├── rest.py │ ├── rest_client.py │ └── schema.py └── utils │ ├── __init__.py │ └── tabpy_user.py └── tests ├── __init__.py ├── integration ├── __init__.py ├── integ_test_base.py ├── resources │ ├── 2019_04_24_to_3018_08_25.crt │ ├── 2019_04_24_to_3018_08_25.key │ ├── data.csv │ ├── deploy_and_evaluate_model.conf │ ├── deploy_and_evaluate_model_auth.conf │ └── pwdfile.txt ├── test_arrow_server.py ├── test_auth.py ├── test_custom_evaluate_timeout.py ├── test_deploy_and_evaluate_model.py ├── test_deploy_and_evaluate_model_auth_on.py ├── test_deploy_and_evaluate_model_ssl.py ├── test_deploy_model_ssl_off_auth_off.py ├── test_deploy_model_ssl_off_auth_on.py ├── test_deploy_model_ssl_on_auth_off.py ├── test_deploy_model_ssl_on_auth_on.py ├── test_evaluate.py ├── test_gzip.py ├── test_minimum_tls_version.py ├── test_url.py └── test_url_ssl.py └── unit ├── __init__.py ├── server_tests ├── __init__.py ├── resources │ ├── expired.crt │ ├── future.crt │ └── valid.crt ├── test_config.py ├── test_endpoint_file_manager.py ├── test_endpoint_handler.py ├── test_endpoints_handler.py ├── test_evaluation_plane_handler.py ├── test_pwd_file.py └── test_service_info_handler.py └── tools_tests ├── __init__.py ├── test_client.py ├── test_rest.py ├── test_rest_object.py └── test_schema.py /.github/ISSUE_TEMPLATE/tabpy-issue-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: TabPy Issue Report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Environment information:** 11 | - OS: [Windows, Linux, Mac] and version 12 | - Python version: [e.g. 3.6.5] 13 | - TabPy release: [e.g. 0.4.1] 14 | 15 | **Describe the issue** 16 | A clear and concise description of what the issue is. 17 | 18 | **To Reproduce** 19 | Steps to reproduce the behavior: what commands to run, what files to modify, where to look for an error. 20 | 21 | **Expected behavior** 22 | A clear and concise description of what you expected to happen. 23 | 24 | **Screenshots** 25 | If applicable, add screenshots to help explain your problem. 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 4 | 5 | Fixes # (issue) 6 | 7 | ## Type of change 8 | 9 | Please delete options that are not relevant. 10 | 11 | - [ ] Bug fix (non-breaking change which fixes an issue) 12 | - [ ] New feature (non-breaking change which adds functionality) 13 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 14 | - [ ] This change requires a documentation update 15 | 16 | # How Has This Been Tested? 17 | 18 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration 19 | 20 | - [ ] Test A 21 | - [ ] Test B 22 | 23 | **Test Configuration**: 24 | * OS and version: 25 | * Python version: 26 | 27 | # Checklist: 28 | 29 | - [ ] My code follows the style guidelines of this project 30 | - [ ] I have performed a self-review of my own code 31 | - [ ] I have commented my code, particularly in hard-to-understand areas 32 | - [ ] I have made corresponding changes to the documentation 33 | - [ ] My changes generate no new warnings 34 | - [ ] I have added tests that prove my fix is effective or that my feature works 35 | - [ ] New and existing unit tests pass locally with my changes 36 | - [ ] Any dependent changes have been merged and published in downstream modules 37 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Code coverage 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: ${{ matrix.python-version }} on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | 10 | strategy: 11 | matrix: 12 | python-version: [3.12] 13 | os: [ubuntu-latest] 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | pip install -r requirements_dev.txt 28 | 29 | - name: Test with pytest 30 | run: 31 | pytest tests --cov=tabpy --cov-config=setup.cfg 32 | env: 33 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 34 | 35 | - name: Run coveralls 36 | run: 37 | coveralls --service=github 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker Image for Release 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | env: 8 | IMAGE_NAME: tabpy 9 | 10 | jobs: 11 | # Push image to GitHub Packages. 12 | # See also https://docs.docker.com/docker-hub/builds/ 13 | push: 14 | name: Push Docker image to GitHub Packages 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout the repo 19 | uses: actions/checkout@v2 20 | 21 | - name: Build image 22 | run: docker build . --file Dockerfile --tag $IMAGE_NAME 23 | 24 | - name: Log into GitHub Container Registry 25 | run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin 26 | 27 | - name: Push image to GitHub Container Registry 28 | run: | 29 | IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME 30 | 31 | # Change all uppercase to lowercase 32 | IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') 33 | 34 | # Strip git ref prefix from version 35 | VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') 36 | 37 | # Strip "v" prefix from tag name 38 | [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') 39 | 40 | # Use Docker `latest` tag convention 41 | [ "$VERSION" == "master" ] && VERSION=latest 42 | 43 | echo IMAGE_ID=$IMAGE_ID 44 | echo VERSION=$VERSION 45 | 46 | docker tag $IMAGE_NAME $IMAGE_ID:$VERSION 47 | docker push $IMAGE_ID:$VERSION 48 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint: 7 | name: ${{ matrix.python-version }} on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | 10 | strategy: 11 | matrix: 12 | python-version: [3.12] 13 | os: [ubuntu-latest] 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v1 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | pip install -r requirements_dev.txt 28 | 29 | - name: Lint with flake8 30 | run: | 31 | # stop the build if there are Python syntax errors or undefined names 32 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 33 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 34 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 35 | 36 | - name: Markdownlint 37 | uses: nosborn/github-action-markdown-cli@v1.1.1 38 | with: 39 | files: . 40 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | name: Test Run on Pull Request 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | name: ${{ matrix.python-version }} on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | 10 | strategy: 11 | matrix: 12 | python-version: ['3.9', '3.10', '3.11', '3.12'] 13 | # TODO: switch macos-13 to macos-latest 14 | os: [ubuntu-latest, windows-latest, macos-13] 15 | 16 | steps: 17 | - uses: actions/checkout@v1 18 | 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements.txt 28 | pip install -r requirements_dev.txt 29 | 30 | - name: Test with pytest 31 | run: | 32 | pytest tests 33 | -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: Test Run on Push 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | name: ${{ matrix.python-version }} on ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | 10 | strategy: 11 | matrix: 12 | python-version: ['3.9', '3.10', '3.11', '3.12'] 13 | # TODO: switch macos-13 to macos-latest 14 | os: [ubuntu-latest, windows-latest, macos-13] 15 | 16 | steps: 17 | - uses: actions/checkout@v1 18 | 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements.txt 28 | pip install -r requirements_dev.txt 29 | 30 | - name: Test with pytest 31 | run: | 32 | pytest tests 33 | 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.pyc 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | *_trial_temp* 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | # NodeJS files and folders 109 | node_modules/ 110 | package.json 111 | package-lock.json 112 | 113 | # OS generated 114 | .DS_Store 115 | 116 | # PyCharm 117 | .idea/ 118 | 119 | # TabPy server artifacts 120 | tabpy/tabpy_server/state.ini 121 | tabpy/tabpy_server/query_objects 122 | tabpy/tabpy_server/staging 123 | 124 | # VS Code 125 | *.code-workspace 126 | .vscode/ 127 | 128 | # etc 129 | setup.bat 130 | *~ 131 | tabpy_log.log.* 132 | -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | # File : .pep8speaks.yml 2 | 3 | flake8: 4 | max-line-length: 98 5 | -------------------------------------------------------------------------------- /.scrutinizer.yml: -------------------------------------------------------------------------------- 1 | build: 2 | environment: 3 | python: 3.12.5 4 | nodes: 5 | coverage: 6 | project_setup: 7 | override: 8 | - pip install -r requirements.txt 9 | - pip install -r requirements_dev.txt 10 | tests: 11 | override: 12 | - command: 'pytest tests --cov=tabpy --cov-config=setup.cfg' 13 | idle_timeout: 600 14 | coverage: 15 | file: '.coverage' 16 | config_file: 'setup.cfg' 17 | format: 'py-cc' 18 | analysis: 19 | project_setup: 20 | override: 21 | - pip install -r requirements.txt 22 | tests: 23 | override: [py-scrutinizer-run] 24 | dependencies: 25 | override: 26 | - pip install . 27 | tests: 28 | before: 29 | - pip install -r requirements.txt 30 | override: 31 | pytest: 32 | idle_timeout: 600 33 | checks: 34 | python: 35 | code_rating: true 36 | duplicate_code: true 37 | filter: 38 | excluded_paths: 39 | - '*/tests/*' 40 | dependency_paths: 41 | - 'lib/*' 42 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v2.13.0 4 | 5 | ### Improvements 6 | 7 | - Add support for deploying functions to a remote TabPy server by setting 8 | `remote_server=True` when creating the Client instance. 9 | 10 | ## v2.12.0 11 | 12 | ### Improvements 13 | 14 | - Add support for public deployed functions that will be visible to users 15 | in Tableau when using the Custom Functions Explorer 16 | - Add functionality to allow users to update existing deployed functions 17 | without needing to redeploy the function itself 18 | 19 | ## v2.11.0 20 | 21 | ### Improvements 22 | 23 | - Add support for Python 3.10, 3.11, and 3.12. End support Python for 24 | 3.7 and 3.8. 25 | 26 | ## v2.10.0 27 | 28 | ### Improvements 29 | 30 | - Add TabPy parameter (TABPY_MINIMUM_TLS_VERSION) to specify the minimum TLS 31 | version that the server will accept for secure connections. Default is 32 | set to TLSv1_2. 33 | 34 | ## v2.9.0 35 | 36 | ### Improvements 37 | 38 | - Require confirmation to continue when starting TabPy without authentication, 39 | with a warning that this is an insecure state and not recommended. 40 | 41 | ## v2.8.0 42 | 43 | ### Improvements 44 | 45 | - Returns 413 error code when request payload exceeds 46 | TABPY_MAX_REQUEST_SIZE_MB config setting. 47 | 48 | ## v2.7.0 49 | 50 | ### Improvements 51 | 52 | - Adds support for data streaming in Arrow columnar format via Apache 53 | Arrow Flight. 54 | 55 | ## v2.6.0 56 | 57 | ### Improvements 58 | 59 | - Fixes deprecation of sklearn in favor of current package name 60 | scikit-learn 61 | 62 | ## v2.5.1 63 | 64 | ### Improvements 65 | 66 | - Gzip encoded requests are now supported by default. This can be disabled in 67 | the config file. 68 | - The INFO method will return the enabled status of features. 69 | 70 | ## v2.5.0 71 | 72 | ### Improvements 73 | 74 | - A server with Adhoc Disabled Flag on with the wrong credentials will now 75 | return wrong credentials error instead of telling the user 76 | that Adhoc Scripts are not allowed on this server. 77 | - Added documentation for how to run TabPy projects with local changes 78 | 79 | ### Breaking changes 80 | 81 | - Discontinued support for Python 3.6 82 | - Added support for Python 3.9 83 | 84 | ## v2.4.0 85 | 86 | ### Improvements 87 | 88 | - Add toggle to turn off evaluate API. 89 | 90 | ### Breaking changes 91 | 92 | - Changing error code to 406 when server not configured for authentication 93 | but credentials are provided by client. 94 | 95 | ## v2.3.2 96 | 97 | ### Improvements 98 | 99 | - Test files added to tar.gz and zip releases. 100 | 101 | ## v2.3.1 102 | 103 | ### Bug fixes 104 | 105 | - Overriding deployed models. 106 | 107 | ## v2.3.0 108 | 109 | ### Improvements 110 | 111 | - Fixed scrutinizer test run failure. 112 | 113 | ## v2.2.0 114 | 115 | ### Breaking changes 116 | 117 | - TabPy fails with 400 when it is not configure for authentication 118 | but credentials are provided by client. 119 | 120 | ### Bug fixes 121 | 122 | - When TabPy is running with no console attached it is not failing 123 | with 500 when trying to respond with 401 status. 124 | - tabpy.query() failing when auth is configured. 125 | 126 | ### Improvements 127 | 128 | - Minor code cleanup. 129 | 130 | ## v1.1.0 131 | 132 | ### Improvements 133 | 134 | - Authorization is now required for the /info API method. 135 | This method did not check authentication previously. This change is 136 | backwards compatible with Tableau clients. 137 | - Improved config parsing flexibility. Previously the 138 | TABPY_EVALUATE_TIMEOUT setting would be set to a default if 139 | tabpy couldn't parse the value. Now it will throw an exception 140 | at startup. 141 | 142 | ## v1.0.0 143 | 144 | ### Improvements 145 | 146 | - Minor: feature name changed to analytics extensions. 147 | - Startup script files deleted. 148 | - Index page updated. 149 | 150 | ### Other 151 | 152 | - TabPy is now Tableau Supported (used to be Community Supported). 153 | 154 | ## v0.9.0 155 | 156 | ### Improvements 157 | 158 | - Models deployment doesn't depend on pip._internal anymore. 159 | - Package size reduced. 160 | 161 | ## v0.8.13 162 | 163 | ### Improvements 164 | 165 | - TabPy works with Python 3.8 now. 166 | - Documentation updates with referencing Tableau Help pages. 167 | - Added Client.remove() method for deleting deployed models. 168 | 169 | ### Bug Fixes 170 | 171 | - Fixed failing Ctrl+C handler. 172 | - Fixed query_timeout bug. 173 | - Fixed None in result collection bug. 174 | - Fixed script evaluation with missing result/return bug. 175 | - Fixed startup failure on Windows for Python 3.8. 176 | 177 | ## v0.8.9 178 | 179 | ### Improvements 180 | 181 | - Added Ctrl+C handler 182 | - Added configurable buffer size for HTTP requests 183 | - Added anvoa to supported pre-deployed models in tabpy 184 | 185 | ## v0.8.7 186 | 187 | ### Improvements 188 | 189 | - Enabled the use of environment variables in the config file. 190 | 191 | ## v0.8.6 192 | 193 | ### Fixes 194 | 195 | - Fixed file names for package building. 196 | - Fixed reading version info for /info call. 197 | 198 | ## v0.8 199 | 200 | ### Improvements 201 | 202 | - TabPy is pip package now 203 | - Models are deployed with updated script 204 | 205 | ## v0.7 206 | 207 | ### Improvements 208 | 209 | - Added t-test model 210 | - Fixed models call with /evaluate for HTTPS 211 | - Migrated to Tornado 6 212 | - Timeout is configurable with TABPY_EVALUATE_TIMEOUT config 213 | file option 214 | 215 | ## v0.6.1 216 | 217 | ### Improvements 218 | 219 | - Scripts, documentation, and integration tests for models 220 | - Small bug fixes 221 | - Added request context logging as a feature controlled with 222 | TABPY_LOG_DETAILS configuration setting. 223 | - Updated documentation for /info method and v1 API. 224 | - Added integration tests. 225 | 226 | ## v0.4 227 | 228 | ### Improvements 229 | 230 | - Added basic access authentication (all methods except /info) 231 | - tabpy-tools can deploy models to TabPy with authentication on 232 | - Increased unit tests coverage 233 | - Travis CI for merge requests: unit tests executed, code style checking 234 | 235 | ## v0.3.2 236 | 237 | ### Breaking changes 238 | 239 | - Logger configuration now is in TabPy config file. 240 | 241 | ### Improvements 242 | 243 | - Remove versioneer and just replace it with VERSION file 244 | - Require Python 3.6.5 245 | - Require jsonschema to be compatible with 2.3.0 246 | - Added setup instructions (known issues) for CentOS 247 | 248 | ## v0.3.1 249 | 250 | - Fixed dependency on tabpy-tools in startup scripts 251 | - Fixed Python version dependency in tabpy-server setup script 252 | 253 | ## v0.3 254 | 255 | ### Breaking changes 256 | 257 | - The config file is now not just Python code but an actual config 258 | - Tornado config file has a different setting for CORS 259 | - Setup scripts are deleted - setup (if needed) happens with the startup script 260 | - tabpy-client is tabpy-tools now 261 | 262 | ### Improvements 263 | 264 | - Secure connection (HTTPS) is supported with Tableau 2019.2 and newer versions 265 | - Documentation is improved with more examples added 266 | - Versioning is done with Versioneer and github release tags 267 | - Improved logging 268 | - Unit tests are passing now 269 | - Configurations for Postman and Swagger are available to use those against running TabPy 270 | 271 | ## v0.2 272 | 273 | - Initial version 274 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # TabPy Contributing Guide 2 | 3 | 4 | 5 | 6 | 7 | - [Environment Setup](#environment-setup) 8 | - [Prerequisites](#prerequisites) 9 | - [Cloning TabPy Repository](#cloning-tabpy-repository) 10 | - [Tests](#tests) 11 | * [Unit Tests](#unit-tests) 12 | * [Integration Tests](#integration-tests) 13 | - [Code Coverage](#code-coverage) 14 | - [TabPy in Python Virtual Environment](#tabpy-in-python-virtual-environment) 15 | - [Documentation Updates](#documentation-updates) 16 | - [TabPy with Swagger](#tabpy-with-swagger) 17 | - [Code styling](#code-styling) 18 | 19 | 20 | 21 | 22 | 23 | ## Environment Setup 24 | 25 | The purpose of this guide is to enable developers of Tabpy to install the project 26 | and run it locally. 27 | 28 | ## Prerequisites 29 | 30 | These are prerequisites for an environment required for a contributor to 31 | be able to work on TabPy changes: 32 | 33 | - Supported 64-bit Python version (see 34 | [README](https://github.com/tableau/TabPy) for a list of compatible versions). 35 | - git 36 | - Node.js for npm packages - install from . 37 | - NPM packages - install all with 38 | `npm install markdown-toc markdownlint` command. 39 | 40 | ## Cloning TabPy Repository 41 | 42 | 1. Open your OS shell. 43 | 2. Navigate to the folder in which you would like to save 44 | your local TabPy repository. 45 | 3. In the command prompt, enter the following commands: 46 | 47 | ```sh 48 | git clone https://github.com/tableau/TabPy.git 49 | cd TabPy 50 | ``` 51 | 52 | 4. Install all dependencies: 53 | 54 | ```sh 55 | python -m pip install --upgrade pip 56 | pip install -r requirements.txt 57 | pip install -r requirements_dev.txt 58 | ``` 59 | 60 | ## Tests 61 | 62 | To run the whole test suite execute the following command: 63 | 64 | ```sh 65 | pytest 66 | ``` 67 | 68 | ### Unit Tests 69 | 70 | Unit tests suite can be executed with the following command: 71 | 72 | ```sh 73 | pytest tests/unit 74 | ``` 75 | 76 | ### Integration Tests 77 | 78 | Integration tests can be executed with the next command: 79 | 80 | ```sh 81 | pytest tests/integration 82 | ``` 83 | 84 | ## Code Coverage 85 | 86 | You can run unit tests to collect code coverage data. To do so run `pytest` 87 | either for server or tools test, or even combined: 88 | 89 | ```sh 90 | pytest tests --cov=tabpy 91 | ``` 92 | 93 | ## TabPy in Python Virtual Environment 94 | 95 | It is possible (and recommended) to run TabPy in a virtual environment. More 96 | details are on 97 | [TabPy in Python virtual environment](docs/tabpy-virtualenv.md) page. 98 | 99 | ## Documentation Updates 100 | 101 | For any process, scripts or API changes documentation needs to be updated accordingly. 102 | Please use markdown validation tools like web-based [markdownlint](https://dlaa.me/markdownlint/) 103 | or npm [markdownlint-cli](https://github.com/igorshubovych/markdownlint-cli). 104 | 105 | TOC for markdown file is built with [markdown-toc](https://www.npmjs.com/package/markdown-toc): 106 | 107 | ```sh 108 | markdown-toc -i docs/server-startup.md 109 | ``` 110 | 111 | To check markdown style for all the documentation use `markdownlint`: 112 | 113 | ```sh 114 | markdownlint . 115 | ``` 116 | 117 | These checks will run as part of the build if you submit a pull request. 118 | 119 | ## TabPy with Swagger 120 | 121 | You can invoke the TabPy Server API against a running TabPy instance with Swagger. 122 | 123 | - Make CORS related changes in TabPy configuration file: update `tabpy/tabpy-server/state.ini` 124 | file in your local repository to have the next settings: 125 | 126 | ```config 127 | [Service Info] 128 | Access-Control-Allow-Origin = * 129 | Access-Control-Allow-Headers = Origin, X-Requested-with, Content-Type 130 | Access-Control-Allow-Methods = GET, OPTIONS, POST 131 | ``` 132 | 133 | - Start a local instance of TabPy server following [TabPy Server Startup Guide](docs/server-startup.md). 134 | - Run a local copy of Swagger editor with steps provided at 135 | [https://github.com/swagger-api/swagger-editor](https://github.com/swagger-api/swagger-editor). 136 | - Open `misc/TabPy.yml` in Swagger editor. 137 | - In case your TabPy server does not run on `localhost:9004` update 138 | `host` value in `TabPy.yml` accordingly. 139 | 140 | ## Code styling 141 | 142 | `flake8` is used to check Python code against our style conventions: 143 | 144 | ```sh 145 | flake8 . 146 | ``` 147 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | WORKDIR /app 4 | 5 | # install the latest TabPy 6 | RUN python3 -m pip install --upgrade pip && python3 -m pip install --upgrade tabpy 7 | 8 | # start TabPy 9 | CMD ["sh", "-c", "tabpy"] 10 | 11 | # run startup script 12 | ADD start.sh / 13 | RUN chmod +x /start.sh 14 | CMD ["/start.sh"] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Salesforce, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | exclude \ 2 | tabpy/tabpy_server/state.ini 3 | 4 | include \ 5 | CHANGELOG \ 6 | LICENSE \ 7 | README.md \ 8 | Procfile \ 9 | tabpy/VERSION \ 10 | tabpy/tabpy_server/state.ini.template \ 11 | tabpy/tabpy_server/static/* \ 12 | tabpy/tabpy_server/common/default.conf 13 | 14 | # Docs and tests 15 | include requirements_dev.txt requirements.txt 16 | recursive-include docs *.md 17 | recursive-include docs *.png 18 | recursive-include misc *.json 19 | recursive-include misc *.yml 20 | recursive-include tests *.conf 21 | recursive-include tests *.crt 22 | recursive-include tests *.key 23 | recursive-include tests *.txt 24 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: export TABPY_PORT=$PORT && export TABPY_PWD_FILE=./file.txt && tabpy-user add -u $USERNAME -p $PASSWORD -f ./file.txt && tabpy -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TabPy 2 | 3 | [![Tableau Supported](https://img.shields.io/badge/Support%20Level-Tableau%20Supported-53bd92.svg)](https://www.tableau.com/support-levels-it-and-developer-tools) 4 | [![GitHub](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://raw.githubusercontent.com/Tableau/TabPy/master/LICENSE) 5 | 6 | [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/tableau/tabpy/Test%20Run%20on%20Push)](https://github.com/tableau/TabPy/actions?query=workflow%3A%22Test+Run+on+Push%22) 7 | [![Coverage Status](https://coveralls.io/repos/github/tableau/TabPy/badge.svg?branch=master)](https://coveralls.io/github/tableau/TabPy?branch=master) 8 | [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/tableau/TabPy/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/tableau/TabPy/?branch=master) 9 | 10 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/tabpy?label=PyPI%20Python%20versions) 11 | [![PyPI version](https://badge.fury.io/py/tabpy.svg)](https://pypi.python.org/pypi/tabpy/) 12 | 13 | [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy?template=https://github.com/tableau/tabpy) 14 | 15 | TabPy (the Tableau Python Server) is an Analytics Extension implementation which 16 | expands Tableau's capabilities by allowing users to execute Python scripts and 17 | saved functions via Tableau's table calculations. 18 | 19 | Consider reading TabPy documentation in the following order: 20 | 21 | * [About TabPy](docs/about.md) 22 | * [TabPy Installation Instructions](docs/server-install.md) 23 | * [TabPy Server Configuration Instructions](docs/server-config.md) 24 | * [Running TabPy in Virtual Environment](docs/tabpy-virtualenv.md) 25 | * [Running TabPy on Heroku](docs/deploy-to-heroku.md) 26 | * [Authoring Python calculations in Tableau](docs/TableauConfiguration.md). 27 | * [TabPy Tools](docs/tabpy-tools.md) 28 | 29 | Important security note: 30 | 31 | * By default, TabPy is configured without username/password authentication. 32 | We strongly advise using TabPy only with authentication enabled. For more 33 | information, see 34 | [TabPy Server Configuration Instructions](docs/server-config.md#authentication). 35 | Without authentication in place, if the TABPY_EVALUATE_ENABLE feature is 36 | enabled (as it is by default), there is the possibility that unauthenticated 37 | individuals could remotely execute code on the machine running TabPy. 38 | Leaving these two settings in their default states together is highly 39 | discouraged. 40 | 41 | Troubleshooting: 42 | 43 | * [TabPy Wiki](https://github.com/tableau/TabPy/wiki) 44 | 45 | More technical topics: 46 | 47 | * [Contributing Guide](CONTRIBUTING.md) for TabPy developers 48 | * [TabPy REST API](docs/server-rest.md) 49 | * [TabPy Security Considerations](docs/security.md) 50 | 51 | Other useful resources: 52 | 53 | * [Tableau Sci-Fi Blog](http://tabscifi.golovatyi.info/) provides tips, tricks, under 54 | the hood, useful resources, and technical details for how to extend 55 | Tableau with data science. 56 | * [Known Issues for the Tableau Analytics Extensions API](https://tableau.github.io/analytics-extensions-api/docs/ae_known_issues.html). 57 | * For all questions not related to the TabPy code (installation, deployment, 58 | connections, Python issues, etc.) and requests use the 59 | [Analytics Extensions Forum](https://community.tableau.com/community/forums/analyticsextensions) 60 | on [Tableau Community](https://community.tableau.com). 61 | * [Building advanced analytics applications with TabPy](https://www.tableau.com/about/blog/2017/1/building-advanced-analytics-applications-tabpy-64916) 62 | * [Building Data Science Applications with TabPy Video Tutorial](https://youtu.be/nRtOMTnBz_Y) 63 | * [TabPy Tutorial on TabWiki](https://community.tableau.com/docs/DOC-10856) 64 | 65 | ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/tableau/TabPy.svg) 66 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TabPy", 3 | "description": "Analytics Extension implementation which expands Tableau's capabilities by allowing users to execute Python scripts and saved functions via Tableau's table calculations.", 4 | "repository": "https://github.com/tableau/TabPy", 5 | "logo": "https://raw.githubusercontent.com/tableau/TabPy/master/tabpy/tabpy_server/static/TabPy_logo.png", 6 | "keywords": ["tableau", "python", "analytics-extension"], 7 | "env":{ 8 | "USERNAME":{ 9 | "description": "Add your username", 10 | "value": "gzanolli", 11 | "required": true 12 | }, 13 | "PASSWORD":{ 14 | "description": "Define your password", 15 | "value": "P@ssw0rd", 16 | "required": true 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master, dev ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '0 22 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /docs/about.md: -------------------------------------------------------------------------------- 1 | # About TabPy 2 | 3 | TabPy framework allows Tableau to remotely execute Python code. It has two components: 4 | 5 | 1. A process built on Tornado, which allows for the remote execution of Python 6 | code through a set of [REST APIs](server-rest.md). The code can either be immediately 7 | executed or persisted in the server process and exposed as a REST endpoint, 8 | to be called later. 9 | 10 | 2. A [tools library](tabpy-tools.md), 11 | based on Python functions which enables the deployment of such endpoints. 12 | 13 | Tableau can connect to the TabPy server to execute Python code on the fly and 14 | display results in Tableau visualizations. Users can control data and parameters 15 | being sent to TabPy by interacting with their Tableau worksheets, dashboard or stories. 16 | 17 | For how to configure Tableau to connect to TabPy server follow steps in 18 | [Tableau Configuration Document](TableauConfiguration.md). 19 | -------------------------------------------------------------------------------- /docs/deploy-to-heroku.md: -------------------------------------------------------------------------------- 1 | # Deploying TabPy to your Heroku account 2 | 3 | To deploy TabPy from master branch to a Heroku account: 4 | 5 | 1. Log in to Heroku with your account via a browser. 6 | If you don't have an account, create one. 7 | 8 | 2. Click the "Deploy to Heroku" button in the Readme. 9 | 10 | 3. Configure the new TabPy server by setting environment 11 | variables through Heroku's web console or API. 12 | 13 | 4. TabPy will run on the default secure port 443. 14 | -------------------------------------------------------------------------------- /docs/img/Example1-SimpleFunctionCall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/docs/img/Example1-SimpleFunctionCall.png -------------------------------------------------------------------------------- /docs/img/Example2-MultipleFunctionCalls.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/docs/img/Example2-MultipleFunctionCalls.png -------------------------------------------------------------------------------- /docs/img/python-calculated-field.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/docs/img/python-calculated-field.png -------------------------------------------------------------------------------- /docs/security.md: -------------------------------------------------------------------------------- 1 | # TabPy Security Considerations 2 | 3 | If security is a significant concern within your organization, 4 | you may want to consider the following as you use TabPy: 5 | 6 | - The REST server and Python execution share the same Python session, 7 | meaning that HTTP requests and user scripts are evaluated in the 8 | same addressable memory and processor threads. 9 | - The tabpy.tabpy_tools client does not perform client-side validation of the 10 | SSL certificate on TabPy Server. 11 | - Python scripts can contain code which can harm security on the server 12 | where the TabPy is running. For example, Python scripts can: 13 | - Access the file system (read/write). 14 | - Install new Python packages which can contain binary code. 15 | - Execute operating system commands. 16 | - Open network connections to other servers and download files. 17 | - Execution of ad-hoc Python scripts can be disabled by turning off the 18 | /evaluate endpoint. To disable /evaluate endpoint, set "TABPY_EVALUATE_ENABLE" 19 | to false in config file. 20 | - Always use the most up-to-date version of Python. 21 | TabPy relies on Tornado and if older verions of Python are used with Tornado 22 | then malicious users can potentially poison Python server web caches 23 | with parameter cloaking. 24 | -------------------------------------------------------------------------------- /docs/server-install.md: -------------------------------------------------------------------------------- 1 | # TabPy Installation Instructions 2 | 3 | These instructions explain how to install and start up TabPy Server. 4 | 5 | 6 | 7 | 8 | 9 | - [TabPy Installation](#tabpy-installation) 10 | - [Starting TabPy](#starting-tabpy) 11 | 12 | 13 | 14 | 15 | 16 | ## TabPy Installation 17 | 18 | First, ensure that you are using a supported 64-bit Python version. 19 | Refer to the [README](https://github.com/tableau/TabPy) for a list 20 | of compatible versions. 21 | 22 | ### Installation 23 | 24 | To install TabPy on to an environment `pip` needs to be installed and 25 | updated first: 26 | 27 | ```sh 28 | python -m pip install --upgrade pip 29 | ``` 30 | 31 | Now TabPy can be install as a package: 32 | 33 | ```sh 34 | pip install tabpy 35 | ``` 36 | 37 | ## Starting TabPy 38 | 39 | To start TabPy with default settings run the following command: 40 | 41 | ```sh 42 | tabpy 43 | ``` 44 | 45 | To run TabPy with custom settings create config file with parameters 46 | explained in [TabPy Server Configuration Instructions](server-config.md) 47 | and specify it in command line: 48 | 49 | ```sh 50 | tabpy --config=path/to/my/config/file.conf 51 | ``` 52 | 53 | It is highly recommended to use Python virtual environment for running TabPy. 54 | Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page 55 | for more details. 56 | 57 | ## Starting a Local TabPy Project 58 | 59 | To create a version of TabPy that incorporates locally-made changes, 60 | use pip to create a package from your local TabPy project 61 | and install it within that directory (preferably a virtual environment): 62 | 63 | ```sh 64 | pip install -e . 65 | ``` 66 | 67 | Then start TabPy just like it was mentioned earlier 68 | 69 | ```sh 70 | tabpy 71 | ``` 72 | -------------------------------------------------------------------------------- /docs/server-rest.md: -------------------------------------------------------------------------------- 1 | # TabPy REST Interface 2 | 3 | The server process exposes several REST APIs to get status and to execute 4 | Python code and query deployed methods. 5 | 6 | 7 | 8 | 9 | 10 | - [Authentication, /info and /evaluate](#authentication-info-and-evaluate) 11 | - [http:get:: /status](#httpget-status) 12 | - [http:get:: /endpoints](#httpget-endpoints) 13 | - [http:get:: /endpoints/:endpoint](#httpget-endpointsendpoint) 14 | - [http:post:: /query/:endpoint](#httppost-queryendpoint) 15 | 16 | 17 | 18 | 19 | 20 | ## Authentication, /info and /evaluate 21 | 22 | Analytics Extensions API v1 is documented at 23 | [https://tableau.github.io/analytics-extensions-api/docs/ae_api_ref.html](https://tableau.github.io/analytics-extensions-api/docs/ae_api_ref.html). 24 | 25 | The following documentation is for methods not currently used by Tableau. 26 | 27 | ## http:get:: /status 28 | 29 | Gets runtime status of deployed endpoints. If no endpoints are deployed in 30 | the server, the returned data is an empty JSON object. 31 | 32 | Example request: 33 | 34 | ```HTTP 35 | GET /status HTTP/1.1 36 | Host: localhost:9004 37 | Accept: application/json 38 | ``` 39 | 40 | Example response: 41 | 42 | ```HTTP 43 | HTTP/1.1 200 OK 44 | Content-Type: application/json 45 | 46 | {"clustering": { 47 | "status": "LoadSuccessful", 48 | "last_error": null, 49 | "version": 1, 50 | "type": "model"}, 51 | "add": { 52 | "status": "LoadSuccessful", 53 | "last_error": null, 54 | "version": 1, 55 | "type": "model"} 56 | } 57 | ``` 58 | 59 | Using curl: 60 | 61 | ```bash 62 | curl -X GET http://localhost:9004/status 63 | ``` 64 | 65 | ## http:get:: /endpoints 66 | 67 | Gets a list of deployed endpoints and their static information. If no 68 | endpoints are deployed in the server, the returned data is an empty JSON object. 69 | 70 | Example request: 71 | 72 | ```HTTP 73 | GET /endpoints HTTP/1.1 74 | Host: localhost:9004 75 | Accept: application/json 76 | ``` 77 | 78 | Example response: 79 | 80 | ```HTTP 81 | HTTP/1.1 200 OK 82 | Content-Type: application/json 83 | 84 | {"clustering": 85 | {"description": "", 86 | "docstring": "-- no docstring found in query function --", 87 | "creation_time": 1469511182, 88 | "version": 1, 89 | "dependencies": [], 90 | "last_modified_time": 1469511182, 91 | "type": "model", 92 | "target": null}, 93 | "add": { 94 | "description": "", 95 | "docstring": "-- no docstring found in query function --", 96 | "creation_time": 1469505967, 97 | "version": 1, 98 | "dependencies": [], 99 | "last_modified_time": 1469505967, 100 | "type": "model", 101 | "target": null} 102 | } 103 | ``` 104 | 105 | Using curl: 106 | 107 | ```bash 108 | curl -X GET http://localhost:9004/endpoints 109 | ``` 110 | 111 | ## http:get:: /endpoints/:endpoint 112 | 113 | Gets the description of a specific deployed endpoint. The endpoint must first 114 | be deployed in the server (see the [TabPy Tools documentation](tabpy-tools.md)). 115 | 116 | Example request: 117 | 118 | ```HTTP 119 | GET /endpoints/add HTTP/1.1 120 | Host: localhost:9004 121 | Accept: application/json 122 | ``` 123 | 124 | Example response: 125 | 126 | ```HTTP 127 | HTTP/1.1 200 OK 128 | Content-Type: application/json 129 | 130 | {"description": "", "docstring": "-- no docstring found in query function --", 131 | "creation_time": 1469505967, "version": 1, "dependencies": [], 132 | "last_modified_time": 1469505967, "type": "model", "target": null} 133 | ``` 134 | 135 | Using curl: 136 | 137 | ```bash 138 | curl -X GET http://localhost:9004/endpoints/add 139 | ``` 140 | 141 | ## http:post:: /query/:endpoint 142 | 143 | Executes a function at the specified endpoint. The function must first be 144 | deployed (see the [TabPy Tools documentation](tabpy-tools.md)). 145 | 146 | This interface expects a JSON body with a `data` key, specifying the values 147 | for the function, according to its original definition. In the example below, 148 | the function `clustering` was defined with a signature of two parameters `x` 149 | and `y`, expecting arrays of numbers. 150 | 151 | Example request: 152 | 153 | ```HTTP 154 | POST /query/clustering HTTP/1.1 155 | Host: localhost:9004 156 | Accept: application/json 157 | 158 | {"data": { 159 | "x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], 160 | "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}} 161 | ``` 162 | 163 | Example response: 164 | 165 | ```HTTP 166 | HTTP/1.1 200 OK 167 | Content-Type: application/json 168 | 169 | {"model": "clustering", "version": 1, "response": [0, 0, 0, 1, 1, 1, 1], 170 | "uuid": "46d3df0e-acca-4560-88f1-67c5aedeb1c4"} 171 | ``` 172 | 173 | Using curl: 174 | 175 | ```bash 176 | curl -X GET http://localhost:9004/query/clustering -d \ 177 | '{"data": {"x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], 178 | "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}}' 179 | ``` 180 | -------------------------------------------------------------------------------- /docs/tabpy-virtualenv.md: -------------------------------------------------------------------------------- 1 | # Running TabPy in Virtual Environment 2 | 3 | 4 | 5 | ## Running TabPy in Python Virtual Environment 6 | 7 | To run TabPy in Python virtual environment follow the steps: 8 | 9 | 1. Install `virtualenv` package: 10 | 11 | ```sh 12 | pip install virtualenv 13 | ``` 14 | 15 | 2. Create virtual environment (replace `my-tabpy-env` with 16 | your virtual environment name): 17 | 18 | ```sh 19 | virtualenv my-tabpy-env 20 | ``` 21 | 22 | 3. Activate the environment. 23 | 1. For Windows run 24 | 25 | ```sh 26 | my-tabpy-env\Scripts\activate 27 | ``` 28 | 29 | 2. For Linux and Mac run 30 | 31 | ```sh 32 | source my-tabpy-env/bin/activate 33 | ``` 34 | 35 | 4. Run TabPy: 36 | 1. Default TabPy 37 | 38 | ```sh 39 | tabpy 40 | ``` 41 | 42 | 2. Local TabPy 43 | 44 | To create a version of TabPy that incorporates locally-made changes, 45 | use pip to create a package from your local TabPy project and install 46 | it within that directory: 47 | 48 | ```sh 49 | pip install -e . 50 | ``` 51 | 52 | Then start TabPy just like it was mentioned earlier 53 | 54 | ```sh 55 | tabpy 56 | ``` 57 | 58 | 5. To deactivate virtual environment run: 59 | 60 | ```sh 61 | deactivate 62 | ``` 63 | 64 | ## Running TabPy in an Anaconda Virtual Environment 65 | 66 | To run TabPy in an Anaconda virtual environment follow the steps: 67 | *NOTE: this assumes you have installed [Anaconda](https://www.anaconda.com/products/individual) 68 | in a Windows environment* 69 | 70 | 1. For Windows open `Anaconda Prompt` from the Windows Start menu, for 71 | Linux and Mac run shell. 72 | 73 | 2. Navigate to your home directory: 74 | 1. On Windows run 75 | 76 | ```sh 77 | cd %USERPROFILE% 78 | ``` 79 | 80 | 2. For Linux and Mac run 81 | 82 | ```sh 83 | cd ~ 84 | ``` 85 | 86 | 3. Create the virtual Anaconda environment 87 | 88 | ```sh 89 | conda create --name my-tabpy-env python=3.12 90 | ``` 91 | 92 | 4. Activate your virtual environment 93 | 94 | ```sh 95 | conda activate my-tabpy-env 96 | ``` 97 | 98 | 5. Install TabPy to your new Anaconda environment by following the instructions 99 | on the [TabPy Server Install](server-install.md) documentation page. 100 | 101 | 6. Run TabPy: 102 | 103 | ```sh 104 | tabpy 105 | ``` 106 | 107 | 7. To deactivate virtual environment run: 108 | 109 | ```sh 110 | conda deactivate 111 | ``` 112 | -------------------------------------------------------------------------------- /misc/TabPy.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "_postman_id": "add0b83b-3c11-4c80-973a-0a5fbf803e58", 4 | "name": "TabPy", 5 | "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" 6 | }, 7 | "item": [ 8 | { 9 | "name": "{{host}}:{{port}}/info", 10 | "request": { 11 | "method": "GET", 12 | "header": [], 13 | "url": { 14 | "raw": "{{host}}:{{port}}/info", 15 | "host": [ 16 | "{{host}}" 17 | ], 18 | "port": "{{port}}", 19 | "path": [ 20 | "info" 21 | ] 22 | } 23 | }, 24 | "response": [] 25 | }, 26 | { 27 | "name": "{{host}}:{{port}}/evaluate", 28 | "request": { 29 | "auth": { 30 | "type": "basic", 31 | "basic": [ 32 | { 33 | "key": "password", 34 | "value": "P@ssw0rd", 35 | "type": "string" 36 | }, 37 | { 38 | "key": "username", 39 | "value": "user1", 40 | "type": "string" 41 | } 42 | ] 43 | }, 44 | "method": "POST", 45 | "header": [ 46 | { 47 | "key": "Content-Type", 48 | "name": "Content-Type", 49 | "value": "application/json", 50 | "type": "text" 51 | }, 52 | { 53 | "key": "TabPy-Client", 54 | "value": "Postman for manual testing", 55 | "type": "text" 56 | }, 57 | { 58 | "key": "TabPy-User", 59 | "value": "ogolovatyi", 60 | "type": "text" 61 | } 62 | ], 63 | "body": { 64 | "mode": "raw", 65 | "raw": "{\n\t\"data\": \n\t{ \n\t\t\"_arg1\" : [1, 2, 3], \n\t\t\"_arg2\" : [3, -1, 5]\n\t},\n\t\"script\": \n\t\"return [x + y for x, y in zip(_arg1, _arg2)]\"\n}\n", 66 | "options": { 67 | "raw": {} 68 | } 69 | }, 70 | "url": { 71 | "raw": "{{host}}:{{port}}/evaluate", 72 | "host": [ 73 | "{{host}}" 74 | ], 75 | "port": "{{port}}", 76 | "path": [ 77 | "evaluate" 78 | ], 79 | "query": [ 80 | { 81 | "key": "TabPy-Client", 82 | "value": "Postman for Manual Testing", 83 | "disabled": true 84 | }, 85 | { 86 | "key": "TabPy-User", 87 | "value": "ogolovatyi", 88 | "disabled": true 89 | } 90 | ] 91 | } 92 | }, 93 | "response": [] 94 | }, 95 | { 96 | "name": "{{endpoint}}/status", 97 | "request": { 98 | "method": "GET", 99 | "header": [], 100 | "url": { 101 | "raw": "{{host}}:{{port}}/status", 102 | "host": [ 103 | "{{host}}" 104 | ], 105 | "port": "{{port}}", 106 | "path": [ 107 | "status" 108 | ] 109 | } 110 | }, 111 | "response": [] 112 | }, 113 | { 114 | "name": "{{host}}:{{port}}/endpoints", 115 | "request": { 116 | "method": "GET", 117 | "header": [], 118 | "url": { 119 | "raw": "{{host}}:{{port}}/endpoints", 120 | "host": [ 121 | "{{host}}" 122 | ], 123 | "port": "{{port}}", 124 | "path": [ 125 | "endpoints" 126 | ] 127 | } 128 | }, 129 | "response": [] 130 | }, 131 | { 132 | "name": "{{host}}:{{port}}/query/model_name", 133 | "request": { 134 | "method": "POST", 135 | "header": [ 136 | { 137 | "key": "Content-Type", 138 | "name": "Content-Type", 139 | "value": "application/json", 140 | "type": "text" 141 | } 142 | ], 143 | "body": { 144 | "mode": "raw", 145 | "raw": "{\r\n \"data\": {\r\n \"x\": [\r\n 6.35,\r\n 6.4,\r\n 6.65,\r\n 8.6,\r\n 8.9,\r\n 9,\r\n 9.1\r\n ],\r\n \"y\": [\r\n 1.95,\r\n 1.95,\r\n 2.05,\r\n 3.05,\r\n 3.05,\r\n 3.1,\r\n 3.15\r\n ]\r\n }\r\n}" 146 | }, 147 | "url": { 148 | "raw": "{{host}}:{{port}}/query/model_name", 149 | "host": [ 150 | "{{host}}" 151 | ], 152 | "port": "{{port}}", 153 | "path": [ 154 | "query", 155 | "model_name" 156 | ] 157 | } 158 | }, 159 | "response": [] 160 | }, 161 | { 162 | "name": "{{host}}:{{port}}/endpoints/model_name", 163 | "request": { 164 | "method": "GET", 165 | "header": [], 166 | "url": { 167 | "raw": "{{host}}:{{port}}/endpoints/model_name", 168 | "host": [ 169 | "{{host}}" 170 | ], 171 | "port": "{{port}}", 172 | "path": [ 173 | "endpoints", 174 | "model_name" 175 | ] 176 | } 177 | }, 178 | "response": [] 179 | } 180 | ] 181 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # installs dependencies from ./setup.py, and the package itself, 2 | # in editable mode 3 | -e . -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | flake8 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pep8] 2 | max-line-length = 98 3 | 4 | [pycodestyle] 5 | max-line-length = 98 6 | 7 | [flake8] 8 | max-line-length = 98 9 | 10 | [bdist_wheel] 11 | universal=1 12 | 13 | [report] 14 | # Exclude lines that match patterns from coverage report. 15 | exclude_lines = 16 | if __name__ == .__main__.: 17 | \\$ 18 | 19 | # Only show one number after decimal point in report. 20 | precision = 1 21 | 22 | [run] 23 | omit = 24 | tabpy/models/* 25 | tabpy/tabpy.py 26 | tabpy/utils/* 27 | tests/* 28 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Web server Tableau uses to run Python scripts. 2 | 3 | TabPy (the Tableau Python Server) is an external service implementation 4 | which expands Tableau's capabilities by allowing users to execute Python 5 | scripts and saved functions via Tableau's table calculations. 6 | """ 7 | 8 | import os 9 | from setuptools import setup, find_packages 10 | import unittest 11 | 12 | 13 | DOCLINES = (__doc__ or "").split("\n") 14 | 15 | 16 | def setup_package(): 17 | def read(fname): 18 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 19 | 20 | setup( 21 | name="tabpy", 22 | version=read("tabpy/VERSION"), 23 | description=DOCLINES[0], 24 | long_description="\n".join(DOCLINES[1:]) + "\n" + read("CHANGELOG"), 25 | long_description_content_type="text/markdown", 26 | url="https://github.com/tableau/TabPy", 27 | author="Tableau", 28 | author_email="github@tableau.com", 29 | maintainer="Tableau", 30 | maintainer_email="github@tableau.com", 31 | download_url="https://pypi.org/project/tabpy", 32 | project_urls={ 33 | "Bug Tracker": "https://github.com/tableau/TabPy/issues", 34 | "Documentation": "https://tableau.github.io/TabPy/", 35 | "Source Code": "https://github.com/tableau/TabPy", 36 | }, 37 | classifiers=[ 38 | "Development Status :: 5 - Production/Stable", 39 | "Intended Audience :: Developers", 40 | "Intended Audience :: Science/Research", 41 | "License :: OSI Approved :: MIT License", 42 | "Programming Language :: Python :: 3.9", 43 | "Programming Language :: Python :: 3.10", 44 | "Programming Language :: Python :: 3.11", 45 | "Programming Language :: Python :: 3.12", 46 | "Topic :: Scientific/Engineering", 47 | "Topic :: Scientific/Engineering :: Information Analysis", 48 | "Operating System :: Microsoft :: Windows", 49 | "Operating System :: POSIX", 50 | "Operating System :: Unix", 51 | "Operating System :: MacOS", 52 | ], 53 | platforms=["Windows", "Linux", "Mac OS-X", "Unix"], 54 | keywords=["tabpy tableau"], 55 | packages=find_packages(exclude=["docs", "misc"]), 56 | package_data={ 57 | "tabpy": [ 58 | "VERSION", 59 | "tabpy_server/state.ini.template", 60 | "tabpy_server/static/*", 61 | "tabpy_server/common/default.conf", 62 | ] 63 | }, 64 | python_requires=">=3.7", 65 | license="MIT", 66 | # Note: many of these required packages are included in base python 67 | # but are listed here because different linux distros use custom 68 | # python installations. And users can remove packages at any point 69 | install_requires=[ 70 | "cloudpickle", 71 | "configparser", 72 | "coverage", 73 | "coveralls", 74 | "docopt", 75 | "future", 76 | "genson", 77 | "hypothesis", 78 | "jsonschema", 79 | "mock", 80 | "nltk", 81 | "numpy", 82 | "pandas", 83 | "pyopenssl", 84 | "pytest", 85 | "pytest-cov", 86 | "requests", 87 | "scipy", 88 | "simplejson", 89 | "scikit-learn", 90 | "textblob", 91 | "tornado", 92 | "twisted", 93 | "urllib3", 94 | "pyarrow", 95 | ], 96 | entry_points={ 97 | "console_scripts": [ 98 | "tabpy=tabpy.tabpy:main", 99 | "tabpy-deploy-models=tabpy.models.deploy_models:main", 100 | "tabpy-user=tabpy.utils.tabpy_user:main", 101 | ], 102 | }, 103 | setup_requires=["pytest-runner"], 104 | test_suite="pytest", 105 | ) 106 | 107 | 108 | if __name__ == "__main__": 109 | setup_package() 110 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start tabpy 4 | tabpy & 5 | 6 | # Wait for server to become available, wait 1 min maximum 7 | attempt_counter=0 8 | max_attempts=20 9 | 10 | until $(curl --output /dev/null --silent --head --fail localhost:9004); do 11 | if [ ${attempt_counter} -eq ${max_attempts} ];then 12 | echo "Maximum attempts reached, tabpy server not started" 13 | exit 1 14 | fi 15 | 16 | echo "Waiting for tabpy server" 17 | attempt_counter=$(($attempt_counter+1)) 18 | sleep 3 19 | done 20 | 21 | # Deploy tabpy models 22 | tabpy-deploy-models & 23 | wait -------------------------------------------------------------------------------- /tabpy/VERSION: -------------------------------------------------------------------------------- 1 | 2.13.0 2 | -------------------------------------------------------------------------------- /tabpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/__init__.py -------------------------------------------------------------------------------- /tabpy/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/models/__init__.py -------------------------------------------------------------------------------- /tabpy/models/deploy_models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import platform 4 | import subprocess 5 | import sys 6 | from tabpy.models.utils import setup_utils 7 | 8 | 9 | def main(): 10 | # Determine if we run python or python3 11 | py = "python" if platform.system() == "Windows" else "python3" 12 | 13 | file_path = sys.argv[1] if len(sys.argv) > 1 else setup_utils.get_default_config_file_path() 14 | print(f"Using config file at {file_path}") 15 | 16 | port, auth_on, prefix = setup_utils.parse_config(file_path) 17 | auth_args = setup_utils.get_creds() if auth_on else [] 18 | 19 | directory = str(Path(__file__).resolve().parent / "scripts") 20 | # Deploy each model in the scripts directory 21 | for filename in os.listdir(directory): 22 | subprocess.run([py, f"{directory}/{filename}", file_path] + auth_args) 23 | 24 | 25 | if __name__ == "__main__": 26 | main() 27 | -------------------------------------------------------------------------------- /tabpy/models/scripts/ANOVA.py: -------------------------------------------------------------------------------- 1 | import scipy.stats as stats 2 | from tabpy.models.utils import setup_utils 3 | 4 | 5 | def anova(_arg1, _arg2, *_argN): 6 | """ 7 | ANOVA is a statistical hypothesis test that is used to compare 8 | two or more group means for equality.For more information on 9 | the function and how to use it please refer to tabpy-tools.md 10 | """ 11 | 12 | cols = [_arg1, _arg2] + list(_argN) 13 | for col in cols: 14 | if not isinstance(col[0], (int, float)): 15 | print("values must be numeric") 16 | raise ValueError 17 | _, p_value = stats.f_oneway(_arg1, _arg2, *_argN) 18 | return p_value 19 | 20 | 21 | if __name__ == "__main__": 22 | setup_utils.deploy_model("anova", anova, "Returns the p-value form an ANOVA test") 23 | -------------------------------------------------------------------------------- /tabpy/models/scripts/PCA.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from numpy import array 3 | from sklearn.decomposition import PCA as sklearnPCA 4 | from sklearn.preprocessing import StandardScaler 5 | from sklearn.preprocessing import LabelEncoder 6 | from sklearn.preprocessing import OneHotEncoder 7 | from tabpy.models.utils import setup_utils 8 | 9 | 10 | def PCA(component, _arg1, _arg2, *_argN): 11 | """ 12 | Principal Component Analysis is a technique that extracts the key 13 | distinct components from a high dimensional space whie attempting 14 | to capture as much of the variance as possible. For more information 15 | on the function and how to use it please refer to tabpy-tools.md 16 | """ 17 | cols = [_arg1, _arg2] + list(_argN) 18 | encodedCols = [] 19 | labelEncoder = LabelEncoder() 20 | oneHotEncoder = OneHotEncoder(categories="auto", sparse=False) 21 | 22 | for col in cols: 23 | if isinstance(col[0], (int, float)): 24 | encodedCols.append(col) 25 | elif type(col[0]) is bool: 26 | intCol = array(col) 27 | encodedCols.append(intCol.astype(int)) 28 | else: 29 | if len(set(col)) > 25: 30 | print( 31 | "ERROR: Non-numeric arguments cannot have more than " 32 | "25 unique values" 33 | ) 34 | raise ValueError 35 | integerEncoded = labelEncoder.fit_transform(array(col)) 36 | integerEncoded = integerEncoded.reshape(len(col), 1) 37 | oneHotEncoded = oneHotEncoder.fit_transform(integerEncoded) 38 | transformedMatrix = oneHotEncoded.transpose() 39 | encodedCols += list(transformedMatrix) 40 | 41 | dataDict = {} 42 | for i in range(len(encodedCols)): 43 | dataDict[f"col{1 + i}"] = list(encodedCols[i]) 44 | 45 | if component <= 0 or component > len(dataDict): 46 | print("ERROR: Component specified must be >= 0 and " "<= number of arguments") 47 | raise ValueError 48 | 49 | df = pd.DataFrame(data=dataDict, dtype=float) 50 | scale = StandardScaler() 51 | scaledData = scale.fit_transform(df) 52 | 53 | pca = sklearnPCA() 54 | pcaComponents = pca.fit_transform(scaledData) 55 | 56 | return pcaComponents[:, component - 1].tolist() 57 | 58 | 59 | if __name__ == "__main__": 60 | setup_utils.deploy_model("PCA", PCA, "Returns the specified principal component") 61 | -------------------------------------------------------------------------------- /tabpy/models/scripts/SentimentAnalysis.py: -------------------------------------------------------------------------------- 1 | from textblob import TextBlob 2 | import nltk 3 | from nltk.sentiment.vader import SentimentIntensityAnalyzer 4 | from tabpy.models.utils import setup_utils 5 | 6 | 7 | import ssl 8 | 9 | _ctx = ssl._create_unverified_context 10 | ssl._create_default_https_context = _ctx 11 | 12 | 13 | nltk.download("vader_lexicon") 14 | nltk.download("punkt") 15 | 16 | 17 | def SentimentAnalysis(_arg1, library="nltk"): 18 | """ 19 | Sentiment Analysis is a procedure that assigns a score from -1 to 1 20 | for a piece of text with -1 being negative and 1 being positive. For 21 | more information on the function and how to use it please refer to 22 | tabpy-tools.md 23 | """ 24 | if not (isinstance(_arg1[0], str)): 25 | raise TypeError 26 | 27 | supportedLibraries = {"nltk", "textblob"} 28 | 29 | library = library.lower() 30 | if library not in supportedLibraries: 31 | raise ValueError 32 | 33 | scores = [] 34 | if library == "nltk": 35 | sid = SentimentIntensityAnalyzer() 36 | for text in _arg1: 37 | sentimentResults = sid.polarity_scores(text) 38 | score = sentimentResults["compound"] 39 | scores.append(score) 40 | elif library == "textblob": 41 | for text in _arg1: 42 | currScore = TextBlob(text) 43 | scores.append(currScore.sentiment.polarity) 44 | return scores 45 | 46 | 47 | if __name__ == "__main__": 48 | setup_utils.deploy_model( 49 | "Sentiment Analysis", 50 | SentimentAnalysis, 51 | "Returns a sentiment score between -1 and 1 for " "a given string", 52 | ) 53 | -------------------------------------------------------------------------------- /tabpy/models/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/models/scripts/__init__.py -------------------------------------------------------------------------------- /tabpy/models/scripts/tTest.py: -------------------------------------------------------------------------------- 1 | from scipy import stats 2 | from tabpy.models.utils import setup_utils 3 | 4 | 5 | def ttest(_arg1, _arg2): 6 | """ 7 | T-Test is a statistical hypothesis test that is used to compare 8 | two sample means or a sample’s mean against a known population mean. 9 | For more information on the function and how to use it please refer 10 | to tabpy-tools.md 11 | """ 12 | # one sample test with mean 13 | if len(_arg2) == 1: 14 | test_stat, p_value = stats.ttest_1samp(_arg1, _arg2) 15 | return p_value 16 | # two sample t-test where _arg1 is numeric and _arg2 is a binary factor 17 | elif len(set(_arg2)) == 2: 18 | # each sample in _arg1 needs to have a corresponding classification 19 | # in _arg2 20 | if not (len(_arg1) == len(_arg2)): 21 | raise ValueError 22 | class1, class2 = set(_arg2) 23 | sample1 = [] 24 | sample2 = [] 25 | for i in range(len(_arg1)): 26 | if _arg2[i] == class1: 27 | sample1.append(_arg1[i]) 28 | else: 29 | sample2.append(_arg1[i]) 30 | test_stat, p_value = stats.ttest_ind(sample1, sample2, equal_var=False) 31 | return p_value 32 | # arg1 is a sample and arg2 is a sample 33 | else: 34 | test_stat, p_value = stats.ttest_ind(_arg1, _arg2, equal_var=False) 35 | return p_value 36 | 37 | 38 | if __name__ == "__main__": 39 | setup_utils.deploy_model("ttest", ttest, "Returns the p-value form a t-test") 40 | -------------------------------------------------------------------------------- /tabpy/models/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/models/utils/__init__.py -------------------------------------------------------------------------------- /tabpy/models/utils/setup_utils.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import getpass 3 | import os 4 | import sys 5 | from tabpy.tabpy_tools.client import Client 6 | 7 | 8 | def get_default_config_file_path(): 9 | import tabpy 10 | 11 | pkg_path = os.path.dirname(tabpy.__file__) 12 | config_file_path = os.path.join(pkg_path, "tabpy_server", "common", "default.conf") 13 | return config_file_path 14 | 15 | 16 | def parse_config(config_file_path): 17 | config = configparser.ConfigParser() 18 | config.read(config_file_path) 19 | tabpy_config = config["TabPy"] 20 | 21 | port = 9004 22 | if "TABPY_PORT" in tabpy_config: 23 | port = tabpy_config["TABPY_PORT"] 24 | 25 | auth_on = "TABPY_PWD_FILE" in tabpy_config 26 | ssl_on = ( 27 | "TABPY_TRANSFER_PROTOCOL" in tabpy_config 28 | and "TABPY_CERTIFICATE_FILE" in tabpy_config 29 | and "TABPY_KEY_FILE" in tabpy_config 30 | ) 31 | prefix = "https" if ssl_on else "http" 32 | return port, auth_on, prefix 33 | 34 | 35 | def get_creds(): 36 | if sys.stdin.isatty(): 37 | user = input("Username: ") 38 | passwd = getpass.getpass("Password: ") 39 | else: 40 | user = sys.stdin.readline().rstrip() 41 | passwd = sys.stdin.readline().rstrip() 42 | return [user, passwd] 43 | 44 | 45 | def deploy_model(funcName, func, funcDescription): 46 | # running from deploy_models.py 47 | config_file_path = sys.argv[1] if len(sys.argv) > 1 else get_default_config_file_path() 48 | port, auth_on, prefix = parse_config(config_file_path) 49 | 50 | connection = Client(f"{prefix}://localhost:{port}/") 51 | 52 | if auth_on: 53 | # credentials are passed in from setup.py 54 | user, passwd = sys.argv[2], sys.argv[3] if len(sys.argv) == 4 else get_creds() 55 | connection.set_credentials(user, passwd) 56 | 57 | connection.deploy(funcName, func, funcDescription, override=True) 58 | print(f"Successfully deployed {funcName}") 59 | -------------------------------------------------------------------------------- /tabpy/tabpy.py: -------------------------------------------------------------------------------- 1 | """ 2 | TabPy Server. 3 | 4 | Usage: 5 | tabpy [-h] | [--help] 6 | tabpy [--config ] [--disable-auth-warning] 7 | 8 | Options: 9 | -h --help Show this screen. 10 | --config Path to a config file. 11 | --disable-auth-warning Disable authentication warning. 12 | """ 13 | 14 | import docopt 15 | import os 16 | from pathlib import Path 17 | 18 | 19 | def read_version(): 20 | ver = "unknown" 21 | 22 | import tabpy 23 | 24 | pkg_path = os.path.dirname(tabpy.__file__) 25 | ver_file_path = os.path.join(pkg_path, "VERSION") 26 | if Path(ver_file_path).exists(): 27 | with open(ver_file_path) as f: 28 | ver = f.read().strip() 29 | else: 30 | ver = f"Version Unknown, (file {ver_file_path} not found)" 31 | 32 | return ver 33 | 34 | 35 | __version__ = read_version() 36 | 37 | 38 | def main(): 39 | args = docopt.docopt(__doc__) 40 | config = args["--config"] or None 41 | 42 | disable_auth_warning = False 43 | if args["--disable-auth-warning"]: 44 | disable_auth_warning = True 45 | 46 | from tabpy.tabpy_server.app.app import TabPyApp 47 | 48 | app = TabPyApp(config, disable_auth_warning) 49 | app.run() 50 | 51 | 52 | if __name__ == "__main__": 53 | main() 54 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_server/app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/app/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_server/app/app_parameters.py: -------------------------------------------------------------------------------- 1 | class ConfigParameters: 2 | """ 3 | Configuration settings names 4 | """ 5 | 6 | TABPY_PWD_FILE = "TABPY_PWD_FILE" 7 | TABPY_PORT = "TABPY_PORT" 8 | TABPY_QUERY_OBJECT_PATH = "TABPY_QUERY_OBJECT_PATH" 9 | TABPY_STATE_PATH = "TABPY_STATE_PATH" 10 | TABPY_TRANSFER_PROTOCOL = "TABPY_TRANSFER_PROTOCOL" 11 | TABPY_CERTIFICATE_FILE = "TABPY_CERTIFICATE_FILE" 12 | TABPY_KEY_FILE = "TABPY_KEY_FILE" 13 | TABPY_MINIMUM_TLS_VERSION = "TABPY_MINIMUM_TLS_VERSION" 14 | TABPY_LOG_DETAILS = "TABPY_LOG_DETAILS" 15 | TABPY_STATIC_PATH = "TABPY_STATIC_PATH" 16 | TABPY_MAX_REQUEST_SIZE_MB = "TABPY_MAX_REQUEST_SIZE_MB" 17 | TABPY_EVALUATE_ENABLE = "TABPY_EVALUATE_ENABLE" 18 | TABPY_EVALUATE_TIMEOUT = "TABPY_EVALUATE_TIMEOUT" 19 | TABPY_GZIP_ENABLE = "TABPY_GZIP_ENABLE" 20 | 21 | # Arrow specific settings 22 | TABPY_ARROW_ENABLE = "TABPY_ARROW_ENABLE" 23 | TABPY_ARROWFLIGHT_PORT = "TABPY_ARROWFLIGHT_PORT" 24 | 25 | 26 | class SettingsParameters: 27 | """ 28 | Application (TabPyApp) settings names 29 | """ 30 | 31 | TransferProtocol = "transfer_protocol" 32 | Port = "port" 33 | ServerVersion = "server_version" 34 | UploadDir = "upload_dir" 35 | CertificateFile = "certificate_file" 36 | KeyFile = "key_file" 37 | MinimumTLSVersion = "minimum_tls_version" 38 | StateFilePath = "state_file_path" 39 | ApiVersions = "versions" 40 | LogRequestContext = "log_request_context" 41 | StaticPath = "static_path" 42 | MaxRequestSizeInMb = "max_request_size_in_mb" 43 | EvaluateTimeout = "evaluate_timeout" 44 | EvaluateEnabled = "evaluate_enabled" 45 | GzipEnabled = "gzip_enabled" 46 | 47 | # Arrow specific settings 48 | ArrowEnabled = "arrow_enabled" 49 | ArrowFlightPort = "arrowflight_port" 50 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/app/arrow_server.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | import ast 20 | import logging 21 | import threading 22 | import time 23 | import uuid 24 | 25 | import pyarrow 26 | import pyarrow.flight 27 | 28 | 29 | logger = logging.getLogger('__main__.' + __name__) 30 | 31 | class FlightServer(pyarrow.flight.FlightServerBase): 32 | def __init__(self, host="localhost", location=None, 33 | tls_certificates=None, verify_client=False, 34 | root_certificates=None, auth_handler=None, middleware=None): 35 | super(FlightServer, self).__init__( 36 | location, auth_handler, tls_certificates, verify_client, 37 | root_certificates, middleware) 38 | self.flights = {} 39 | self.host = host 40 | self.tls_certificates = tls_certificates 41 | self.location = location 42 | 43 | @classmethod 44 | def descriptor_to_key(self, descriptor): 45 | return (descriptor.descriptor_type.value, descriptor.command, 46 | tuple(descriptor.path or tuple())) 47 | 48 | def _make_flight_info(self, key, descriptor, table): 49 | if self.tls_certificates: 50 | location = pyarrow.flight.Location.for_grpc_tls( 51 | self.host, self.port) 52 | else: 53 | location = pyarrow.flight.Location.for_grpc_tcp( 54 | self.host, self.port) 55 | endpoints = [pyarrow.flight.FlightEndpoint(repr(key), [location]), ] 56 | 57 | mock_sink = pyarrow.MockOutputStream() 58 | stream_writer = pyarrow.RecordBatchStreamWriter( 59 | mock_sink, table.schema) 60 | stream_writer.write_table(table) 61 | stream_writer.close() 62 | data_size = mock_sink.size() 63 | 64 | return pyarrow.flight.FlightInfo(table.schema, 65 | descriptor, endpoints, 66 | table.num_rows, data_size) 67 | 68 | def list_flights(self, context, criteria): 69 | for key, table in self.flights.items(): 70 | if key[1] is not None: 71 | descriptor = \ 72 | pyarrow.flight.FlightDescriptor.for_command(key[1]) 73 | else: 74 | descriptor = pyarrow.flight.FlightDescriptor.for_path(*key[2]) 75 | 76 | yield self._make_flight_info(key, descriptor, table) 77 | 78 | def get_flight_info(self, context, descriptor): 79 | key = FlightServer.descriptor_to_key(descriptor) 80 | logger.info(f"get_flight_info: key={key}") 81 | if key in self.flights: 82 | table = self.flights[key] 83 | return self._make_flight_info(key, descriptor, table) 84 | raise KeyError('Flight not found.') 85 | 86 | def do_put(self, context, descriptor, reader, writer): 87 | key = FlightServer.descriptor_to_key(descriptor) 88 | logger.info(f"do_put: key={key}") 89 | self.flights[key] = reader.read_all() 90 | 91 | def do_get(self, context, ticket): 92 | logger.info(f"do_get: ticket={ticket}") 93 | key = ast.literal_eval(ticket.ticket.decode()) 94 | if key not in self.flights: 95 | logger.warn(f"do_get: key={key} not found") 96 | return None 97 | logger.info(f"do_get: returning key={key}") 98 | flight = self.flights.pop(key) 99 | return pyarrow.flight.RecordBatchStream(flight) 100 | 101 | def list_actions(self, context): 102 | return iter([ 103 | ("getUniquePath", "Get a unique FlightDescriptor path to put data to."), 104 | ("clear", "Clear the stored flights."), 105 | ("shutdown", "Shut down this server."), 106 | ]) 107 | 108 | def do_action(self, context, action): 109 | logger.info(f"do_action: action={action.type}") 110 | if action.type == "getUniquePath": 111 | uniqueId = str(uuid.uuid4()) 112 | logger.info(f"getUniquePath id={uniqueId}") 113 | yield uniqueId.encode('utf-8') 114 | elif action.type == "clear": 115 | self._clear() 116 | elif action.type == "healthcheck": 117 | pass 118 | elif action.type == "shutdown": 119 | self._clear() 120 | yield pyarrow.flight.Result(pyarrow.py_buffer(b'Shutdown!')) 121 | # Shut down on background thread to avoid blocking current 122 | # request 123 | threading.Thread(target=self._shutdown).start() 124 | else: 125 | raise KeyError("Unknown action {!r}".format(action.type)) 126 | 127 | def _clear(self): 128 | """Clear the stored flights.""" 129 | self.flights = {} 130 | 131 | def _shutdown(self): 132 | """Shut down after a delay.""" 133 | logger.info("Server is shutting down...") 134 | time.sleep(2) 135 | self.shutdown() 136 | 137 | def start(server): 138 | logger.info(f"Serving on {server.location}") 139 | server.serve() 140 | 141 | 142 | if __name__ == '__main__': 143 | start() -------------------------------------------------------------------------------- /tabpy/tabpy_server/app/util.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from datetime import datetime 3 | import logging 4 | from OpenSSL import crypto 5 | import os 6 | 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def validate_cert(cert_file_path): 12 | with open(cert_file_path, "r") as f: 13 | cert_buf = f.read() 14 | 15 | cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_buf) 16 | 17 | date_format, encoding = "%Y%m%d%H%M%SZ", "ascii" 18 | not_before = datetime.strptime(cert.get_notBefore().decode(encoding), date_format) 19 | not_after = datetime.strptime(cert.get_notAfter().decode(encoding), date_format) 20 | now = datetime.utcnow() 21 | 22 | https_error = "Error using HTTPS: " 23 | if now < not_before: 24 | msg = https_error + f"The certificate provided is not valid until {not_before}." 25 | logger.critical(msg) 26 | raise RuntimeError(msg) 27 | if now > not_after: 28 | msg = https_error + f"The certificate provided expired on {not_after}." 29 | logger.critical(msg) 30 | raise RuntimeError(msg) 31 | 32 | 33 | def parse_pwd_file(pwd_file_name): 34 | """ 35 | Parses passwords file and returns set of credentials. 36 | 37 | Parameters 38 | ---------- 39 | pwd_file_name : str 40 | Passwords file name. 41 | 42 | Returns 43 | ------- 44 | succeeded : bool 45 | True if specified file was parsed successfully. 46 | False if there were any issues with parsing specified file. 47 | 48 | credentials : dict 49 | Credentials from the file. Empty if succeeded is False. 50 | """ 51 | logger.info(f"Parsing passwords file {pwd_file_name}...") 52 | 53 | if not os.path.isfile(pwd_file_name): 54 | logger.critical(f"Passwords file {pwd_file_name} not found") 55 | return False, {} 56 | 57 | credentials = {} 58 | with open(pwd_file_name) as pwd_file: 59 | pwd_file_reader = csv.reader(pwd_file, delimiter=" ") 60 | for row in pwd_file_reader: 61 | # skip empty lines 62 | if len(row) == 0: 63 | continue 64 | 65 | # skip commented lines 66 | if row[0][0] == "#": 67 | continue 68 | 69 | if len(row) != 2: 70 | logger.error(f'Incorrect entry "{row}" in password file') 71 | return False, {} 72 | 73 | login = row[0].lower() 74 | if login in credentials: 75 | logger.error( 76 | f"Multiple entries for username {login} in password file" 77 | ) 78 | return False, {} 79 | 80 | if len(row[1]) > 0: 81 | credentials[login] = row[1] 82 | logger.debug(f"Found username {login}") 83 | else: 84 | logger.warning(f"Found username {row[0]} but no password") 85 | return False, {} 86 | 87 | logger.info("Authentication is enabled") 88 | return True, credentials 89 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/common/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_server/common/default.conf: -------------------------------------------------------------------------------- 1 | [TabPy] 2 | # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects 3 | # TABPY_PORT = 9004 4 | # TABPY_STATE_PATH = ./tabpy/tabpy_server 5 | 6 | # Where static pages live 7 | # TABPY_STATIC_PATH = ./tabpy/tabpy_server/static 8 | 9 | # For how to configure TabPy authentication read 10 | # Authentication section in docs/server-config.md. 11 | # TABPY_PWD_FILE = /path/to/password/file.txt 12 | 13 | # To set up secure TabPy uncomment and modify the following lines. 14 | # Note only PEM-encoded x509 certificates are supported. 15 | # TABPY_TRANSFER_PROTOCOL = https 16 | # TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt 17 | # TABPY_KEY_FILE = /path/to/key/file.key 18 | # TABPY_MINIMUM_TLS_VERSION = TLSv1_2 19 | 20 | # Log additional request details including caller IP, full URL, client 21 | # end user info if provided. 22 | # TABPY_LOG_DETAILS = true 23 | 24 | # Limit request size (in Mb) - any request which size exceeds 25 | # specified amount will be rejected by TabPy. 26 | # Default value is 100 Mb. 27 | # TABPY_MAX_REQUEST_SIZE_MB = 100 28 | 29 | # Enable evaluate api to execute ad-hoc Python scripts 30 | # Enabled by default. Disabling it will result in 404 error. 31 | # TABPY_EVALUATE_ENABLE = true 32 | 33 | # Configure how long a custom script provided to the /evaluate method 34 | # will run before throwing a TimeoutError. 35 | # The value should be a float representing the timeout time in seconds. 36 | # TABPY_EVALUATE_TIMEOUT = 30 37 | 38 | # Enable Gzip compression for requests and responses. 39 | # TABPY_GZIP_ENABLE = true 40 | 41 | [loggers] 42 | keys=root 43 | 44 | [handlers] 45 | keys=rootHandler,rotatingFileHandler 46 | 47 | [formatters] 48 | keys=rootFormatter 49 | 50 | [logger_root] 51 | level=DEBUG 52 | handlers=rootHandler,rotatingFileHandler 53 | qualname=root 54 | propagete=0 55 | 56 | [handler_rootHandler] 57 | class=StreamHandler 58 | level=INFO 59 | formatter=rootFormatter 60 | args=(sys.stdout,) 61 | 62 | [handler_rotatingFileHandler] 63 | class=handlers.RotatingFileHandler 64 | level=DEBUG 65 | formatter=rootFormatter 66 | args=('tabpy_log.log', 'a', 1000000, 5) 67 | 68 | [formatter_rootFormatter] 69 | format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s 70 | datefmt=%Y-%m-%d,%H:%M:%S 71 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/common/endpoint_file_mgr.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functionality required for managing endpoint objects in 3 | TabPy. It provides a way to download endpoint files from remote 4 | and then properly cleanup local the endpoint files on update/remove of endpoint 5 | objects. 6 | 7 | The local temporary files for TabPy will by default located at 8 | /tmp/query_objects 9 | 10 | """ 11 | import logging 12 | import os 13 | import shutil 14 | from re import compile as _compile 15 | 16 | 17 | _name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") 18 | 19 | 20 | def _check_endpoint_name(name, logger=logging.getLogger(__name__)): 21 | """Checks that the endpoint name is valid by comparing it with an RE and 22 | checking that it is not reserved.""" 23 | if not isinstance(name, str): 24 | msg = "Endpoint name must be a string" 25 | logger.log(logging.CRITICAL, msg) 26 | raise TypeError(msg) 27 | 28 | if name == "": 29 | msg = "Endpoint name cannot be empty" 30 | logger.log(logging.CRITICAL, msg) 31 | raise ValueError(msg) 32 | 33 | if not _name_checker.match(name): 34 | msg = ( 35 | "Endpoint name can only contain: a-z, A-Z, 0-9," 36 | " underscore, hyphens and spaces." 37 | ) 38 | logger.log(logging.CRITICAL, msg) 39 | raise ValueError(msg) 40 | 41 | 42 | def grab_files(directory): 43 | """ 44 | Generator that returns all files in a directory. 45 | """ 46 | if not os.path.isdir(directory): 47 | return 48 | else: 49 | for name in os.listdir(directory): 50 | full_path = os.path.join(directory, name) 51 | if os.path.isdir(full_path): 52 | for entry in grab_files(full_path): 53 | yield entry 54 | elif os.path.isfile(full_path): 55 | yield full_path 56 | 57 | 58 | def cleanup_endpoint_files( 59 | name, query_path, logger=logging.getLogger(__name__), retain_versions=None 60 | ): 61 | """ 62 | Cleanup the disk space a certain endpiont uses. 63 | 64 | Parameters 65 | ---------- 66 | name : str 67 | The endpoint name 68 | 69 | retain_version : int, optional 70 | If given, then all files for this endpoint are removed except the 71 | folder for the given version, otherwise, all files for that endpoint 72 | are removed. 73 | """ 74 | _check_endpoint_name(name, logger=logger) 75 | local_dir = os.path.join(query_path, name) 76 | 77 | # nothing to clean, this is true for state file path where we load 78 | # Query Object directly from the state path instead of downloading 79 | # to temporary location 80 | if not os.path.exists(local_dir): 81 | return 82 | 83 | if not retain_versions: 84 | shutil.rmtree(local_dir) 85 | else: 86 | retain_folders = [ 87 | os.path.join(local_dir, str(version)) for version in retain_versions 88 | ] 89 | logger.log(logging.INFO, f"Retain folders: {retain_folders}") 90 | 91 | for file_or_dir in os.listdir(local_dir): 92 | candidate_dir = os.path.join(local_dir, file_or_dir) 93 | if os.path.isdir(candidate_dir) and (candidate_dir not in retain_folders): 94 | shutil.rmtree(candidate_dir) 95 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/common/messages.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from abc import ABCMeta 3 | from collections import namedtuple 4 | import json 5 | 6 | 7 | class Msg: 8 | """ 9 | An abstract base class for all messages used for communicating between 10 | the WebServices. 11 | 12 | The minimal functionality is the ability to instantiate a Msg from JSON 13 | and to write a Msg instance to JSON. 14 | 15 | We use namedtuples because they are lightweight and immutable. The splat 16 | operator (*) that we inherit from namedtuple is also convenient. We empty 17 | __slots__ to avoid unnecessary overhead. 18 | """ 19 | 20 | __metaclass__ = ABCMeta 21 | 22 | @abc.abstractmethod 23 | def for_json(self): 24 | d = self._asdict() 25 | type_str = self.__class__.__name__ 26 | d.update({"type": type_str}) 27 | return d 28 | 29 | @abc.abstractmethod 30 | def to_json(self): 31 | return json.dumps(self.for_json()) 32 | 33 | @staticmethod 34 | def from_json(str): 35 | d = json.loads(str) 36 | type_str = d["type"] 37 | del d["type"] 38 | return eval(type_str)(**d) 39 | 40 | 41 | class LoadSuccessful( 42 | namedtuple( 43 | "LoadSuccessful", ["uri", "path", "version", "is_update", "endpoint_type"] 44 | ), 45 | Msg, 46 | ): 47 | __slots__ = () 48 | 49 | 50 | class LoadFailed(namedtuple("LoadFailed", ["uri", "version", "error_msg"]), Msg): 51 | __slots__ = () 52 | 53 | 54 | class LoadInProgress( 55 | namedtuple( 56 | "LoadInProgress", ["uri", "path", "version", "is_update", "endpoint_type"] 57 | ), 58 | Msg, 59 | ): 60 | __slots__ = () 61 | 62 | 63 | class Query(namedtuple("Query", ["uri", "params"]), Msg): 64 | __slots__ = () 65 | 66 | 67 | class QuerySuccessful( 68 | namedtuple("QuerySuccessful", ["uri", "version", "response"]), Msg 69 | ): 70 | __slots__ = () 71 | 72 | 73 | class LoadObject( 74 | namedtuple("LoadObject", ["uri", "url", "version", "is_update", "endpoint_type"]), 75 | Msg, 76 | ): 77 | __slots__ = () 78 | 79 | 80 | class DeleteObjects(namedtuple("DeleteObjects", ["uris"]), Msg): 81 | __slots__ = () 82 | 83 | 84 | # Used for testing to flush out objects 85 | class FlushObjects(namedtuple("FlushObjects", []), Msg): 86 | __slots__ = () 87 | 88 | 89 | class ObjectsDeleted(namedtuple("ObjectsDeleted", ["uris"]), Msg): 90 | __slots__ = () 91 | 92 | 93 | class ObjectsFlushed(namedtuple("ObjectsFlushed", ["n_before", "n_after"]), Msg): 94 | __slots__ = () 95 | 96 | 97 | class CountObjects(namedtuple("CountObjects", []), Msg): 98 | __slots__ = () 99 | 100 | 101 | class ObjectCount(namedtuple("ObjectCount", ["count"]), Msg): 102 | __slots__ = () 103 | 104 | 105 | class ListObjects(namedtuple("ListObjects", []), Msg): 106 | __slots__ = () 107 | 108 | 109 | class ObjectList(namedtuple("ObjectList", ["objects"]), Msg): 110 | __slots__ = () 111 | 112 | 113 | class UnknownURI(namedtuple("UnknownURI", ["uri"]), Msg): 114 | __slots__ = () 115 | 116 | 117 | class UnknownMessage(namedtuple("UnknownMessage", ["msg"]), Msg): 118 | __slots__ = () 119 | 120 | 121 | class DownloadSkipped( 122 | namedtuple("DownloadSkipped", ["uri", "version", "msg", "host"]), Msg 123 | ): 124 | __slots__ = () 125 | 126 | 127 | class QueryFailed(namedtuple("QueryFailed", ["uri", "error"]), Msg): 128 | __slots__ = () 129 | 130 | 131 | class QueryError(namedtuple("QueryError", ["uri", "error"]), Msg): 132 | __slots__ = () 133 | 134 | 135 | class CheckHealth(namedtuple("CheckHealth", []), Msg): 136 | __slots__ = () 137 | 138 | 139 | class Healthy(namedtuple("Healthy", []), Msg): 140 | __slots__ = () 141 | 142 | 143 | class Unhealthy(namedtuple("Unhealthy", []), Msg): 144 | __slots__ = () 145 | 146 | 147 | class Ping(namedtuple("Ping", ["id"]), Msg): 148 | __slots__ = () 149 | 150 | 151 | class Pong(namedtuple("Pong", ["id"]), Msg): 152 | __slots__ = () 153 | 154 | 155 | class Listening(namedtuple("Listening", []), Msg): 156 | __slots__ = () 157 | 158 | 159 | class EngineFailure(namedtuple("EngineFailure", ["error"]), Msg): 160 | __slots__ = () 161 | 162 | 163 | class FlushLogs(namedtuple("FlushLogs", []), Msg): 164 | __slots__ = () 165 | 166 | 167 | class LogsFlushed(namedtuple("LogsFlushed", []), Msg): 168 | __slots__ = () 169 | 170 | 171 | class ServiceError(namedtuple("ServiceError", ["error"]), Msg): 172 | __slots__ = () 173 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/common/util.py: -------------------------------------------------------------------------------- 1 | def format_exception(e, context): 2 | err_msg = f"{e.__class__.__name__} : {str(e)}" 3 | return err_msg 4 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | from tabpy.tabpy_server.handlers.base_handler import BaseHandler 2 | from tabpy.tabpy_server.handlers.management_handler import ManagementHandler 3 | 4 | from tabpy.tabpy_server.handlers.endpoint_handler import EndpointHandler 5 | from tabpy.tabpy_server.handlers.endpoints_handler import EndpointsHandler 6 | from tabpy.tabpy_server.handlers.evaluation_plane_handler import EvaluationPlaneDisabledHandler 7 | from tabpy.tabpy_server.handlers.evaluation_plane_handler import EvaluationPlaneHandler 8 | from tabpy.tabpy_server.handlers.query_plane_handler import QueryPlaneHandler 9 | from tabpy.tabpy_server.handlers.service_info_handler import ServiceInfoHandler 10 | from tabpy.tabpy_server.handlers.status_handler import StatusHandler 11 | from tabpy.tabpy_server.handlers.upload_destination_handler import ( 12 | UploadDestinationHandler, 13 | ) 14 | from tabpy.tabpy_server.handlers.no_op_auth_handler import NoOpAuthHandler 15 | from tabpy.tabpy_server.handlers.basic_auth_server_middleware_factory import ( 16 | BasicAuthServerMiddlewareFactory, 17 | ) 18 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/basic_auth_server_middleware_factory.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import secrets 3 | 4 | from pyarrow.flight import ServerMiddlewareFactory, ServerMiddleware 5 | from pyarrow.flight import FlightUnauthenticatedError 6 | 7 | from tabpy.tabpy_server.handlers.util import hash_password 8 | 9 | class BasicAuthServerMiddleware(ServerMiddleware): 10 | def __init__(self, token): 11 | self.token = token 12 | 13 | def sending_headers(self): 14 | return {"authorization": f"Bearer {self.token}"} 15 | 16 | class BasicAuthServerMiddlewareFactory(ServerMiddlewareFactory): 17 | def __init__(self, creds): 18 | self.creds = creds 19 | self.tokens = {} 20 | 21 | def is_valid_user(self, username, password): 22 | if username not in self.creds: 23 | return False 24 | hashed_pwd = hash_password(username, password) 25 | return self.creds[username].lower() == hashed_pwd.lower() 26 | 27 | def start_call(self, info, headers): 28 | auth_header = None 29 | for header in headers: 30 | if header.lower() == "authorization": 31 | auth_header = headers[header][0] 32 | break 33 | 34 | if not auth_header: 35 | raise FlightUnauthenticatedError("No credentials supplied") 36 | 37 | auth_type, _, value = auth_header.partition(" ") 38 | 39 | if auth_type == "Basic": 40 | decoded = base64.b64decode(value).decode("utf-8") 41 | username, _, password = decoded.partition(":") 42 | if not self.is_valid_user(username, password): 43 | raise FlightUnauthenticatedError("Invalid credentials") 44 | token = secrets.token_urlsafe(32) 45 | self.tokens[token] = username 46 | return BasicAuthServerMiddleware(token) 47 | 48 | raise FlightUnauthenticatedError("No credentials supplied") 49 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/endpoint_handler.py: -------------------------------------------------------------------------------- 1 | """ 2 | HTTP handeler to serve specific endpoint request like 3 | http://myserver:9004/endpoints/mymodel 4 | 5 | For how generic endpoints requests is served look 6 | at endpoints_handler.py 7 | """ 8 | 9 | import json 10 | import logging 11 | import shutil 12 | from tabpy.tabpy_server.common.util import format_exception 13 | from tabpy.tabpy_server.handlers import ManagementHandler 14 | from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD 15 | from tabpy.tabpy_server.management.state import get_query_object_path 16 | from tabpy.tabpy_server.psws.callbacks import on_state_change 17 | from tabpy.tabpy_server.handlers.util import AuthErrorStates 18 | from tornado import gen 19 | 20 | 21 | class EndpointHandler(ManagementHandler): 22 | def initialize(self, app): 23 | super(EndpointHandler, self).initialize(app) 24 | 25 | def get(self, endpoint_name): 26 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 27 | self.fail_with_auth_error() 28 | return 29 | 30 | self.logger.log(logging.DEBUG, f"Processing GET for /endpoints/{endpoint_name}") 31 | 32 | self._add_CORS_header() 33 | if not endpoint_name: 34 | self.write(json.dumps(self.tabpy_state.get_endpoints())) 35 | else: 36 | if endpoint_name in self.tabpy_state.get_endpoints(): 37 | self.write(json.dumps(self.tabpy_state.get_endpoints()[endpoint_name])) 38 | else: 39 | self.error_out( 40 | 404, 41 | "Unknown endpoint", 42 | info=f"Endpoint {endpoint_name} is not found", 43 | ) 44 | 45 | @gen.coroutine 46 | def put(self, name): 47 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 48 | self.fail_with_auth_error() 49 | return 50 | 51 | self.logger.log(logging.DEBUG, f"Processing PUT for /endpoints/{name}") 52 | 53 | try: 54 | if not self.request.body: 55 | self.error_out(400, "Input body cannot be empty") 56 | self.finish() 57 | return 58 | try: 59 | request_data = json.loads(self.request.body.decode("utf-8")) 60 | except BaseException as ex: 61 | self.error_out( 62 | 400, log_message="Failed to decode input body", info=str(ex) 63 | ) 64 | self.finish() 65 | return 66 | 67 | # check if endpoint exists 68 | endpoints = self.tabpy_state.get_endpoints(name) 69 | if len(endpoints) == 0: 70 | self.error_out(404, f"endpoint {name} does not exist.") 71 | self.finish() 72 | return 73 | 74 | version_after_update = int(endpoints[name]["version"]) 75 | if request_data.get('should_update_version'): 76 | version_after_update += 1 77 | self.logger.log(logging.INFO, f"Endpoint info: {request_data}") 78 | err_msg = yield self._add_or_update_endpoint( 79 | "update", name, version_after_update, request_data 80 | ) 81 | if err_msg: 82 | self.error_out(400, err_msg) 83 | self.finish() 84 | else: 85 | self.write(self.tabpy_state.get_endpoints(name)) 86 | self.finish() 87 | 88 | except Exception as e: 89 | err_msg = format_exception(e, "update_endpoint") 90 | self.error_out(500, err_msg) 91 | self.finish() 92 | 93 | @gen.coroutine 94 | def delete(self, name): 95 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 96 | self.fail_with_auth_error() 97 | return 98 | 99 | self.logger.log(logging.DEBUG, f"Processing DELETE for /endpoints/{name}") 100 | 101 | try: 102 | endpoints = self.tabpy_state.get_endpoints(name) 103 | if len(endpoints) == 0: 104 | self.error_out(404, f"endpoint {name} does not exist.") 105 | self.finish() 106 | return 107 | 108 | # update state 109 | try: 110 | endpoint_info = self.tabpy_state.delete_endpoint(name) 111 | except Exception as e: 112 | self.error_out(400, f"Error when removing endpoint: {e.message}") 113 | self.finish() 114 | return 115 | 116 | # delete files 117 | if endpoint_info["type"] != "alias": 118 | query_path = get_query_object_path( 119 | self.settings["state_file_path"], name, None 120 | ) 121 | staging_path = query_path.replace("/query_objects/", "/staging/endpoints/") 122 | try: 123 | yield self._delete_po_future(query_path) 124 | yield self._delete_po_future(staging_path) 125 | except Exception as e: 126 | self.error_out(400, f"Error while deleting: {e}") 127 | self.finish() 128 | return 129 | 130 | self.set_status(204) 131 | self.finish() 132 | 133 | except Exception as e: 134 | err_msg = format_exception(e, "delete endpoint") 135 | self.error_out(500, err_msg) 136 | self.finish() 137 | 138 | on_state_change( 139 | self.settings, self.tabpy_state, self.python_service, self.logger 140 | ) 141 | 142 | @gen.coroutine 143 | def _delete_po_future(self, delete_path): 144 | future = STAGING_THREAD.submit(shutil.rmtree, delete_path) 145 | ret = yield future 146 | raise gen.Return(ret) 147 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/endpoints_handler.py: -------------------------------------------------------------------------------- 1 | """ 2 | HTTP handeler to serve general endpoints request, specifically 3 | http://myserver:9004/endpoints 4 | 5 | For how individual endpoint requests are served look 6 | at endpoint_handler.py 7 | """ 8 | 9 | import json 10 | import logging 11 | from tabpy.tabpy_server.common.util import format_exception 12 | from tabpy.tabpy_server.handlers import ManagementHandler 13 | from tabpy.tabpy_server.handlers.util import AuthErrorStates 14 | from tornado import gen 15 | 16 | 17 | class EndpointsHandler(ManagementHandler): 18 | def initialize(self, app): 19 | super(EndpointsHandler, self).initialize(app) 20 | 21 | def get(self): 22 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 23 | self.fail_with_auth_error() 24 | return 25 | 26 | self._add_CORS_header() 27 | self.write(json.dumps(self.tabpy_state.get_endpoints())) 28 | 29 | @gen.coroutine 30 | def post(self): 31 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 32 | self.fail_with_auth_error() 33 | return 34 | 35 | try: 36 | if not self.request.body: 37 | self.error_out(400, "Input body cannot be empty") 38 | self.finish() 39 | return 40 | 41 | try: 42 | request_data = json.loads(self.request.body.decode("utf-8")) 43 | except Exception as ex: 44 | self.error_out(400, "Failed to decode input body", str(ex)) 45 | self.finish() 46 | return 47 | 48 | if "name" not in request_data: 49 | self.error_out(400, "name is required to add an endpoint.") 50 | self.finish() 51 | return 52 | 53 | name = request_data["name"] 54 | 55 | # check if endpoint already exist 56 | if name in self.tabpy_state.get_endpoints(): 57 | self.error_out(400, f"endpoint {name} already exists.") 58 | self.finish() 59 | return 60 | 61 | self.logger.log(logging.DEBUG, f'Adding endpoint "{name}"') 62 | err_msg = yield self._add_or_update_endpoint("add", name, 1, request_data) 63 | if err_msg: 64 | self.error_out(400, err_msg) 65 | else: 66 | self.logger.log(logging.DEBUG, f"Endpoint {name} successfully added") 67 | self.set_status(201) 68 | self.write(self.tabpy_state.get_endpoints(name)) 69 | self.finish() 70 | return 71 | 72 | except Exception as e: 73 | err_msg = format_exception(e, "/add_endpoint") 74 | self.error_out(500, "error adding endpoint", err_msg) 75 | self.finish() 76 | return 77 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/management_handler.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shutil 4 | from re import compile as _compile 5 | from uuid import uuid4 as random_uuid 6 | 7 | from tornado import gen 8 | 9 | from tabpy.tabpy_server.app.app_parameters import SettingsParameters 10 | from tabpy.tabpy_server.handlers import BaseHandler 11 | from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD 12 | from tabpy.tabpy_server.management.state import get_query_object_path 13 | from tabpy.tabpy_server.psws.callbacks import on_state_change 14 | 15 | 16 | def copy_from_local(localpath, remotepath, is_dir=False): 17 | if is_dir: 18 | if not os.path.exists(remotepath): 19 | # remote folder does not exist 20 | shutil.copytree(localpath, remotepath) 21 | else: 22 | # remote folder exists, copy each file 23 | src_files = os.listdir(localpath) 24 | for file_name in src_files: 25 | full_file_name = os.path.join(localpath, file_name) 26 | if os.path.isdir(full_file_name): 27 | # copy folder recursively 28 | full_remote_path = os.path.join(remotepath, file_name) 29 | shutil.copytree(full_file_name, full_remote_path) 30 | else: 31 | # copy each file 32 | shutil.copy(full_file_name, remotepath) 33 | else: 34 | shutil.copy(localpath, remotepath) 35 | 36 | 37 | class ManagementHandler(BaseHandler): 38 | def initialize(self, app): 39 | super(ManagementHandler, self).initialize(app) 40 | self.port = self.settings[SettingsParameters.Port] 41 | 42 | def _get_protocol(self): 43 | return "http://" 44 | 45 | @gen.coroutine 46 | def _add_or_update_endpoint(self, action, name, version, request_data): 47 | """ 48 | Add or update an endpoint 49 | """ 50 | self.logger.log(logging.DEBUG, f"Adding/updating model {name}...") 51 | 52 | if not isinstance(name, str): 53 | msg = "Endpoint name must be a string" 54 | self.logger.log(logging.CRITICAL, msg) 55 | raise TypeError(msg) 56 | 57 | name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") 58 | if not name_checker.match(name): 59 | raise gen.Return( 60 | "endpoint name can only contain: a-z, A-Z, 0-9," 61 | " underscore, hyphens and spaces." 62 | ) 63 | 64 | if self.settings.get("add_or_updating_endpoint"): 65 | msg = ( 66 | "Another endpoint update is already in progress" 67 | ", please wait a while and try again" 68 | ) 69 | self.logger.log(logging.CRITICAL, msg) 70 | raise RuntimeError(msg) 71 | 72 | self.settings["add_or_updating_endpoint"] = random_uuid() 73 | try: 74 | docstring = None 75 | if "docstring" in request_data: 76 | docstring = str( 77 | bytes(request_data["docstring"], "utf-8").decode("unicode_escape") 78 | ) 79 | 80 | description = request_data.get("description", None) 81 | endpoint_type = request_data.get("type", None) 82 | methods = request_data.get("methods", []) 83 | dependencies = request_data.get("dependencies", None) 84 | target = request_data.get("target", None) 85 | schema = request_data.get("schema", None) 86 | src_path = request_data.get("src_path", None) 87 | is_public = request_data.get("is_public", None) 88 | target_path = get_query_object_path( 89 | self.settings[SettingsParameters.StateFilePath], name, version 90 | ) 91 | 92 | path_checker = _compile(r"^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$") 93 | # copy from staging 94 | if src_path: 95 | if not isinstance(src_path, str): 96 | raise gen.Return("src_path must be a string.") 97 | if not path_checker.match(src_path): 98 | raise gen.Return(f"Invalid source path for endpoint {name}") 99 | 100 | yield self._copy_po_future(src_path, target_path) 101 | elif endpoint_type != "alias": 102 | raise gen.Return("src_path is required to add/update an endpoint.") 103 | else: 104 | # alias special logic: 105 | if not target: 106 | raise gen.Return("Target is required for alias endpoint.") 107 | dependencies = [target] 108 | 109 | # update local config 110 | try: 111 | if action == "add": 112 | self.tabpy_state.add_endpoint( 113 | name=name, 114 | description=description, 115 | docstring=docstring, 116 | endpoint_type=endpoint_type, 117 | methods=methods, 118 | dependencies=dependencies, 119 | target=target, 120 | schema=schema, 121 | is_public=is_public, 122 | ) 123 | else: 124 | self.tabpy_state.update_endpoint( 125 | name=name, 126 | description=description, 127 | docstring=docstring, 128 | endpoint_type=endpoint_type, 129 | methods=methods, 130 | dependencies=dependencies, 131 | target=target, 132 | schema=schema, 133 | version=version, 134 | is_public=is_public, 135 | ) 136 | 137 | except Exception as e: 138 | raise gen.Return(f"Error when changing TabPy state: {e}") 139 | 140 | on_state_change( 141 | self.settings, self.tabpy_state, self.python_service, self.logger 142 | ) 143 | 144 | finally: 145 | self.settings["add_or_updating_endpoint"] = None 146 | 147 | @gen.coroutine 148 | def _copy_po_future(self, src_path, target_path): 149 | future = STAGING_THREAD.submit( 150 | copy_from_local, src_path, target_path, is_dir=True 151 | ) 152 | ret = yield future 153 | raise gen.Return(ret) 154 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/no_op_auth_handler.py: -------------------------------------------------------------------------------- 1 | from pyarrow.flight import ServerAuthHandler 2 | 3 | class NoOpAuthHandler(ServerAuthHandler): 4 | def authenticate(self, outgoing, incoming): 5 | pass 6 | 7 | def is_valid(self, token): 8 | return "" 9 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/service_info_handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | from tabpy.tabpy_server.app.app_parameters import SettingsParameters 3 | from tabpy.tabpy_server.handlers import ManagementHandler 4 | from tabpy.tabpy_server.handlers.util import AuthErrorStates 5 | 6 | class ServiceInfoHandler(ManagementHandler): 7 | def initialize(self, app): 8 | super(ServiceInfoHandler, self).initialize(app) 9 | 10 | def get(self): 11 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 12 | self.fail_with_auth_error() 13 | return 14 | 15 | self._add_CORS_header() 16 | info = {} 17 | info["description"] = self.tabpy_state.get_description() 18 | info["creation_time"] = self.tabpy_state.creation_time 19 | info["state_path"] = self.settings[SettingsParameters.StateFilePath] 20 | info["server_version"] = self.settings[SettingsParameters.ServerVersion] 21 | info["name"] = self.tabpy_state.name 22 | info["versions"] = self.settings[SettingsParameters.ApiVersions] 23 | self.write(json.dumps(info)) 24 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/status_handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from tabpy.tabpy_server.handlers import BaseHandler 4 | from tabpy.tabpy_server.handlers.util import AuthErrorStates 5 | 6 | class StatusHandler(BaseHandler): 7 | def initialize(self, app): 8 | super(StatusHandler, self).initialize(app) 9 | 10 | def get(self): 11 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 12 | self.fail_with_auth_error() 13 | return 14 | 15 | self._add_CORS_header() 16 | 17 | status_dict = {} 18 | for k, v in self.python_service.ps.query_objects.items(): 19 | status_dict[k] = { 20 | "version": v["version"], 21 | "type": v["type"], 22 | "status": v["status"], 23 | "last_error": v["last_error"], 24 | } 25 | 26 | self.logger.log(logging.DEBUG, f"Found models: {status_dict}") 27 | self.write(json.dumps(status_dict)) 28 | self.finish() 29 | return 30 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/upload_destination_handler.py: -------------------------------------------------------------------------------- 1 | from tabpy.tabpy_server.app.app_parameters import SettingsParameters 2 | from tabpy.tabpy_server.handlers import ManagementHandler 3 | import os 4 | from tabpy.tabpy_server.handlers.util import AuthErrorStates 5 | 6 | 7 | _QUERY_OBJECT_STAGING_FOLDER = "staging" 8 | 9 | 10 | class UploadDestinationHandler(ManagementHandler): 11 | def initialize(self, app): 12 | super(UploadDestinationHandler, self).initialize(app) 13 | 14 | def get(self): 15 | if self.should_fail_with_auth_error() != AuthErrorStates.NONE: 16 | self.fail_with_auth_error() 17 | return 18 | 19 | path = self.settings[SettingsParameters.StateFilePath] 20 | path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) 21 | self.write({"path": path}) 22 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/handlers/util.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | from hashlib import pbkdf2_hmac 3 | from enum import Enum, auto 4 | 5 | 6 | class AuthErrorStates(Enum): 7 | NONE = auto() 8 | NotAuthorized = auto() 9 | NotRequired = auto() 10 | 11 | def hash_password(username, pwd): 12 | """ 13 | Hashes password using PKDBF2 method: 14 | hash = PKDBF2('sha512', pwd, salt=username, 10000) 15 | 16 | Parameters 17 | ---------- 18 | username : str 19 | User name (login). Used as salt for hashing. 20 | User name is lowercased befor being used in hashing. 21 | Salt is formatted as '_$salt@tabpy:$_' to 22 | guarantee there's at least 16 characters. 23 | 24 | pwd : str 25 | Password to hash. 26 | 27 | Returns 28 | ------- 29 | str 30 | Sting representation (hexidecimal) for PBKDF2 hash 31 | for the password. 32 | """ 33 | salt = f"_$salt@tabpy:{username.lower()}$_" 34 | 35 | hash = pbkdf2_hmac( 36 | hash_name="sha512", password=pwd.encode(), salt=salt.encode(), iterations=10000 37 | ) 38 | return binascii.hexlify(hash).decode() 39 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/management/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_server/management/util.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | try: 5 | from ConfigParser import ConfigParser as _ConfigParser 6 | except ImportError: 7 | from configparser import ConfigParser as _ConfigParser 8 | from tabpy.tabpy_server.app.app_parameters import ConfigParameters, SettingsParameters 9 | 10 | 11 | def write_state_config(state, settings, logger=logging.getLogger(__name__)): 12 | if SettingsParameters.StateFilePath in settings: 13 | state_path = settings[SettingsParameters.StateFilePath] 14 | else: 15 | msg = f"{ConfigParameters.TABPY_STATE_PATH} is not set" 16 | logger.log(logging.CRITICAL, msg) 17 | raise ValueError(msg) 18 | 19 | logger.log(logging.DEBUG, f"State path is {state_path}") 20 | state_key = os.path.join(state_path, "state.ini") 21 | tmp_state_file = state_key 22 | 23 | with open(tmp_state_file, "w") as f: 24 | state.write(f) 25 | 26 | 27 | def _get_state_from_file(state_path, logger=logging.getLogger(__name__)): 28 | state_key = os.path.join(state_path, "state.ini") 29 | tmp_state_file = state_key 30 | 31 | if not os.path.exists(tmp_state_file): 32 | msg = f"Missing config file at {tmp_state_file}" 33 | logger.log(logging.CRITICAL, msg) 34 | raise ValueError(msg) 35 | 36 | config = _ConfigParser(allow_no_value=True) 37 | config.optionxform = str 38 | config.read(tmp_state_file) 39 | 40 | if not config.has_section("Service Info"): 41 | msg = "Config error: Expected [Service Info] section in " f"{tmp_state_file}" 42 | logger.log(logging.CRITICAL, msg) 43 | raise ValueError(msg) 44 | 45 | return config 46 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/psws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/psws/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_server/psws/callbacks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tabpy.tabpy_server.app.app_parameters import SettingsParameters 3 | from tabpy.tabpy_server.common.messages import ( 4 | LoadObject, 5 | DeleteObjects, 6 | ListObjects, 7 | ObjectList, 8 | ) 9 | from tabpy.tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files 10 | from tabpy.tabpy_server.common.util import format_exception 11 | from tabpy.tabpy_server.management.state import TabPyState, get_query_object_path 12 | from tabpy.tabpy_server.management import util 13 | from time import sleep 14 | from tornado import gen 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def wait_for_endpoint_loaded(python_service, object_uri): 21 | """ 22 | This method waits for the object to be loaded. 23 | """ 24 | logger.info("Waiting for object to be loaded...") 25 | while True: 26 | msg = ListObjects() 27 | list_object_msg = python_service.manage_request(msg) 28 | if not isinstance(list_object_msg, ObjectList): 29 | logger.error(f"Error loading endpoint {object_uri}: {list_object_msg}") 30 | return 31 | 32 | for (uri, info) in list_object_msg.objects.items(): 33 | if uri == object_uri: 34 | if info["status"] != "LoadInProgress": 35 | logger.info(f'Object load status: {info["status"]}') 36 | return 37 | 38 | sleep(0.1) 39 | 40 | 41 | @gen.coroutine 42 | def init_ps_server(settings, tabpy_state): 43 | logger.info("Initializing TabPy Server...") 44 | existing_pos = tabpy_state.get_endpoints() 45 | for (object_name, obj_info) in existing_pos.items(): 46 | try: 47 | object_version = obj_info["version"] 48 | get_query_object_path( 49 | settings[SettingsParameters.StateFilePath], object_name, object_version 50 | ) 51 | except Exception as e: 52 | logger.error( 53 | f"Exception encounted when downloading object: {object_name}" 54 | f", error: {e}" 55 | ) 56 | 57 | 58 | @gen.coroutine 59 | def init_model_evaluator(settings, tabpy_state, python_service): 60 | """ 61 | This will go through all models that the service currently have and 62 | initialize them. 63 | """ 64 | logger.info("Initializing models...") 65 | 66 | existing_pos = tabpy_state.get_endpoints() 67 | 68 | for (object_name, obj_info) in existing_pos.items(): 69 | object_version = obj_info["version"] 70 | object_type = obj_info["type"] 71 | object_path = get_query_object_path( 72 | settings[SettingsParameters.StateFilePath], object_name, object_version 73 | ) 74 | 75 | logger.info( 76 | f"Load endpoint: {object_name}, " 77 | f"version: {object_version}, " 78 | f"type: {object_type}" 79 | ) 80 | if object_type == "alias": 81 | msg = LoadObject( 82 | object_name, obj_info["target"], object_version, False, "alias" 83 | ) 84 | else: 85 | local_path = object_path 86 | msg = LoadObject( 87 | object_name, local_path, object_version, False, object_type 88 | ) 89 | python_service.manage_request(msg) 90 | 91 | 92 | def _get_latest_service_state(settings, tabpy_state, new_ps_state, python_service): 93 | """ 94 | Update the endpoints from the latest remote state file. 95 | 96 | Returns 97 | -------- 98 | (has_changes, endpoint_diff): 99 | has_changes: True or False 100 | endpoint_diff: Summary of what has changed, one entry for each changes 101 | """ 102 | # Shortcut when nothing is changed 103 | changes = {"endpoints": {}} 104 | 105 | # update endpoints 106 | new_endpoints = new_ps_state.get_endpoints() 107 | diff = {} 108 | current_endpoints = python_service.ps.query_objects 109 | for (endpoint_name, endpoint_info) in new_endpoints.items(): 110 | existing_endpoint = current_endpoints.get(endpoint_name) 111 | if (existing_endpoint is None) or endpoint_info["version"] != existing_endpoint[ 112 | "version" 113 | ]: 114 | # Either a new endpoint or new endpoint version 115 | path_to_new_version = get_query_object_path( 116 | settings[SettingsParameters.StateFilePath], 117 | endpoint_name, 118 | endpoint_info["version"], 119 | ) 120 | endpoint_type = endpoint_info.get("type", "model") 121 | diff[endpoint_name] = ( 122 | endpoint_type, 123 | endpoint_info["version"], 124 | path_to_new_version, 125 | ) 126 | 127 | # add removed models too 128 | for (endpoint_name, endpoint_info) in current_endpoints.items(): 129 | if endpoint_name not in new_endpoints.keys(): 130 | endpoint_type = current_endpoints[endpoint_name].get("type", "model") 131 | diff[endpoint_name] = (endpoint_type, None, None) 132 | 133 | if diff: 134 | changes["endpoints"] = diff 135 | 136 | return (True, changes) 137 | 138 | 139 | @gen.coroutine 140 | def on_state_change( 141 | settings, tabpy_state, python_service, logger=logging.getLogger(__name__) 142 | ): 143 | try: 144 | logger.log(logging.INFO, "Loading state from state file") 145 | config = util._get_state_from_file( 146 | settings[SettingsParameters.StateFilePath], logger=logger 147 | ) 148 | new_ps_state = TabPyState(config=config, settings=settings) 149 | 150 | (has_changes, changes) = _get_latest_service_state( 151 | settings, tabpy_state, new_ps_state, python_service 152 | ) 153 | if not has_changes: 154 | logger.info("Nothing changed, return.") 155 | return 156 | 157 | new_endpoints = new_ps_state.get_endpoints() 158 | for object_name in changes["endpoints"]: 159 | (object_type, object_version, object_path) = changes["endpoints"][ 160 | object_name 161 | ] 162 | 163 | if not object_path and not object_version: # removal 164 | logger.info(f"Removing object: URI={object_name}") 165 | 166 | python_service.manage_request(DeleteObjects([object_name])) 167 | 168 | cleanup_endpoint_files( 169 | object_name, settings[SettingsParameters.UploadDir], logger=logger 170 | ) 171 | 172 | else: 173 | endpoint_info = new_endpoints[object_name] 174 | is_update = object_version > 1 175 | if object_type == "alias": 176 | msg = LoadObject( 177 | object_name, 178 | endpoint_info["target"], 179 | object_version, 180 | is_update, 181 | "alias", 182 | ) 183 | else: 184 | local_path = object_path 185 | msg = LoadObject( 186 | object_name, local_path, object_version, is_update, object_type 187 | ) 188 | 189 | python_service.manage_request(msg) 190 | wait_for_endpoint_loaded(python_service, object_name) 191 | 192 | # cleanup old version of endpoint files 193 | if object_version > 2: 194 | cleanup_endpoint_files( 195 | object_name, 196 | settings[SettingsParameters.UploadDir], 197 | logger=logger, 198 | retain_versions=[object_version, object_version - 1], 199 | ) 200 | 201 | except Exception as e: 202 | err_msg = format_exception(e, "on_state_change") 203 | logger.log( 204 | logging.ERROR, f"Error submitting update model request: error={err_msg}" 205 | ) 206 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/state.ini.template: -------------------------------------------------------------------------------- 1 | [Service Info] 2 | Name = TabPy Server 3 | Description = 4 | Creation Time = 0 5 | Access-Control-Allow-Origin = 6 | Access-Control-Allow-Headers = 7 | Access-Control-Allow-Methods = 8 | 9 | [Query Objects Service Versions] 10 | 11 | [Query Objects Docstrings] 12 | 13 | [Meta] 14 | Revision Number = 1 15 | 16 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/static/TabPy_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/static/TabPy_logo.png -------------------------------------------------------------------------------- /tabpy/tabpy_server/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Tableau Python Server 10 | 11 | 22 | 23 | 54 | 55 | 56 | 57 |

TabPy Server Info:

58 |

59 | 60 |

Deployed Models:

61 |

62 | 63 |

Useful links:

64 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /tabpy/tabpy_server/static/tableau.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_server/static/tableau.png -------------------------------------------------------------------------------- /tabpy/tabpy_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/tabpy_tools/__init__.py -------------------------------------------------------------------------------- /tabpy/tabpy_tools/custom_query_object.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import platform 3 | import sys 4 | from .query_object import QueryObject as _QueryObject 5 | 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class CustomQueryObject(_QueryObject): 11 | def __init__(self, query, description=""): 12 | """Create a new CustomQueryObject. 13 | 14 | Parameters 15 | ----------- 16 | 17 | query : function 18 | Function that defines a custom query method. The query can have any 19 | signature, but input and output of the query needs to be JSON 20 | serializable. 21 | 22 | description : str 23 | The description of the custom query object 24 | 25 | """ 26 | super().__init__(description) 27 | 28 | self.custom_query = query 29 | 30 | def query(self, *args, **kwargs): 31 | """Query the custom defined query method using the given input. 32 | 33 | Parameters 34 | ---------- 35 | args : list 36 | positional arguments to the query 37 | 38 | kwargs : dict 39 | keyword arguments to the query 40 | 41 | Returns 42 | ------- 43 | out: object. 44 | The results depends on the implementation of the query method. 45 | Typically the return value will be whatever that function returns. 46 | 47 | See Also 48 | -------- 49 | QueryObject 50 | """ 51 | # include the dependent files in sys path so that the query can run 52 | # correctly 53 | 54 | try: 55 | logger.debug( 56 | "Running custom query with arguments " f"({args}, {kwargs})..." 57 | ) 58 | ret = self.custom_query(*args, **kwargs) 59 | except Exception as e: 60 | logger.exception( 61 | "Exception hit when running custom query, error: " f"{str(e)}" 62 | ) 63 | raise 64 | 65 | logger.debug(f"Received response {ret}") 66 | try: 67 | return self._make_serializable(ret) 68 | except Exception as e: 69 | logger.exception( 70 | "Cannot properly serialize custom query result, " f"error: {str(e)}" 71 | ) 72 | raise 73 | 74 | def get_docstring(self): 75 | """Get doc string from customized query""" 76 | default_docstring = "-- no docstring found in query function --" 77 | 78 | # TODO: fix docstring parsing on Windows systems 79 | if sys.platform == 'win32': 80 | return default_docstring 81 | 82 | ds = getattr(self.custom_query, '__doc__', None) 83 | return ds if ds and isinstance(ds, str) else default_docstring 84 | 85 | def get_methods(self): 86 | return [self.get_query_method()] 87 | 88 | def get_query_method(self): 89 | return {"method": "query"} 90 | -------------------------------------------------------------------------------- /tabpy/tabpy_tools/query_object.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | import os 4 | import json 5 | import shutil 6 | 7 | import cloudpickle as _cloudpickle 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class QueryObject(abc.ABC): 14 | """ 15 | Derived class needs to implement the following interface: 16 | * query() -- given input, return query result 17 | * get_docstring() -- returns documentation for the Query Object 18 | """ 19 | 20 | def __init__(self, description=""): 21 | self.description = description 22 | 23 | def get_dependencies(self): 24 | """All endpoints this endpoint depends on""" 25 | return [] 26 | 27 | @abc.abstractmethod 28 | def query(self, input): 29 | """execute query on the provided input""" 30 | pass 31 | 32 | @abc.abstractmethod 33 | def get_docstring(self): 34 | """Returns documentation for the query object 35 | 36 | By default, this method returns the docstring for 'query' method 37 | Derived class may overwrite this method to dynamically create docstring 38 | """ 39 | pass 40 | 41 | def save(self, path): 42 | """ Save query object to the given local path 43 | 44 | Parameters 45 | ---------- 46 | path : str 47 | The location to save the query object to 48 | """ 49 | if os.path.exists(path): 50 | logger.warning( 51 | f'Overwriting existing file "{path}" when saving query object' 52 | ) 53 | rm_fn = os.remove if os.path.isfile(path) else shutil.rmtree 54 | rm_fn(path) 55 | self._save_local(path) 56 | 57 | def _save_local(self, path): 58 | """Save current query object to local path 59 | """ 60 | try: 61 | os.makedirs(path) 62 | except OSError as e: 63 | import errno 64 | 65 | if e.errno == errno.EEXIST and os.path.isdir(path): 66 | pass 67 | else: 68 | raise 69 | 70 | with open(os.path.join(path, "pickle_archive"), "wb") as f: 71 | _cloudpickle.dump(self, f) 72 | 73 | @classmethod 74 | def load(cls, path): 75 | """ Load query object from given path 76 | """ 77 | new_po = None 78 | new_po = cls._load_local(path) 79 | 80 | logger.info(f'Loaded query object "{type(new_po).__name__}" successfully') 81 | 82 | return new_po 83 | 84 | @classmethod 85 | def _load_local(cls, path): 86 | path = os.path.abspath(os.path.expanduser(path)) 87 | with open(os.path.join(path, "pickle_archive"), "rb") as f: 88 | return _cloudpickle.load(f) 89 | 90 | @classmethod 91 | def _make_serializable(cls, result): 92 | """Convert a result from object query to python data structure that can 93 | easily serialize over network 94 | """ 95 | try: 96 | json.dumps(result) 97 | except TypeError: 98 | raise TypeError( 99 | "Result from object query is not json serializable: " f"{result}" 100 | ) 101 | 102 | return result 103 | 104 | # Returns an array of dictionary that contains the methods and their 105 | # corresponding schema information. 106 | @abc.abstractmethod 107 | def get_methods(self): 108 | return None 109 | -------------------------------------------------------------------------------- /tabpy/tabpy_tools/schema.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import genson 3 | import jsonschema 4 | 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def _generate_schema_from_example_and_description(input, description): 10 | """ 11 | With an example input, a schema is automatically generated that conforms 12 | to the example in json-schema.org. The description given by the users 13 | is then added to the schema. 14 | """ 15 | s = genson.SchemaBuilder(None) 16 | s.add_object(input) 17 | input_schema = s.to_schema() 18 | 19 | if description is not None: 20 | if "properties" in input_schema: 21 | # Case for input = {'x':1}, input_description='not a dict' 22 | if not isinstance(description, dict): 23 | msg = f"{input} and {description} do not match" 24 | logger.error(msg) 25 | raise Exception(msg) 26 | 27 | for key in description: 28 | # Case for input = {'x':1}, 29 | # input_description={'x':'x value', 'y':'y value'} 30 | if key not in input_schema["properties"]: 31 | msg = f"{key} not found in {input}" 32 | logger.error(msg) 33 | raise Exception(msg) 34 | else: 35 | input_schema["properties"][key]["description"] = description[key] 36 | else: 37 | if isinstance(description, dict): 38 | raise Exception(f"{input} and {description} do not match") 39 | else: 40 | input_schema["description"] = description 41 | 42 | try: 43 | # This should not fail unless there are bugs with either genson or 44 | # jsonschema. 45 | jsonschema.validate(input, input_schema) 46 | except Exception as e: 47 | logger.error(f"Internal error validating schema: {str(e)}") 48 | raise 49 | 50 | return input_schema 51 | 52 | 53 | def generate_schema(input, output, input_description=None, output_description=None): 54 | """ 55 | Generate schema from a given sample input and output. 56 | A generated schema can be passed to a server together with a function to 57 | annotate it with information about input and output parameters, and 58 | examples thereof. The schema needs to follow the conventions of JSON Schema 59 | (see json-schema.org). 60 | 61 | Parameters 62 | ----------- 63 | input : any python type | dict 64 | output: any python type | dict 65 | input_description : str | dict, optional 66 | output_description : str | dict, optional 67 | 68 | References 69 | ----------- 70 | - `Json Schema ` 71 | 72 | Examples 73 | ---------- 74 | .. sourcecode:: python 75 | For just one input parameter, state the example directly. 76 | >>> from tabpy.tabpy_tools.schema import generate_schema 77 | >>> schema = generate_schema( 78 | input=5, 79 | output=25, 80 | input_description='input value', 81 | output_description='the squared value of input') 82 | >>> schema 83 | {'sample': 5, 84 | 'input': {'type': 'integer', 'description': 'input value'}, 85 | 'output': {'type': 'integer', 'description': 'the squared value of input'}} 86 | For two or more input parameters, specify them using a dictionary. 87 | >>> import graphlab 88 | >>> schema = generate_schema( 89 | input={'x': 3, 'y': 2}, 90 | output=6, 91 | input_description={'x': 'value of x', 92 | 'y': 'value of y'}, 93 | output_description='x times y') 94 | >>> schema 95 | {'sample': {'y': 2, 'x': 3}, 96 | 'input': {'required': ['x', 'y'], 97 | 'type': 'object', 98 | 'properties': {'y': {'type': 'integer', 'description': 'value of y'}, 99 | 'x': {'type': 'integer', 'description': 'value of x'}}}, 100 | 'output': {'type': 'integer', 'description': 'x times y'}} 101 | """ # noqa: E501 102 | input_schema = _generate_schema_from_example_and_description( 103 | input, input_description 104 | ) 105 | output_schema = _generate_schema_from_example_and_description( 106 | output, output_description 107 | ) 108 | return {"input": input_schema, "sample": input, "output": output_schema} 109 | -------------------------------------------------------------------------------- /tabpy/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tabpy/utils/__init__.py -------------------------------------------------------------------------------- /tabpy/utils/tabpy_user.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility for managing user names and passwords for TabPy. 3 | For more information about how to configure and use authentication for 4 | TabPy read the documentation at https://github.com/tableau/TabPy. 5 | 6 | Usage: 7 | tabpy-user add (-u NAME | --user ) [-p PWD | --password PWD] (-f FILE | --pwdfile FILE) 8 | tabpy-user update (-u NAME | --user ) [-p PWD | --password PWD] (-f FILE | --pwdfile FILE) 9 | tabpy-user -h | --help 10 | 11 | Options: 12 | -h --help Show this screen. 13 | -u NAME --username NAME Username to add to password file. 14 | -p PWD --password PWD Password for the username. If not specified a 15 | password will be generated. 16 | -f FILE --pwdfile FILE Fully qualified path to passwords file. 17 | """ 18 | 19 | import docopt 20 | import logging 21 | import secrets 22 | from tabpy.tabpy_server.app.util import parse_pwd_file 23 | from tabpy.tabpy_server.handlers.util import hash_password 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def generate_password(pwd_len=16): 29 | # List of characters to generate password from. 30 | # We want to avoid to use similarly looking pairs like 31 | # (O, 0), (1, l), etc. 32 | lower_case_letters = "abcdefghijkmnpqrstuvwxyz" 33 | upper_case_letters = "ABCDEFGHIJKLMPQRSTUVWXYZ" 34 | digits = "23456789" 35 | 36 | # and for punctuation we want to exclude some characters 37 | # like inverted comma which can be hard to find and/or 38 | # type 39 | # change this string if you are supporting an 40 | # international keyboard with differing keys available 41 | punctuation = "!#$%&()*+,-./:;<=>?@[\\]^_{|}~" 42 | 43 | # we also want to try to have more letters and digits in 44 | # generated password than punctuation marks 45 | password_chars = ( 46 | lower_case_letters 47 | + lower_case_letters 48 | + upper_case_letters 49 | + upper_case_letters 50 | + digits 51 | + digits 52 | + punctuation 53 | ) 54 | pwd = "".join(secrets.choice(password_chars) for i in range(pwd_len)) 55 | logger.info(f'Generated password: "{pwd}"') 56 | return pwd 57 | 58 | 59 | def store_passwords_file(pwdfile, credentials): 60 | with open(pwdfile, "wt") as f: 61 | for username, pwd in credentials.items(): 62 | f.write(f"{username} {pwd}\n") 63 | return True 64 | 65 | 66 | def add_user(args, credentials): 67 | username = args["--username"].lower() 68 | logger.info(f'Adding username "{username}"') 69 | 70 | if username in credentials: 71 | logger.error( 72 | f"Can't add username {username} as it is already present" 73 | " in passwords file. Do you want to run the " 74 | '"update" command instead?' 75 | ) 76 | return False 77 | 78 | password = args["--password"] 79 | logger.info(f'Adding username "{username}" with password "{password}"...') 80 | credentials[username] = hash_password(username, password) 81 | 82 | if store_passwords_file(args["--pwdfile"], credentials): 83 | logger.info(f'Added username "{username}" with password "{password}"') 84 | else: 85 | logger.info( 86 | f'Could not add username "{username}" , ' f'password "{password}" to file' 87 | ) 88 | 89 | 90 | def update_user(args, credentials): 91 | username = args["--username"].lower() 92 | logger.info(f'Updating username "{username}"') 93 | 94 | if username not in credentials: 95 | logger.error( 96 | f'Username "{username}" not found in passwords file. ' 97 | 'Do you want to run "add" command instead?' 98 | ) 99 | return False 100 | 101 | password = args["--password"] 102 | logger.info(f'Updating username "{username}" password to "{password}"') 103 | credentials[username] = hash_password(username, password) 104 | return store_passwords_file(args["--pwdfile"], credentials) 105 | 106 | 107 | def process_command(args, credentials): 108 | if args["add"]: 109 | return add_user(args, credentials) 110 | elif args["update"]: 111 | return update_user(args, credentials) 112 | else: 113 | logger.error(f'Unknown command "{args.command}"') 114 | return False 115 | 116 | 117 | def main(): 118 | logging.basicConfig(level=logging.DEBUG, format="%(message)s") 119 | 120 | args = docopt.docopt(__doc__) 121 | 122 | succeeded, credentials = parse_pwd_file(args["--pwdfile"]) 123 | if not succeeded and not args["add"]: 124 | return 125 | 126 | if args["--password"] is None: 127 | args["--password"] = generate_password() 128 | 129 | process_command(args, credentials) 130 | return 131 | 132 | 133 | if __name__ == "__main__": 134 | main() 135 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/resources/2019_04_24_to_3018_08_25.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDyjCCArICCQD3+Tk53RbuzTANBgkqhkiG9w0BAQsFADCBpTELMAkGA1UEBhMC 3 | VVMxCzAJBgNVBAgMAldBMREwDwYDVQQHDAhLaXJrbGFuZDEZMBcGA1UECgwQVGFi 4 | bGVhdSBTb2Z0d2FyZTEbMBkGA1UECwwSQWR2YW5jZWQgQW5hbHl0aWNzMRcwFQYD 5 | VQQDDA5PbGVrIEdvbG92YXR5aTElMCMGCSqGSIb3DQEJARYWb2dvbG92YXR5aUB0 6 | YWJsZWF1LmNvbTAgFw0xOTA0MjQyMTU1NDVaGA8zMDE4MDgyNTIxNTU0NVowgaUx 7 | CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJXQTERMA8GA1UEBwwIS2lya2xhbmQxGTAX 8 | BgNVBAoMEFRhYmxlYXUgU29mdHdhcmUxGzAZBgNVBAsMEkFkdmFuY2VkIEFuYWx5 9 | dGljczEXMBUGA1UEAwwOT2xlayBHb2xvdmF0eWkxJTAjBgkqhkiG9w0BCQEWFm9n 10 | b2xvdmF0eWlAdGFibGVhdS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 11 | AoIBAQCnoOgC0w3mSS2uoRQOcKtkC3ueHxo8hDXsdnBaCdcvo8ixqvYiKP/twZCb 12 | sz+5YGFGkCwGWrdX9U9Iy/70r1fLyoZ89oswjf4ei3FczFfjTB1l4pgnDBYKWgQm 13 | IdkZ3n26YmNWm/4e3cm61KYY8fJN0v9Ql5NBxH+xRrvwqgkFRZJcIuAEa7k28FD/ 14 | KaMLOgDMxtuFXcoQSwT75ggmhM89aeE4kKf+MbG7dkwoV3y1hZG/gW6BryLfo2xA 15 | YlaQwtzPBPhgE8gsqxtO7l+wxv03JOnkPQNBWHAf7MtlkqdM6g03UrWmfTFhqzPE 16 | rzsiWOXDxD2c5HSiss24HHrgF37rAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAIQb 17 | TwPcrDeYUwIz8TWEEcIX3jJoDMyR6Q+AEmM8C8ed0JlavE2qPiZc+lLr8Dc1B+fE 18 | 7UkxHPZOw0fCtrQ3I3+0Z/6YfqZs3m1f1I8Yr6SSW6NjAj7+mmMQ8DuJGb5yuefP 19 | Z0LV8F+OwUXNI7bsl0Q4UKt8QQ0ovI3I6w8HVsuy7zyEUN268tiK58bMkSfbzVal 20 | UvIcXqZyBFKQ/ZZ3BknI8b3ibya7h7R/92CsMDfPAQASZcBwKJ64RW9Wi5Gzzqmp 21 | D2Vk6MdyOgp4bD9wDqm4f6p20FewagSL5/c3lk1EjoCye6UAH2cnqPRTowI2elJg 22 | W9mrYH2k9L2cnnUIyx4= 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/integration/resources/2019_04_24_to_3018_08_25.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAp6DoAtMN5kktrqEUDnCrZAt7nh8aPIQ17HZwWgnXL6PIsar2 3 | Iij/7cGQm7M/uWBhRpAsBlq3V/VPSMv+9K9Xy8qGfPaLMI3+HotxXMxX40wdZeKY 4 | JwwWCloEJiHZGd59umJjVpv+Ht3JutSmGPHyTdL/UJeTQcR/sUa78KoJBUWSXCLg 5 | BGu5NvBQ/ymjCzoAzMbbhV3KEEsE++YIJoTPPWnhOJCn/jGxu3ZMKFd8tYWRv4Fu 6 | ga8i36NsQGJWkMLczwT4YBPILKsbTu5fsMb9NyTp5D0DQVhwH+zLZZKnTOoNN1K1 7 | pn0xYaszxK87Iljlw8Q9nOR0orLNuBx64Bd+6wIDAQABAoIBACsVRAxVymDBtigH 8 | 5mu/sY1JFkCRpeCf6mwYFNBPbysjYVWopxIoj37AHTanX1151Aaaz3XiovTMa9A9 9 | /g1Nc7dBGkfL5gJYvFOFa2F6c6xLx9KD5q9Cf/exIxfZ4z6u3Imm9/kupqWwQ0Tt 10 | mrMWnDw8WrqP+p0Qr/EUSQGV8jOUP3SyA3zQbEbsBkettz4Nil9NITwbQrbnG6o7 11 | RnNuxCwRwO3QB1p0YOXnKytU8xIu78Xg3qwVx81QP3/omB9jNCY52p1FRLzGO21q 12 | JFZLIcP7hECesn5v9dZa99oChU+Rdzi93VEymwmZBVl8givNdWAe1Y7c/Z8fIWS/ 13 | FQVvRgECgYEA12uZRlLDnDoqYTj5Ots87Kpy15wU5lA2ASGNgaQLXZBgh8ID/8o5 14 | QRk1/CNW2i/cW/fI7XRaiFoHjrvD6XgT0m/bWOezFWXoemkMR/jR2kYZjS0vpc04 15 | wd/rvrHr4nbSQE1cVBwLrYzrYJ6qrGybx9P6k0fhu2fUIJIJGEOTQ0ECgYEAxzSa 16 | f4KLGSrZjfZl695z+83VUG5aeg8V407nA6RBw3XeK9BjHhqfsRnFAfvhtyTolm3M 17 | QOaZLhSnhnW5HSdYW0QEtW4Lb5GkiGdZSArjM5zD/MgHitlOm9r0IL+nBbtNQYrd 18 | i85pTeeIlG7CTFGtx3b9EiQYHYl2xeS3QbblcysCgYBvQVPk3OPHsMaodZtKSWY6 19 | uIEdV6/3jt+FUAXcOZPhG6qvEoWsOo29UD7wXHQDtYoyOVOdR2VmXFDg55pz3p8m 20 | JLz9OpTj7UDWz6AXH6uJ9oBFyFt+XvH8NyBy2UMBL+rAaPPRQLbLSCdcPDXbXTBL 21 | UPBt1kb/2czVkXZ/AI9ywQKBgQCGgsm0QhTk6J9AkdmenHZa2FEq32kutFMGSzgI 22 | qHhToJploW/cWwPr1UfHICr4vO5k7T0Xsd5LVF0OmR1nRzMNZW98hxMnwgOEq6yI 23 | zfk+16MrZHJbWoMPEJj6KA+C+kefc0JH7hgDJ8181RFT8W9TmdAm2MKD51eRJvBr 24 | ajGjQwKBgQCvNf8Ds4Smy/5ANyOjK3/iPZiGiVgyaKCJOKNHQ+pAD0JS8XfOh/Km 25 | KiXv8jBEQcChB7YoYKBUfXwpSLFruJU3kCLvN4MHQAgV0BfVx8MkcLJh6K+wMGPX 26 | Es5hj6r4RQQblJaj9q8qb3+9uG3k7Sn4TXc0TYg7ml32ugXSXMxfKg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /tests/integration/resources/data.csv: -------------------------------------------------------------------------------- 1 | Username,Identifier,First name,Last name 2 | booker12,9012,Rachel,Booker 3 | grey07,2070,Laura,Grey 4 | johnson81,4081,Craig,Johnson 5 | jenkins46,9346,Mary,Jenkins 6 | smith79,5079,Jamie,Smith 7 | mbell001,4294,Mike,Bell 8 | davies87,4387,Emma,Davies 9 | peterson39,7039,Julia,Peterson 10 | morris22,1922,Jack,Morris 11 | carter71,4071,Oliver,Carter 12 | wright13,7813,Holly,Wright 13 | murphy80,1080,Lisa,Murphy 14 | stewart64,9564,Kevin,Stewart 15 | brown42,6042,Daniel,Brown 16 | chavez28,2128,Carla,Chavez 17 | kim33,1033,Grace,Kim 18 | jones02,1002,Megan,Jones 19 | miller95,1295,Sean,Miller 20 | foster62,5062,Rachel,Foster 21 | perry85,9085,Lauren,Perry 22 | hill27,4727,Sam,Hill 23 | gonzalez06,2106,Jose,Gonzalez 24 | ward09,2509,Melanie,Ward 25 | thompson58,8058,Leah,Thompson 26 | scott61,3061,Tyler,Scott 27 | taylor12,1012,Connor,Taylor 28 | adams20,9020,Benjamin,Adams 29 | wilson01,9001,Samantha,Wilson 30 | mitchell05,3005,Taylor,Mitchell 31 | lee14,4014,Minji,Lee 32 | hernandez07,707,Julio,Hernandez 33 | wilkins63,4063,Brad,Wilkins 34 | watson75,7075,Rebecca,Watson 35 | holland47,1047,Adam,Holland 36 | simmons92,1092,Jasmine,Simmons 37 | ross39,8039,Stephanie,Ross 38 | mccarthy57,1257,Patrick,McCarthy 39 | edwards68,5068,Abby,Edwards 40 | blair43,5043,Shane,Blair 41 | nichols18,4018,Karen,Nichols 42 | campbell25,8025,Allison,Campbell 43 | sanchez11,5011,Maria,Sanchez 44 | smith07,3007,Victoria,Smith 45 | williams48,7048,Justin,Williams 46 | parker36,3036,Sarah,Parker 47 | cruz85,8085,Ruben,Cruz -------------------------------------------------------------------------------- /tests/integration/resources/deploy_and_evaluate_model.conf: -------------------------------------------------------------------------------- 1 | [TabPy] 2 | # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects 3 | TABPY_PORT = 9008 4 | # TABPY_STATE_PATH = ./tabpy/tabpy_server 5 | 6 | # Where static pages live 7 | # TABPY_STATIC_PATH = ./tabpy/tabpy_server/static 8 | 9 | # For how to configure TabPy authentication read 10 | # Authentication section in docs/server-config.md. 11 | # TABPY_PWD_FILE = /path/to/password/file.txt 12 | 13 | # To set up secure TabPy uncomment and modify the following lines. 14 | # Note only PEM-encoded x509 certificates are supported. 15 | # TABPY_TRANSFER_PROTOCOL = https 16 | # TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt 17 | # TABPY_KEY_FILE = path/to/key/file.key 18 | 19 | # Log additional request details including caller IP, full URL, client 20 | # end user info if provided. 21 | # TABPY_LOG_DETAILS = true 22 | 23 | # Configure how long a custom script provided to the /evaluate method 24 | # will run before throwing a TimeoutError. 25 | # The value should be a float representing the timeout time in seconds. 26 | #TABPY_EVALUATE_TIMEOUT = 30 27 | 28 | [loggers] 29 | keys=root 30 | 31 | [handlers] 32 | keys=rootHandler,rotatingFileHandler 33 | 34 | [formatters] 35 | keys=rootFormatter 36 | 37 | [logger_root] 38 | level=DEBUG 39 | handlers=rootHandler,rotatingFileHandler 40 | qualname=root 41 | propagete=0 42 | 43 | [handler_rootHandler] 44 | class=StreamHandler 45 | level=DEBUG 46 | formatter=rootFormatter 47 | args=(sys.stdout,) 48 | 49 | [handler_rotatingFileHandler] 50 | class=handlers.RotatingFileHandler 51 | level=DEBUG 52 | formatter=rootFormatter 53 | args=('tabpy_log.log', 'a', 1000000, 5) 54 | 55 | [formatter_rootFormatter] 56 | format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s 57 | datefmt=%Y-%m-%d,%H:%M:%S 58 | -------------------------------------------------------------------------------- /tests/integration/resources/deploy_and_evaluate_model_auth.conf: -------------------------------------------------------------------------------- 1 | [TabPy] 2 | # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects 3 | TABPY_PORT = 9009 4 | # TABPY_STATE_PATH = ./tabpy/tabpy_server 5 | 6 | # Where static pages live 7 | # TABPY_STATIC_PATH = ./tabpy/tabpy_server/static 8 | 9 | # For how to configure TabPy authentication read 10 | # Authentication section in docs/server-config.md. 11 | TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt 12 | 13 | # To set up secure TabPy uncomment and modify the following lines. 14 | # Note only PEM-encoded x509 certificates are supported. 15 | # TABPY_TRANSFER_PROTOCOL = https 16 | # TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt 17 | # TABPY_KEY_FILE = path/to/key/file.key 18 | 19 | # Log additional request details including caller IP, full URL, client 20 | # end user info if provided. 21 | # TABPY_LOG_DETAILS = true 22 | 23 | # Configure how long a custom script provided to the /evaluate method 24 | # will run before throwing a TimeoutError. 25 | # The value should be a float representing the timeout time in seconds. 26 | #TABPY_EVALUATE_TIMEOUT = 30 27 | 28 | [loggers] 29 | keys=root 30 | 31 | [handlers] 32 | keys=rootHandler,rotatingFileHandler 33 | 34 | [formatters] 35 | keys=rootFormatter 36 | 37 | [logger_root] 38 | level=DEBUG 39 | handlers=rootHandler,rotatingFileHandler 40 | qualname=root 41 | propagete=0 42 | 43 | [handler_rootHandler] 44 | class=StreamHandler 45 | level=DEBUG 46 | formatter=rootFormatter 47 | args=(sys.stdout,) 48 | 49 | [handler_rotatingFileHandler] 50 | class=handlers.RotatingFileHandler 51 | level=DEBUG 52 | formatter=rootFormatter 53 | args=('tabpy_log.log', 'a', 1000000, 5) 54 | 55 | [formatter_rootFormatter] 56 | format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s 57 | datefmt=%Y-%m-%d,%H:%M:%S 58 | -------------------------------------------------------------------------------- /tests/integration/resources/pwdfile.txt: -------------------------------------------------------------------------------- 1 | user1 b8a63cf588cd2399da615042de4732f5b121c4d1042acfb91598a56d2dd3e1d7b9f785213262eddbbd00c7a8c9c3e89d7cb98f31d405cc644b1aeba92c3de40b 2 | -------------------------------------------------------------------------------- /tests/integration/test_arrow_server.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import threading 3 | import _thread 4 | import pyarrow 5 | import os 6 | import pyarrow.csv as csv 7 | 8 | from tabpy.tabpy_server.app.arrow_server import FlightServer 9 | import tabpy.tabpy_server.app.arrow_server as pa 10 | 11 | class TestArrowServer(unittest.TestCase): 12 | @classmethod 13 | def setUpClass(cls): 14 | host = "localhost" 15 | port = 13620 16 | scheme = "grpc+tcp" 17 | location = "{}://{}:{}".format(scheme, host, port) 18 | cls.arrow_server = FlightServer(host, location) 19 | def start_server(): 20 | pa.start(cls.arrow_server) 21 | _thread.start_new_thread(start_server, ()) 22 | cls.arrow_client = pyarrow.flight.FlightClient(location) 23 | 24 | @classmethod 25 | def tearDownClass(cls): 26 | cls.arrow_server.shutdown() 27 | 28 | def setUp(self): 29 | self.resources_path = os.path.join(os.path.dirname(__file__), "resources") 30 | self.arrow_server.flights = {} 31 | 32 | def get_descriptor(self, data_path): 33 | return pyarrow.flight.FlightDescriptor.for_path(data_path) 34 | 35 | def write_data(self, data_path): 36 | table = csv.read_csv(data_path) 37 | descriptor = self.get_descriptor(data_path) 38 | writer, _ = self.arrow_client.do_put(descriptor, table.schema) 39 | writer.write_table(table) 40 | writer.close() 41 | return table 42 | 43 | def test_server_do_put(self): 44 | self.write_data(os.path.join(self.resources_path, "data.csv")) 45 | flight_info = list(self.arrow_server.list_flights(None, None)) 46 | self.assertEqual(len(flight_info), 1) 47 | 48 | def test_server_do_get(self): 49 | table = self.write_data(os.path.join(self.resources_path, "data.csv")) 50 | descriptor = self.get_descriptor(os.path.join(self.resources_path, "data.csv")) 51 | self.assertEqual(len(self.arrow_server.flights), 1) 52 | info = self.arrow_client.get_flight_info(descriptor) 53 | reader = self.arrow_client.do_get(info.endpoints[0].ticket) 54 | self.assertTrue(reader.read_all().equals(table)) 55 | self.assertEqual(len(self.arrow_server.flights), 0) 56 | 57 | def test_list_flights_on_new_server(self): 58 | flight_info = list(self.arrow_server.list_flights(None, None)) 59 | self.assertEqual(len(flight_info), 0) 60 | -------------------------------------------------------------------------------- /tests/integration/test_auth.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from . import integ_test_base 3 | 4 | 5 | class TestAuth(integ_test_base.IntegTestBase): 6 | def setUp(self): 7 | super(TestAuth, self).setUp() 8 | self.payload = """{ 9 | "data": { "_arg1": [1, 2] }, 10 | "script": "return [x * 2 for x in _arg1]" 11 | }""" 12 | 13 | def _get_pwd_file(self) -> str: 14 | return "./tests/integration/resources/pwdfile.txt" 15 | 16 | def test_missing_credentials_fails(self): 17 | headers = { 18 | "Content-Type": "application/json", 19 | "TabPy-Client": "Integration tests for Auth", 20 | } 21 | 22 | conn = self._get_connection() 23 | conn.request("POST", "/evaluate", self.payload, headers) 24 | res = conn.getresponse() 25 | 26 | self.assertEqual(401, res.status) 27 | 28 | def test_invalid_password(self): 29 | headers = { 30 | "Content-Type": "application/json", 31 | "TabPy-Client": "Integration tests for Auth", 32 | "Authorization": "Basic " 33 | + base64.b64encode("user1:wrong_password".encode("utf-8")).decode("utf-8"), 34 | } 35 | 36 | conn = self._get_connection() 37 | conn.request("POST", "/evaluate", self.payload, headers) 38 | res = conn.getresponse() 39 | 40 | self.assertEqual(401, res.status) 41 | 42 | def test_invalid_username(self): 43 | # Uncomment the following line to preserve 44 | # test case output and other files (config, state, ect.) 45 | # in system temp folder. 46 | # self.set_delete_temp_folder(False) 47 | 48 | headers = { 49 | "Content-Type": "application/json", 50 | "TabPy-Client": "Integration tests for Auth", 51 | "Authorization": "Basic " 52 | + base64.b64encode("wrong_user:P@ssw0rd".encode("utf-8")).decode("utf-8"), 53 | } 54 | 55 | conn = self._get_connection() 56 | conn.request("POST", "/evaluate", self.payload, headers) 57 | res = conn.getresponse() 58 | 59 | self.assertEqual(401, res.status) 60 | 61 | def test_valid_credentials(self): 62 | headers = { 63 | "Content-Type": "application/json", 64 | "TabPy-Client": "Integration tests for Auth", 65 | "Authorization": "Basic " 66 | + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 67 | } 68 | 69 | conn = self._get_connection() 70 | conn.request("POST", "/evaluate", self.payload, headers) 71 | res = conn.getresponse() 72 | 73 | self.assertEqual(200, res.status) 74 | -------------------------------------------------------------------------------- /tests/integration/test_custom_evaluate_timeout.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | 3 | 4 | class TestCustomEvaluateTimeout(integ_test_base.IntegTestBase): 5 | def _get_evaluate_timeout(self) -> str: 6 | return "3" 7 | 8 | def test_custom_evaluate_timeout_with_script(self): 9 | # Uncomment the following line to preserve 10 | # test case output and other files (config, state, ect.) 11 | # in system temp folder. 12 | self.set_delete_temp_folder(False) 13 | 14 | payload = """ 15 | { 16 | "data": { "_arg1": 1 }, 17 | "script": 18 | "import time\\ntime.sleep(100)\\nreturn 1" 19 | } 20 | """ 21 | headers = { 22 | "Content-Type": "application/json", 23 | "TabPy-Client": "Integration test for testing custom evaluate timeouts " 24 | "with scripts.", 25 | } 26 | 27 | conn = self._get_connection() 28 | conn.request("POST", "/evaluate", payload, headers) 29 | res = conn.getresponse() 30 | actual_error_message = res.read().decode("utf-8") 31 | 32 | self.assertEqual(408, res.status) 33 | self.assertEqual( 34 | '{"message": ' 35 | '"User defined script timed out. Timeout is set to 3.0 s.", ' 36 | '"info": {}}', 37 | actual_error_message, 38 | ) 39 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_and_evaluate_model.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | 3 | 4 | class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase): 5 | def _get_config_file_name(self) -> str: 6 | return "./tests/integration/resources/deploy_and_evaluate_model.conf" 7 | 8 | def _get_port(self) -> str: 9 | return "9008" 10 | 11 | def test_deploy_and_evaluate_model(self): 12 | # Uncomment the following line to preserve 13 | # test case output and other files (config, state, ect.) 14 | # in system temp folder. 15 | # self.set_delete_temp_folder(False) 16 | 17 | self.deploy_models(self._get_username(), self._get_password()) 18 | 19 | payload = """{ 20 | "data": { "_arg1": ["happy", "sad", "neutral"] }, 21 | "script": 22 | "return tabpy.query('Sentiment Analysis',_arg1)['response']" 23 | }""" 24 | 25 | conn = self._get_connection() 26 | conn.request("POST", "/evaluate", payload) 27 | SentimentAnalysis_eval = conn.getresponse() 28 | self.assertEqual(200, SentimentAnalysis_eval.status) 29 | SentimentAnalysis_eval.read() 30 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_and_evaluate_model_auth_on.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | 3 | 4 | class TestDeployAndEvaluateModelAuthOn(integ_test_base.IntegTestBase): 5 | def _get_config_file_name(self) -> str: 6 | return "./tests/integration/resources/deploy_and_evaluate_model_auth.conf" 7 | 8 | def _get_port(self) -> str: 9 | return "9009" 10 | 11 | def test_deploy_and_evaluate_model(self): 12 | # Uncomment the following line to preserve 13 | # test case output and other files (config, state, ect.) 14 | # in system temp folder. 15 | # self.set_delete_temp_folder(False) 16 | 17 | self.deploy_models(self._get_username(), self._get_password()) 18 | 19 | headers = { 20 | "Content-Type": "application/json", 21 | "Authorization": "Basic dXNlcjE6UEBzc3cwcmQ=", 22 | "Host": "localhost:9009", 23 | } 24 | payload = """{ 25 | "data": { "_arg1": ["happy", "sad", "neutral"] }, 26 | "script": 27 | "return tabpy.query('Sentiment Analysis',_arg1)['response']" 28 | }""" 29 | 30 | conn = self._get_connection() 31 | conn.request("POST", "/evaluate", payload, headers) 32 | SentimentAnalysis_eval = conn.getresponse() 33 | self.assertEqual(200, SentimentAnalysis_eval.status) 34 | SentimentAnalysis_eval.read() 35 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_and_evaluate_model_ssl.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | import requests 3 | 4 | 5 | class TestDeployAndEvaluateModelSSL(integ_test_base.IntegTestBase): 6 | def _get_port(self): 7 | return "9005" 8 | 9 | def _get_transfer_protocol(self) -> str: 10 | return "https" 11 | 12 | def _get_certificate_file_name(self) -> str: 13 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.crt" 14 | 15 | def _get_key_file_name(self) -> str: 16 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.key" 17 | 18 | def test_deploy_and_evaluate_model_ssl(self): 19 | # Uncomment the following line to preserve 20 | # test case output and other files (config, state, ect.) 21 | # in system temp folder. 22 | # self.set_delete_temp_folder(False) 23 | 24 | self.deploy_models(self._get_username(), self._get_password()) 25 | 26 | payload = """{ 27 | "data": { "_arg1": ["happy", "sad", "neutral"] }, 28 | "script": 29 | "return tabpy.query('Sentiment%20Analysis',_arg1)['response']" 30 | }""" 31 | 32 | session = requests.Session() 33 | # Do not verify servers' cert to be signed by trusted CA 34 | session.verify = False 35 | # Do not warn about insecure request 36 | requests.packages.urllib3.disable_warnings() 37 | response = session.post( 38 | f"{self._get_transfer_protocol()}://" 39 | f"localhost:{self._get_port()}/evaluate", 40 | data=payload, 41 | ) 42 | 43 | self.assertEqual(200, response.status_code) 44 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_model_ssl_off_auth_off.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | 3 | 4 | class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): 5 | def test_deploy_ssl_off_auth_off(self): 6 | # Uncomment the following line to preserve 7 | # test case output and other files (config, state, ect.) 8 | # in system temp folder. 9 | # self.set_delete_temp_folder(False) 10 | 11 | self.deploy_models(self._get_username(), self._get_password()) 12 | 13 | conn = self._get_connection() 14 | 15 | models = ["PCA", "Sentiment%20Analysis", "ttest", "anova"] 16 | for m in models: 17 | conn.request("GET", f"/endpoints/{m}") 18 | m_request = conn.getresponse() 19 | self.assertEqual(200, m_request.status) 20 | m_request.read() 21 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_model_ssl_off_auth_on.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | import base64 3 | 4 | 5 | class TestDeployModelSSLOffAuthOn(integ_test_base.IntegTestBase): 6 | def _get_pwd_file(self) -> str: 7 | return "./tests/integration/resources/pwdfile.txt" 8 | 9 | def test_deploy_ssl_off_auth_on(self): 10 | self.deploy_models(self._get_username(), self._get_password()) 11 | 12 | headers = { 13 | "Content-Type": "application/json", 14 | "TabPy-Client": "Integration test for deploying models with auth", 15 | "Authorization": "Basic " 16 | + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 17 | } 18 | 19 | conn = self._get_connection() 20 | 21 | models = ["PCA", "Sentiment%20Analysis", "ttest", "anova"] 22 | for m in models: 23 | conn.request("GET", f"/endpoints/{m}", headers=headers) 24 | m_request = conn.getresponse() 25 | self.assertEqual(200, m_request.status) 26 | m_request.read() 27 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_model_ssl_on_auth_off.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | import requests 3 | 4 | 5 | class TestDeployModelSSLOnAuthOff(integ_test_base.IntegTestBase): 6 | def _get_transfer_protocol(self) -> str: 7 | return "https" 8 | 9 | def _get_certificate_file_name(self) -> str: 10 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.crt" 11 | 12 | def _get_key_file_name(self) -> str: 13 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.key" 14 | 15 | def test_deploy_ssl_on_auth_off(self): 16 | self.deploy_models(self._get_username(), self._get_password()) 17 | 18 | session = requests.Session() 19 | # Do not verify servers' cert to be signed by trusted CA 20 | session.verify = False 21 | # Do not warn about insecure request 22 | requests.packages.urllib3.disable_warnings() 23 | 24 | models = ["PCA", "Sentiment%20Analysis", "ttest", "anova"] 25 | for m in models: 26 | m_response = session.get( 27 | url=f"{self._get_transfer_protocol()}://" 28 | f"localhost:9004/endpoints/{m}" 29 | ) 30 | self.assertEqual(200, m_response.status_code) 31 | -------------------------------------------------------------------------------- /tests/integration/test_deploy_model_ssl_on_auth_on.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | import base64 3 | import requests 4 | 5 | 6 | class TestDeployModelSSLOnAuthOn(integ_test_base.IntegTestBase): 7 | def _get_transfer_protocol(self) -> str: 8 | return "https" 9 | 10 | def _get_certificate_file_name(self) -> str: 11 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.crt" 12 | 13 | def _get_key_file_name(self) -> str: 14 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.key" 15 | 16 | def _get_pwd_file(self) -> str: 17 | return "./tests/integration/resources/pwdfile.txt" 18 | 19 | def test_deploy_ssl_on_auth_on(self): 20 | # Uncomment the following line to preserve 21 | # test case output and other files (config, state, ect.) 22 | # in system temp folder. 23 | # self.set_delete_temp_folder(False) 24 | 25 | self.deploy_models(self._get_username(), self._get_password()) 26 | 27 | headers = { 28 | "Content-Type": "application/json", 29 | "TabPy-Client": "Integration test for deploying models with auth", 30 | "Authorization": "Basic " 31 | + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 32 | } 33 | 34 | session = requests.Session() 35 | # Do not verify servers' cert to be signed by trusted CA 36 | session.verify = False 37 | # Do not warn about insecure request 38 | requests.packages.urllib3.disable_warnings() 39 | 40 | models = ["PCA", "Sentiment%20Analysis", "ttest", "anova"] 41 | for m in models: 42 | m_response = session.get( 43 | url=f"{self._get_transfer_protocol()}://" 44 | f"localhost:9004/endpoints/{m}", 45 | headers=headers, 46 | ) 47 | self.assertEqual(200, m_response.status_code) 48 | 49 | def test_override_model_ssl_on_auth_on(self): 50 | # Uncomment the following line to preserve 51 | # test case output and other files (config, state, ect.) 52 | # in system temp folder. 53 | # self.set_delete_temp_folder(False) 54 | 55 | self.deploy_models(self._get_username(), self._get_password()) 56 | 57 | # Override models 58 | self.deploy_models(self._get_username(), self._get_password()) 59 | 60 | headers = { 61 | "Content-Type": "application/json", 62 | "TabPy-Client": "Integration test for deploying models with auth", 63 | "Authorization": "Basic " 64 | + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 65 | } 66 | 67 | session = requests.Session() 68 | # Do not verify servers' cert to be signed by trusted CA 69 | session.verify = False 70 | # Do not warn about insecure request 71 | requests.packages.urllib3.disable_warnings() 72 | 73 | models = ["PCA", "Sentiment%20Analysis", "ttest", "anova"] 74 | for m in models: 75 | m_response = session.get( 76 | url=f"{self._get_transfer_protocol()}://" 77 | f"localhost:9004/endpoints/{m}", 78 | headers=headers, 79 | ) 80 | self.assertEqual(200, m_response.status_code) 81 | -------------------------------------------------------------------------------- /tests/integration/test_evaluate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script evaluation tests. 3 | """ 4 | 5 | from . import integ_test_base 6 | import json 7 | 8 | 9 | class TestEvaluate(integ_test_base.IntegTestBase): 10 | def test_single_value_returned(self): 11 | payload = """ 12 | { 13 | "data": { "_arg1": 2, "_arg2": 40 }, 14 | "script": 15 | "return _arg1 + _arg2" 16 | } 17 | """ 18 | headers = { 19 | "Content-Type": "application/json", 20 | } 21 | 22 | conn = self._get_connection() 23 | conn.request("POST", "/evaluate", payload, headers) 24 | response = conn.getresponse() 25 | result = response.read().decode("utf-8") 26 | 27 | self.assertEqual(200, response.status) 28 | self.assertEqual("42", result) 29 | 30 | def test_collection_returned(self): 31 | payload = """ 32 | { 33 | "data": { "_arg1": [2, 3], "_arg2": [40, 0.1415926] }, 34 | "script": 35 | "return [x + y for x, y in zip(_arg1, _arg2)]" 36 | } 37 | """ 38 | headers = { 39 | "Content-Type": "application/json", 40 | } 41 | 42 | conn = self._get_connection() 43 | conn.request("POST", "/evaluate", payload, headers) 44 | response = conn.getresponse() 45 | result = response.read().decode("utf-8") 46 | 47 | self.assertEqual(200, response.status) 48 | self.assertEqual("[42, 3.1415926]", result) 49 | 50 | def test_none_returned(self): 51 | payload = """ 52 | { 53 | "data": { "_arg1": 2, "_arg2": 40 }, 54 | "script": 55 | "return None" 56 | } 57 | """ 58 | headers = { 59 | "Content-Type": "application/json", 60 | } 61 | 62 | conn = self._get_connection() 63 | conn.request("POST", "/evaluate", payload, headers) 64 | response = conn.getresponse() 65 | result = response.read().decode("utf-8") 66 | 67 | self.assertEqual(200, response.status) 68 | self.assertEqual("null", result) 69 | 70 | def test_nothing_returned(self): 71 | payload = """ 72 | { 73 | "data": { "_arg1": [2], "_arg2": [40] }, 74 | "script": 75 | "res = [x + y for x, y in zip(_arg1, _arg2)]" 76 | } 77 | """ 78 | headers = { 79 | "Content-Type": "application/json", 80 | } 81 | 82 | conn = self._get_connection() 83 | conn.request("POST", "/evaluate", payload, headers) 84 | response = conn.getresponse() 85 | result = response.read().decode("utf-8") 86 | 87 | self.assertEqual(200, response.status) 88 | self.assertEqual("null", result) 89 | 90 | def test_syntax_error(self): 91 | payload = """ 92 | { 93 | "data": { "_arg1": [2], "_arg2": [40] }, 94 | "script": 95 | "% ^ !! return Nothing" 96 | } 97 | """ 98 | headers = { 99 | "Content-Type": "application/json", 100 | } 101 | 102 | conn = self._get_connection() 103 | conn.request("POST", "/evaluate", payload, headers) 104 | response = conn.getresponse() 105 | result = json.loads(response.read().decode("utf-8")) 106 | 107 | self.assertEqual(500, response.status) 108 | self.assertEqual("Error processing script", result["message"]) 109 | self.assertTrue(result["info"].startswith("SyntaxError")) 110 | -------------------------------------------------------------------------------- /tests/integration/test_gzip.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script evaluation tests. 3 | """ 4 | 5 | from . import integ_test_base 6 | import json 7 | import gzip 8 | import os 9 | import requests 10 | 11 | 12 | class TestEvaluate(integ_test_base.IntegTestBase): 13 | def _get_config_file_name(self) -> str: 14 | """ 15 | Generates config file. Overwrite this function for tests to 16 | run against not default state file. 17 | 18 | Returns 19 | ------- 20 | str 21 | Absolute path to config file. 22 | """ 23 | config_file = open(os.path.join(self.tmp_dir, "test.conf"), "w+") 24 | config_file.write( 25 | "[TabPy]\n" 26 | f"TABPY_QUERY_OBJECT_PATH = {self.tmp_dir}/query_objects\n" 27 | f"TABPY_PORT = {self._get_port()}\n" 28 | f"TABPY_GZIP_ENABLE = TRUE\n" 29 | f"TABPY_STATE_PATH = {self.tmp_dir}\n" 30 | ) 31 | 32 | pwd_file = self._get_pwd_file() 33 | if pwd_file is not None: 34 | pwd_file = os.path.abspath(pwd_file) 35 | config_file.write(f"TABPY_PWD_FILE = {pwd_file}\n") 36 | 37 | transfer_protocol = self._get_transfer_protocol() 38 | if transfer_protocol is not None: 39 | config_file.write(f"TABPY_TRANSFER_PROTOCOL = {transfer_protocol}\n") 40 | 41 | cert_file_name = self._get_certificate_file_name() 42 | if cert_file_name is not None: 43 | cert_file_name = os.path.abspath(cert_file_name) 44 | config_file.write(f"TABPY_CERTIFICATE_FILE = {cert_file_name}\n") 45 | 46 | key_file_name = self._get_key_file_name() 47 | if key_file_name is not None: 48 | key_file_name = os.path.abspath(key_file_name) 49 | config_file.write(f"TABPY_KEY_FILE = {key_file_name}\n") 50 | 51 | evaluate_timeout = self._get_evaluate_timeout() 52 | if evaluate_timeout is not None: 53 | config_file.write(f"TABPY_EVALUATE_TIMEOUT = {evaluate_timeout}\n") 54 | 55 | config_file.close() 56 | 57 | self.delete_config_file = True 58 | return config_file.name 59 | 60 | def test_single_value_returned(self): 61 | payload = """ 62 | { 63 | "data": { "_arg1": 2, "_arg2": 40 }, 64 | "script": 65 | "return _arg1 + _arg2" 66 | } 67 | """ 68 | headers = { 69 | "Content-Type": "application/json", 70 | "Content-Encoding": "gzip", 71 | } 72 | 73 | url = self._get_url() + "/evaluate" 74 | response = requests.request("POST", url, data=gzip.compress(payload.encode('utf-8')), 75 | headers=headers) 76 | result = json.loads(response.text) 77 | 78 | self.assertEqual(200, response.status_code) 79 | self.assertEqual(42, result) 80 | 81 | def test_syntax_error(self): 82 | payload = """ 83 | { 84 | "data": { "_arg1": [2], "_arg2": [40] }, 85 | "script": 86 | "% ^ !! return Nothing" 87 | } 88 | """ 89 | headers = { 90 | "Content-Type": "application/json", 91 | "Content-Encoding": "gzip", 92 | } 93 | 94 | url = self._get_url() + "/evaluate" 95 | response = requests.request("POST", url, data=gzip.compress(payload.encode('utf-8')), 96 | headers=headers) 97 | result = json.loads(response.text) 98 | 99 | self.assertEqual(500, response.status_code) 100 | self.assertEqual("Error processing script", result["message"]) 101 | self.assertTrue(result["info"].startswith("SyntaxError")) 102 | -------------------------------------------------------------------------------- /tests/integration/test_minimum_tls_version.py: -------------------------------------------------------------------------------- 1 | from . import integ_test_base 2 | import os 3 | 4 | class TestMinimumTLSVersion(integ_test_base.IntegTestBase): 5 | def _get_log_contents(self): 6 | with open(self.log_file_path, 'r') as f: 7 | return f.read() 8 | 9 | def _get_config_file_name(self, tls_version: str) -> str: 10 | config_file = open(os.path.join(self.tmp_dir, "test.conf"), "w+") 11 | config_file.write( 12 | "[TabPy]\n" 13 | "TABPY_PORT = 9005\n" 14 | "TABPY_TRANSFER_PROTOCOL = https\n" 15 | "TABPY_CERTIFICATE_FILE = ./tests/integration/resources/2019_04_24_to_3018_08_25.crt\n" 16 | "TABPY_KEY_FILE = ./tests/integration/resources/2019_04_24_to_3018_08_25.key\n" 17 | ) 18 | 19 | if tls_version is not None: 20 | config_file.write(f"TABPY_MINIMUM_TLS_VERSION = {tls_version}") 21 | 22 | pwd_file = self._get_pwd_file() 23 | if pwd_file is not None: 24 | pwd_file = os.path.abspath(pwd_file) 25 | config_file.write(f"TABPY_PWD_FILE = {pwd_file}\n") 26 | 27 | config_file.close() 28 | self.delete_config_file = True 29 | return config_file.name 30 | 31 | class TestMinimumTLSVersionValid(TestMinimumTLSVersion): 32 | def _get_config_file_name(self) -> str: 33 | return super()._get_config_file_name("TLSv1_3") 34 | 35 | def test_minimum_tls_version_valid(self): 36 | log_contents = self._get_log_contents() 37 | self.assertIn("Setting minimum TLS version to TLSv1_3", log_contents) 38 | 39 | class TestMinimumTLSVersionInvalid(TestMinimumTLSVersion): 40 | def _get_config_file_name(self) -> str: 41 | return super()._get_config_file_name("TLSv-1.3") 42 | 43 | def test_minimum_tls_version_invalid(self): 44 | log_contents = self._get_log_contents() 45 | self.assertIn("Unrecognized value for TABPY_MINIMUM_TLS_VERSION", log_contents) 46 | self.assertIn("Setting minimum TLS version to TLSv1_2", log_contents) 47 | 48 | class TestMinimumTLSVersionNotSpecified(TestMinimumTLSVersion): 49 | def _get_config_file_name(self) -> str: 50 | return super()._get_config_file_name(None) 51 | 52 | def test_minimum_tls_version_not_specified(self): 53 | log_contents = self._get_log_contents() 54 | self.assertIn("Setting minimum TLS version to TLSv1_2", log_contents) 55 | -------------------------------------------------------------------------------- /tests/integration/test_url.py: -------------------------------------------------------------------------------- 1 | """ 2 | All other misc. URL-related integration tests. 3 | """ 4 | 5 | from . import integ_test_base 6 | 7 | 8 | class TestURL(integ_test_base.IntegTestBase): 9 | def test_notexistent_url(self): 10 | # Uncomment the following line to preserve 11 | # test case output and other files (config, state, ect.) 12 | # in system temp folder. 13 | # self.set_delete_temp_folder(False) 14 | 15 | conn = self._get_connection() 16 | conn.request("GET", "/unicorn") 17 | res = conn.getresponse() 18 | 19 | self.assertEqual(404, res.status) 20 | 21 | def test_static_page(self): 22 | conn = self._get_connection() 23 | conn.request("GET", "/") 24 | res = conn.getresponse() 25 | 26 | self.assertEqual(200, res.status) 27 | -------------------------------------------------------------------------------- /tests/integration/test_url_ssl.py: -------------------------------------------------------------------------------- 1 | """ 2 | All other misc. URL-related integration tests for 3 | when SSL is turned on for TabPy. 4 | """ 5 | 6 | from . import integ_test_base 7 | import requests 8 | 9 | 10 | class TestURL_SSL(integ_test_base.IntegTestBase): 11 | def _get_port(self): 12 | return "9005" 13 | 14 | def _get_transfer_protocol(self) -> str: 15 | return "https" 16 | 17 | def _get_certificate_file_name(self) -> str: 18 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.crt" 19 | 20 | def _get_key_file_name(self) -> str: 21 | return "./tests/integration/resources/2019_04_24_to_3018_08_25.key" 22 | 23 | def test_notexistent_url(self): 24 | session = requests.Session() 25 | # Do not verify servers' cert to be signed by trusted CA 26 | session.verify = False 27 | # Do not warn about insecure request 28 | requests.packages.urllib3.disable_warnings() 29 | response = session.get(url=f"https://localhost:{self._get_port()}/unicorn") 30 | 31 | self.assertEqual(404, response.status_code) 32 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/server_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tableau/TabPy/fe613a11bc8310aaa38d06385a714221de9bd1d8/tests/unit/server_tests/__init__.py -------------------------------------------------------------------------------- /tests/unit/server_tests/resources/expired.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB+TCCAWICCQCyw6pPSfyxMTANBgkqhkiG9w0BAQsFADBBMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCVFgxCzAJBgNVBAcMAkFBMQswCQYDVQQKDAJBQTELMAkGA1UE 4 | CwwCQUEwHhcNMTgwODE3MTk0NzE4WhcNMTgwODE4MTk0NzE4WjBBMQswCQYDVQQG 5 | EwJVUzELMAkGA1UECAwCVFgxCzAJBgNVBAcMAkFBMQswCQYDVQQKDAJBQTELMAkG 6 | A1UECwwCQUEwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAN2HZs5zQhn8Wkmu 7 | WRpfH+zkN7S/nDFQ0j3cptygIrcbCL2l/cc3CZ5ZBtOFs9ZMo8ZrliT5lU94SlIH 8 | g8TVQw6fRPaL8gAwEusyxYyIS/o68jXXFK1CCpEJDfp4I/ZmLXBiImqHV2EzN7qF 9 | kvmOCgFMb97/3IysOzD4QUguZT1dAgMBAAEwDQYJKoZIhvcNAQELBQADgYEAw5yK 10 | BCiGavlhSm1mHETVo1XOhF9Gy34aj3PTO/lWXsWPwLWYnKLOE1/4bP1AgPs+RNqx 11 | PIas1ylWaLzw5HE0r+JKmtT6TtEveM+GBOKumfC/wFhL6IFyQBJiPLZKbPqSElXZ 12 | kxZdnApMcoYB6kYkUrjcsZZSfQ1w3BgAmsMjzQk= 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /tests/unit/server_tests/resources/future.crt: -------------------------------------------------------------------------------- 1 | Certificate: 2 | Data: 3 | Version: 1 (0x0) 4 | Serial Number: 1 (0x1) 5 | Signature Algorithm: sha1WithRSAEncryption 6 | Issuer: C=UA, ST=Dnipropetrovs'k reg., O=Planet Express, OU=Delivery Crew, CN=Fry\x1B[D\x1B[D\x1B[DH=\x1B[Phillip J. Fry/emailAddress=pfry@planetexpress.com 7 | Validity 8 | Not Before: Jan 1 00:00:00 3001 GMT 9 | Not After : Jan 2 00:00:00 3001 GMT 10 | Subject: C=UA, ST=Dnipropetrovs'k reg., O=Planet Express, OU=Delivery Crew, CN=Fry\x1B[D\x1B[D\x1B[DH=\x1B[Phillip J. Fry/emailAddress=pfry@planetexpress.com 11 | Subject Public Key Info: 12 | Public Key Algorithm: rsaEncryption 13 | Public-Key: (2048 bit) 14 | Modulus: 15 | 00:9b:da:01:8d:b3:c6:71:ed:2a:17:e9:33:f9:f7: 16 | 0a:4c:35:1b:25:48:c1:8d:d2:ed:18:e2:51:b1:50: 17 | 20:e7:85:24:ca:13:dc:08:41:3d:ab:93:dd:1b:98: 18 | 1b:c9:a3:31:46:2c:59:0c:04:15:97:ed:2d:ea:e7: 19 | 00:de:bf:31:2f:52:4d:1b:d5:b1:50:32:7c:8e:a7: 20 | a4:1d:c1:e4:fe:35:32:c0:72:4c:38:f4:0b:99:76: 21 | 89:2f:f9:32:1d:99:05:ec:b8:b0:46:8f:3e:7a:24: 22 | 63:18:a6:01:d3:af:a6:02:3c:1a:67:f6:c2:c5:82: 23 | de:e6:8f:49:4f:02:74:aa:d7:77:2c:63:f1:31:66: 24 | 54:a1:d4:79:59:05:5b:01:b8:c5:48:b8:79:d8:38: 25 | cd:30:11:bc:f7:a7:39:d8:14:85:6c:a5:10:8f:80: 26 | 08:97:3f:12:bc:85:18:dc:a1:e2:b7:4f:0c:4a:90: 27 | 71:32:c5:8b:b4:cf:ae:a3:6c:c6:c6:00:49:49:fb: 28 | 34:14:1d:72:4f:2f:0c:3b:53:1a:33:72:ed:69:8b: 29 | 51:45:2f:1c:c5:76:71:ca:94:f0:86:a8:d8:fd:dd: 30 | 6d:5a:c2:d5:74:69:6e:af:9a:94:66:aa:68:9f:ef: 31 | 87:7e:e3:3d:11:40:5c:e8:5f:33:9f:8d:8b:a1:1d: 32 | 96:45 33 | Exponent: 65537 (0x10001) 34 | Signature Algorithm: sha1WithRSAEncryption 35 | 13:78:d5:41:2b:07:bd:ec:35:90:f4:37:26:6d:e1:2c:e1:2a: 36 | 61:0b:c8:40:27:31:76:0d:a8:8e:fe:e2:a9:e5:74:fa:14:96: 37 | 71:fc:b4:3d:ac:dd:44:93:6e:cf:50:35:67:84:db:90:53:3a: 38 | 2e:47:df:98:5e:cc:c3:c7:7c:24:bf:06:2d:d0:4a:70:30:33: 39 | 1f:2e:59:5e:40:74:ba:f3:05:c4:25:1d:6e:e2:b2:1c:71:6d: 40 | b2:24:e0:8c:ab:c2:00:40:6c:ab:d8:57:84:90:f1:02:24:18: 41 | 29:9f:4c:b3:30:64:fa:25:5c:12:56:8a:a5:28:20:b2:b5:68: 42 | 86:4f:60:83:3c:d1:37:32:de:49:89:25:7a:cb:15:e6:54:a2: 43 | 47:b6:15:36:80:84:fa:61:8a:9c:63:e7:00:76:83:90:fc:90: 44 | 31:b8:fa:2f:32:45:1d:35:dc:ee:f4:c9:0b:de:9d:33:89:9a: 45 | 1a:fd:19:e0:63:23:7c:60:70:d6:a9:87:d1:24:24:c4:5f:57: 46 | 34:69:32:d8:18:fd:9a:25:8c:c6:88:c1:65:1c:3a:9c:6c:e5: 47 | 86:fb:e7:7f:dd:6e:a5:ac:d1:0e:65:fe:0c:19:12:63:4b:50: 48 | d7:ff:5d:cc:9d:0c:10:4a:b5:ad:bc:33:78:d0:84:91:11:04: 49 | c1:75:59:98 50 | -----BEGIN CERTIFICATE----- 51 | MIID1DCCArwCAQEwDQYJKoZIhvcNAQEFBQAwga0xCzAJBgNVBAYTAlVBMR0wGwYD 52 | VQQIDBREbmlwcm9wZXRyb3ZzJ2sgcmVnLjEXMBUGA1UECgwOUGxhbmV0IEV4cHJl 53 | c3MxFjAUBgNVBAsMDURlbGl2ZXJ5IENyZXcxJzAlBgNVBAMMHkZyeRtbRBtbRBtb 54 | REg9G1tQaGlsbGlwIEouIEZyeTElMCMGCSqGSIb3DQEJARYWcGZyeUBwbGFuZXRl 55 | eHByZXNzLmNvbTAiGA8zMDAxMDEwMTAwMDAwMFoYDzMwMDEwMTAyMDAwMDAwWjCB 56 | rTELMAkGA1UEBhMCVUExHTAbBgNVBAgMFERuaXByb3BldHJvdnMnayByZWcuMRcw 57 | FQYDVQQKDA5QbGFuZXQgRXhwcmVzczEWMBQGA1UECwwNRGVsaXZlcnkgQ3JldzEn 58 | MCUGA1UEAwweRnJ5G1tEG1tEG1tESD0bW1BoaWxsaXAgSi4gRnJ5MSUwIwYJKoZI 59 | hvcNAQkBFhZwZnJ5QHBsYW5ldGV4cHJlc3MuY29tMIIBIjANBgkqhkiG9w0BAQEF 60 | AAOCAQ8AMIIBCgKCAQEAm9oBjbPGce0qF+kz+fcKTDUbJUjBjdLtGOJRsVAg54Uk 61 | yhPcCEE9q5PdG5gbyaMxRixZDAQVl+0t6ucA3r8xL1JNG9WxUDJ8jqekHcHk/jUy 62 | wHJMOPQLmXaJL/kyHZkF7LiwRo8+eiRjGKYB06+mAjwaZ/bCxYLe5o9JTwJ0qtd3 63 | LGPxMWZUodR5WQVbAbjFSLh52DjNMBG896c52BSFbKUQj4AIlz8SvIUY3KHit08M 64 | SpBxMsWLtM+uo2zGxgBJSfs0FB1yTy8MO1MaM3LtaYtRRS8cxXZxypTwhqjY/d1t 65 | WsLVdGlur5qUZqpon++HfuM9EUBc6F8zn42LoR2WRQIDAQABMA0GCSqGSIb3DQEB 66 | BQUAA4IBAQATeNVBKwe97DWQ9DcmbeEs4SphC8hAJzF2DaiO/uKp5XT6FJZx/LQ9 67 | rN1Ek27PUDVnhNuQUzouR9+YXszDx3wkvwYt0EpwMDMfLlleQHS68wXEJR1u4rIc 68 | cW2yJOCMq8IAQGyr2FeEkPECJBgpn0yzMGT6JVwSVoqlKCCytWiGT2CDPNE3Mt5J 69 | iSV6yxXmVKJHthU2gIT6YYqcY+cAdoOQ/JAxuPovMkUdNdzu9MkL3p0ziZoa/Rng 70 | YyN8YHDWqYfRJCTEX1c0aTLYGP2aJYzGiMFlHDqcbOWG++d/3W6lrNEOZf4MGRJj 71 | S1DX/13MnQwQSrWtvDN40ISREQTBdVmY 72 | -----END CERTIFICATE----- 73 | -------------------------------------------------------------------------------- /tests/unit/server_tests/resources/valid.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID1DCCArwCAQEwDQYJKoZIhvcNAQEFBQAwga0xCzAJBgNVBAYTAlVBMR0wGwYD 3 | VQQIDBREbmlwcm9wZXRyb3ZzJ2sgcmVnLjEXMBUGA1UECgwOUGxhbmV0IEV4cHJl 4 | c3MxFjAUBgNVBAsMDURlbGl2ZXJ5IENyZXcxJzAlBgNVBAMMHkZyeRtbRBtbRBtb 5 | REg9G1tQaGlsbGlwIEouIEZyeTElMCMGCSqGSIb3DQEJARYWcGZyeUBwbGFuZXRl 6 | eHByZXNzLmNvbTAiGA8yMDAxMDEwMTAwMDAwMFoYDzM5OTkwMTAyMDAwMDAwWjCB 7 | rTELMAkGA1UEBhMCVUExHTAbBgNVBAgMFERuaXByb3BldHJvdnMnayByZWcuMRcw 8 | FQYDVQQKDA5QbGFuZXQgRXhwcmVzczEWMBQGA1UECwwNRGVsaXZlcnkgQ3JldzEn 9 | MCUGA1UEAwweRnJ5G1tEG1tEG1tESD0bW1BoaWxsaXAgSi4gRnJ5MSUwIwYJKoZI 10 | hvcNAQkBFhZwZnJ5QHBsYW5ldGV4cHJlc3MuY29tMIIBIjANBgkqhkiG9w0BAQEF 11 | AAOCAQ8AMIIBCgKCAQEAm9oBjbPGce0qF+kz+fcKTDUbJUjBjdLtGOJRsVAg54Uk 12 | yhPcCEE9q5PdG5gbyaMxRixZDAQVl+0t6ucA3r8xL1JNG9WxUDJ8jqekHcHk/jUy 13 | wHJMOPQLmXaJL/kyHZkF7LiwRo8+eiRjGKYB06+mAjwaZ/bCxYLe5o9JTwJ0qtd3 14 | LGPxMWZUodR5WQVbAbjFSLh52DjNMBG896c52BSFbKUQj4AIlz8SvIUY3KHit08M 15 | SpBxMsWLtM+uo2zGxgBJSfs0FB1yTy8MO1MaM3LtaYtRRS8cxXZxypTwhqjY/d1t 16 | WsLVdGlur5qUZqpon++HfuM9EUBc6F8zn42LoR2WRQIDAQABMA0GCSqGSIb3DQEB 17 | BQUAA4IBAQBmMbevyMNZ/QsQh/kQbYhtMsGvugxRVf8q9/BCFkOw3OTpNUBKk+FW 18 | ty/PVbINi6MPkz4a32FsQjBW/4+Ke0AjcEPUzR45A5yaNOa+qTUOObyCj6CvKipt 19 | DylpLreQ0V/SBJfEO/f0LHvhD8IGuiybc89PAqQWzZWk6WSWqIb1JR12xTTOf2BD 20 | R2UR/1XCrlivovT6SqjMGmWav5TQUXnP0SED7zif1+RhSkyjfGLmhp1Uh6T7ZEDu 21 | YRnFw0rIpPMJ4U9aMTwwXo6MDQXobxzRCuLhfc7AgQqIOPaKPXpOKn4Ao7dC5WhY 22 | ebN/TaTQRnzVVuAKocnliaMpAvU88AXI 23 | -----END CERTIFICATE----- 24 | -------------------------------------------------------------------------------- /tests/unit/server_tests/test_endpoint_file_manager.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tabpy.tabpy_server.common.endpoint_file_mgr import _check_endpoint_name 3 | 4 | 5 | class TestEndpointFileManager(unittest.TestCase): 6 | def test_endpoint_name_not_str(self): 7 | self.assertRaises(TypeError, _check_endpoint_name, 2) 8 | 9 | def test_endpoint_name_empty_str(self): 10 | self.assertRaises(ValueError, _check_endpoint_name, "") 11 | 12 | def test_endpoint_name_wrong_regex(self): 13 | self.assertRaises(ValueError, _check_endpoint_name, "****") 14 | -------------------------------------------------------------------------------- /tests/unit/server_tests/test_endpoint_handler.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import sys 4 | import tempfile 5 | 6 | from tabpy.tabpy_server.app.app import TabPyApp 7 | from tabpy.tabpy_server.app.app import _init_asyncio_patch 8 | from tabpy.tabpy_server.handlers.util import hash_password 9 | from tornado.testing import AsyncHTTPTestCase 10 | 11 | 12 | class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): 13 | @classmethod 14 | def setUpClass(cls): 15 | _init_asyncio_patch() 16 | prefix = "__TestEndpointHandlerWithAuth_" 17 | # create password file 18 | cls.pwd_file = tempfile.NamedTemporaryFile( 19 | mode="w+t", prefix=prefix, suffix=".txt", delete=False 20 | ) 21 | username = "username" 22 | password = "password" 23 | cls.pwd_file.write(f"{username} {hash_password(username, password)}") 24 | cls.pwd_file.close() 25 | 26 | # create state.ini dir and file 27 | cls.state_dir = tempfile.mkdtemp(prefix=prefix) 28 | cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") 29 | cls.state_file.write( 30 | "[Service Info]\n" 31 | "Name = TabPy Serve\n" 32 | "Description = \n" 33 | "Creation Time = 0\n" 34 | "Access-Control-Allow-Origin = \n" 35 | "Access-Control-Allow-Headers = \n" 36 | "Access-Control-Allow-Methods = \n" 37 | "\n" 38 | "[Query Objects Service Versions]\n" 39 | "\n" 40 | "[Query Objects Docstrings]\n" 41 | "\n" 42 | "[Meta]\n" 43 | "Revision Number = 1\n" 44 | ) 45 | cls.state_file.close() 46 | 47 | # create config file 48 | cls.config_file = tempfile.NamedTemporaryFile( 49 | mode="w+t", prefix=prefix, suffix=".conf", delete=False 50 | ) 51 | cls.config_file.write( 52 | "[TabPy]\n" 53 | f"TABPY_PWD_FILE = {cls.pwd_file.name}\n" 54 | f"TABPY_STATE_PATH = {cls.state_dir}" 55 | ) 56 | cls.config_file.close() 57 | 58 | @classmethod 59 | def tearDownClass(cls): 60 | os.remove(cls.pwd_file.name) 61 | os.remove(cls.state_file.name) 62 | os.remove(cls.config_file.name) 63 | os.rmdir(cls.state_dir) 64 | 65 | def get_app(self): 66 | self.app = TabPyApp(self.config_file.name) 67 | return self.app._create_tornado_web_app() 68 | 69 | def test_no_creds_required_auth_fails(self): 70 | response = self.fetch("/endpoints/anything") 71 | self.assertEqual(401, response.code) 72 | 73 | def test_invalid_creds_fails(self): 74 | response = self.fetch( 75 | "/endpoints/anything", 76 | method="GET", 77 | headers={ 78 | "Authorization": "Basic {}".format( 79 | base64.b64encode("user:wrong_password".encode("utf-8")).decode( 80 | "utf-8" 81 | ) 82 | ) 83 | }, 84 | ) 85 | self.assertEqual(401, response.code) 86 | 87 | def test_valid_creds_pass(self): 88 | response = self.fetch( 89 | "/endpoints/", 90 | method="GET", 91 | headers={ 92 | "Authorization": "Basic {}".format( 93 | base64.b64encode("username:password".encode("utf-8")).decode( 94 | "utf-8" 95 | ) 96 | ) 97 | }, 98 | ) 99 | self.assertEqual(200, response.code) 100 | 101 | def test_valid_creds_unknown_endpoint_fails(self): 102 | response = self.fetch( 103 | "/endpoints/unknown_endpoint", 104 | method="GET", 105 | headers={ 106 | "Authorization": "Basic {}".format( 107 | base64.b64encode("username:password".encode("utf-8")).decode( 108 | "utf-8" 109 | ) 110 | ) 111 | }, 112 | ) 113 | self.assertEqual(404, response.code) 114 | 115 | 116 | class TestEndpointHandlerWithoutAuth(AsyncHTTPTestCase): 117 | @classmethod 118 | def setUpClass(cls): 119 | _init_asyncio_patch() 120 | prefix = "__TestEndpointHandlerWithoutAuth_" 121 | 122 | # create state.ini dir and file 123 | cls.state_dir = tempfile.mkdtemp(prefix=prefix) 124 | cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") 125 | cls.state_file.write( 126 | "[Service Info]\n" 127 | "Name = TabPy Serve\n" 128 | "Description = \n" 129 | "Creation Time = 0\n" 130 | "Access-Control-Allow-Origin = \n" 131 | "Access-Control-Allow-Headers = \n" 132 | "Access-Control-Allow-Methods = \n" 133 | "\n" 134 | "[Query Objects Service Versions]\n" 135 | "\n" 136 | "[Query Objects Docstrings]\n" 137 | "\n" 138 | "[Meta]\n" 139 | "Revision Number = 1\n" 140 | ) 141 | cls.state_file.close() 142 | 143 | @classmethod 144 | def tearDownClass(cls): 145 | os.remove(cls.state_file.name) 146 | os.rmdir(cls.state_dir) 147 | 148 | def get_app(self): 149 | self.app = TabPyApp(None) 150 | return self.app._create_tornado_web_app() 151 | 152 | def test_creds_no_auth_fails(self): 153 | response = self.fetch( 154 | "/endpoints/", 155 | method="GET", 156 | headers={ 157 | "Authorization": "Basic {}".format( 158 | base64.b64encode("username:password".encode("utf-8")).decode( 159 | "utf-8" 160 | ) 161 | ) 162 | }, 163 | ) 164 | self.assertEqual(406, response.code) 165 | -------------------------------------------------------------------------------- /tests/unit/server_tests/test_endpoints_handler.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import tempfile 4 | 5 | from tabpy.tabpy_server.app.app import TabPyApp 6 | from tabpy.tabpy_server.handlers.util import hash_password 7 | from tornado.testing import AsyncHTTPTestCase 8 | 9 | 10 | class TestEndpointsHandlerWithAuth(AsyncHTTPTestCase): 11 | @classmethod 12 | def setUpClass(cls): 13 | prefix = "__TestEndpointsHandlerWithAuth_" 14 | # create password file 15 | cls.pwd_file = tempfile.NamedTemporaryFile( 16 | mode="w+t", prefix=prefix, suffix=".txt", delete=False 17 | ) 18 | username = "username" 19 | password = "password" 20 | cls.pwd_file.write(f"{username} {hash_password(username, password)}") 21 | cls.pwd_file.close() 22 | 23 | # create state.ini dir and file 24 | cls.state_dir = tempfile.mkdtemp(prefix=prefix) 25 | cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") 26 | cls.state_file.write( 27 | "[Service Info]\n" 28 | "Name = TabPy Serve\n" 29 | "Description = \n" 30 | "Creation Time = 0\n" 31 | "Access-Control-Allow-Origin = \n" 32 | "Access-Control-Allow-Headers = \n" 33 | "Access-Control-Allow-Methods = \n" 34 | "\n" 35 | "[Query Objects Service Versions]\n" 36 | "\n" 37 | "[Query Objects Docstrings]\n" 38 | "\n" 39 | "[Meta]\n" 40 | "Revision Number = 1\n" 41 | ) 42 | cls.state_file.close() 43 | 44 | # create config file 45 | cls.config_file = tempfile.NamedTemporaryFile( 46 | mode="w+t", prefix=prefix, suffix=".conf", delete=False 47 | ) 48 | cls.config_file.write( 49 | "[TabPy]\n" 50 | f"TABPY_PWD_FILE = {cls.pwd_file.name}\n" 51 | f"TABPY_STATE_PATH = {cls.state_dir}" 52 | ) 53 | cls.config_file.close() 54 | 55 | @classmethod 56 | def tearDownClass(cls): 57 | os.remove(cls.pwd_file.name) 58 | os.remove(cls.state_file.name) 59 | os.remove(cls.config_file.name) 60 | os.rmdir(cls.state_dir) 61 | 62 | def get_app(self): 63 | self.app = TabPyApp(self.config_file.name) 64 | return self.app._create_tornado_web_app() 65 | 66 | def test_no_creds_required_auth_fails(self): 67 | response = self.fetch("/endpoints") 68 | self.assertEqual(401, response.code) 69 | 70 | def test_invalid_creds_fails(self): 71 | response = self.fetch( 72 | "/endpoints", 73 | method="GET", 74 | headers={ 75 | "Authorization": "Basic {}".format( 76 | base64.b64encode("user:wrong_password".encode("utf-8")).decode( 77 | "utf-8" 78 | ) 79 | ) 80 | }, 81 | ) 82 | self.assertEqual(401, response.code) 83 | 84 | def test_valid_creds_pass(self): 85 | response = self.fetch( 86 | "/endpoints", 87 | method="GET", 88 | headers={ 89 | "Authorization": "Basic {}".format( 90 | base64.b64encode("username:password".encode("utf-8")).decode( 91 | "utf-8" 92 | ) 93 | ) 94 | }, 95 | ) 96 | self.assertEqual(200, response.code) 97 | 98 | 99 | class TestEndpointsHandlerWithoutAuth(AsyncHTTPTestCase): 100 | @classmethod 101 | def setUpClass(cls): 102 | prefix = "__TestEndpointsHandlerWithoutAuth_" 103 | 104 | # create state.ini dir and file 105 | cls.state_dir = tempfile.mkdtemp(prefix=prefix) 106 | cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") 107 | cls.state_file.write( 108 | "[Service Info]\n" 109 | "Name = TabPy Serve\n" 110 | "Description = \n" 111 | "Creation Time = 0\n" 112 | "Access-Control-Allow-Origin = \n" 113 | "Access-Control-Allow-Headers = \n" 114 | "Access-Control-Allow-Methods = \n" 115 | "\n" 116 | "[Query Objects Service Versions]\n" 117 | "\n" 118 | "[Query Objects Docstrings]\n" 119 | "\n" 120 | "[Meta]\n" 121 | "Revision Number = 1\n" 122 | ) 123 | cls.state_file.close() 124 | 125 | @classmethod 126 | def tearDownClass(cls): 127 | os.remove(cls.state_file.name) 128 | os.rmdir(cls.state_dir) 129 | 130 | def get_app(self): 131 | self.app = TabPyApp(None) 132 | return self.app._create_tornado_web_app() 133 | 134 | def test_creds_no_auth_fails(self): 135 | response = self.fetch( 136 | "/endpoints", 137 | method="GET", 138 | headers={ 139 | "Authorization": "Basic {}".format( 140 | base64.b64encode("username:password".encode("utf-8")).decode( 141 | "utf-8" 142 | ) 143 | ) 144 | }, 145 | ) 146 | self.assertEqual(406, response.code) 147 | -------------------------------------------------------------------------------- /tests/unit/server_tests/test_pwd_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from tempfile import NamedTemporaryFile 4 | 5 | from tabpy.tabpy_server.app.app import TabPyApp 6 | 7 | 8 | class TestPasswordFile(unittest.TestCase): 9 | def setUp(self): 10 | self.config_file = NamedTemporaryFile(mode="w", delete=False) 11 | self.config_file.close() 12 | self.pwd_file = NamedTemporaryFile(mode="w", delete=False) 13 | self.pwd_file.close() 14 | 15 | def tearDown(self): 16 | os.remove(self.config_file.name) 17 | self.config_file = None 18 | os.remove(self.pwd_file.name) 19 | self.pwd_file = None 20 | 21 | def _set_file(self, file_name, value): 22 | with open(file_name, "w") as f: 23 | f.write(value) 24 | 25 | def test_given_no_pwd_file_expect_empty_credentials_list(self): 26 | self._set_file( 27 | self.config_file.name, "[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = http" 28 | ) 29 | 30 | app = TabPyApp(self.config_file.name) 31 | self.assertDictEqual( 32 | app.credentials, 33 | {}, 34 | "Expected no credentials with no password file provided", 35 | ) 36 | 37 | def test_given_empty_pwd_file_expect_app_fails(self): 38 | self._set_file( 39 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 40 | ) 41 | 42 | self._set_file(self.pwd_file.name, "# just a comment") 43 | 44 | with self.assertRaises(RuntimeError) as cm: 45 | TabPyApp(self.config_file.name) 46 | ex = cm.exception 47 | self.assertEqual( 48 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 49 | ) 50 | 51 | def test_given_missing_pwd_file_expect_app_fails(self): 52 | self._set_file(self.config_file.name, "[TabPy]\n" "TABPY_PWD_FILE = foo") 53 | 54 | with self.assertRaises(RuntimeError) as cm: 55 | TabPyApp(self.config_file.name) 56 | ex = cm.exception 57 | self.assertEqual( 58 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 59 | ) 60 | 61 | def test_given_one_password_in_pwd_file_expect_one_credentials_entry(self): 62 | self._set_file( 63 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 64 | ) 65 | 66 | login = "user_name_123" 67 | pwd = "someting@something_else" 68 | self._set_file(self.pwd_file.name, "# passwords\n" "\n" f"{login} {pwd}") 69 | 70 | app = TabPyApp(self.config_file.name) 71 | 72 | self.assertEqual(len(app.credentials), 1) 73 | self.assertIn(login, app.credentials) 74 | self.assertEqual(app.credentials[login], pwd) 75 | 76 | def test_given_username_but_no_password_expect_parsing_fails(self): 77 | self._set_file( 78 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 79 | ) 80 | 81 | login = "user_name_123" 82 | pwd = "" 83 | self._set_file(self.pwd_file.name, "# passwords\n" "\n" f"{login} {pwd}") 84 | 85 | with self.assertRaises(RuntimeError) as cm: 86 | TabPyApp(self.config_file.name) 87 | ex = cm.exception 88 | self.assertEqual( 89 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 90 | ) 91 | 92 | def test_given_duplicate_usernames_expect_parsing_fails(self): 93 | self._set_file( 94 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 95 | ) 96 | 97 | login = "user_name_123" 98 | pwd = "hashedpw" 99 | self._set_file( 100 | self.pwd_file.name, "# passwords\n" "\n" f"{login} {pwd}\n{login} {pwd}" 101 | ) 102 | 103 | with self.assertRaises(RuntimeError) as cm: 104 | TabPyApp(self.config_file.name) 105 | ex = cm.exception 106 | self.assertEqual( 107 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 108 | ) 109 | 110 | def test_given_one_line_with_too_many_params_expect_app_fails(self): 111 | self._set_file( 112 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 113 | ) 114 | 115 | self._set_file( 116 | self.pwd_file.name, 117 | "# passwords\n" "user1 pwd1\n" "user_2 pwd#2" "user1 pwd@3", 118 | ) 119 | 120 | with self.assertRaises(RuntimeError) as cm: 121 | TabPyApp(self.config_file.name) 122 | ex = cm.exception 123 | self.assertEqual( 124 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 125 | ) 126 | 127 | def test_given_different_cases_in_pwd_file_expect_app_fails(self): 128 | self._set_file( 129 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 130 | ) 131 | 132 | self._set_file( 133 | self.pwd_file.name, 134 | "# passwords\n" "user1 pwd1\n" "user_2 pwd#2" "UseR1 pwd@3", 135 | ) 136 | 137 | with self.assertRaises(RuntimeError) as cm: 138 | TabPyApp(self.config_file.name) 139 | ex = cm.exception 140 | self.assertEqual( 141 | f"Failed to read password file {self.pwd_file.name}", ex.args[0] 142 | ) 143 | 144 | def test_given_multiple_credentials_expect_all_parsed(self): 145 | self._set_file( 146 | self.config_file.name, "[TabPy]\n" f"TABPY_PWD_FILE = {self.pwd_file.name}" 147 | ) 148 | creds = {"user_1": "pwd_1", "user@2": "pwd@2", "user#3": "pwd#3"} 149 | 150 | pwd_file_context = "" 151 | for login in creds: 152 | pwd_file_context += f"{login} {creds[login]}\n" 153 | 154 | self._set_file(self.pwd_file.name, pwd_file_context) 155 | app = TabPyApp(self.config_file.name) 156 | 157 | self.assertCountEqual(creds, app.credentials) 158 | for login in creds: 159 | self.assertIn(login, app.credentials) 160 | self.assertEqual(creds[login], app.credentials[login]) 161 | -------------------------------------------------------------------------------- /tests/unit/server_tests/test_service_info_handler.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | import os 4 | from tabpy.tabpy_server.app.app import TabPyApp 5 | from tabpy.tabpy_server.app.app_parameters import SettingsParameters 6 | import tempfile 7 | from tornado.testing import AsyncHTTPTestCase 8 | 9 | 10 | def _create_expected_info_response(settings, tabpy_state): 11 | return { 12 | "description": tabpy_state.get_description(), 13 | "creation_time": tabpy_state.creation_time, 14 | "state_path": settings["state_file_path"], 15 | "server_version": settings[SettingsParameters.ServerVersion], 16 | "name": tabpy_state.name, 17 | "versions": settings["versions"] 18 | } 19 | 20 | 21 | class BaseTestServiceInfoHandler(AsyncHTTPTestCase): 22 | def get_app(self): 23 | if hasattr(self, 'config_file') and hasattr(self.config_file, 'name'): 24 | self.app = TabPyApp(self.config_file.name) 25 | else: 26 | self.app = TabPyApp() 27 | return self.app._create_tornado_web_app() 28 | 29 | @classmethod 30 | def tearDownClass(cls): 31 | os.remove(cls.state_file.name) 32 | os.remove(cls.config_file.name) 33 | os.rmdir(cls.state_dir) 34 | 35 | @classmethod 36 | def setUpClass(cls): 37 | # create state.ini dir and file 38 | cls.state_dir = tempfile.mkdtemp(prefix=cls.prefix) 39 | with open(os.path.join(cls.state_dir, "state.ini"), "w+") as cls.state_file: 40 | cls.state_file.write( 41 | "[Service Info]\n" 42 | "Name = TabPy Serve\n" 43 | "Description = \n" 44 | "Creation Time = 0\n" 45 | "Access-Control-Allow-Origin = \n" 46 | "Access-Control-Allow-Headers = \n" 47 | "Access-Control-Allow-Methods = \n" 48 | "\n" 49 | "[Query Objects Service Versions]\n" 50 | "\n" 51 | "[Query Objects Docstrings]\n" 52 | "\n" 53 | "[Meta]\n" 54 | "Revision Number = 1\n" 55 | ) 56 | cls.state_file.close() 57 | 58 | # create config file 59 | cls.config_file = tempfile.NamedTemporaryFile( 60 | prefix=cls.prefix, suffix=".conf", delete=False, mode='w' 61 | ) 62 | cls.config_file.write("[TabPy]\n") 63 | if hasattr(cls, 'tabpy_config'): 64 | for k in cls.tabpy_config: 65 | cls.config_file.write(k) 66 | cls.config_file.close() 67 | 68 | 69 | class TestServiceInfoHandlerWithAuth(BaseTestServiceInfoHandler): 70 | @classmethod 71 | def setUpClass(cls): 72 | cls.prefix = "__TestServiceInfoHandlerWithAuth_" 73 | cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] 74 | super(TestServiceInfoHandlerWithAuth, cls).setUpClass() 75 | 76 | def test_given_server_with_auth_expect_error_info_response(self): 77 | response = self.fetch("/info") 78 | self.assertEqual(response.code, 401) 79 | 80 | def test_given_server_with_auth_expect_correct_info_response(self): 81 | header = { 82 | "Content-Type": "application/json", 83 | "TabPy-Client": "Integration test for deploying models with auth", 84 | "Authorization": "Basic " + 85 | base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 86 | } 87 | 88 | response = self.fetch("/info", headers=header) 89 | self.assertEqual(response.code, 200) 90 | actual_response = json.loads(response.body) 91 | expected_response = _create_expected_info_response( 92 | self.app.settings, self.app.tabpy_state 93 | ) 94 | 95 | self.assertDictEqual(actual_response, expected_response) 96 | self.assertTrue("versions" in actual_response) 97 | versions = actual_response["versions"] 98 | self.assertTrue("v1" in versions) 99 | v1 = versions["v1"] 100 | self.assertTrue("features" in v1) 101 | features = v1["features"] 102 | self.assertDictEqual( 103 | {"authentication": {"methods": {"basic-auth": {}}, "required": True}, 104 | 'arrow_enabled': False, 'evaluate_enabled': True, 'gzip_enabled': True}, 105 | features, 106 | ) 107 | 108 | 109 | class TestServiceInfoHandlerWithoutAuth(BaseTestServiceInfoHandler): 110 | @classmethod 111 | def setUpClass(cls): 112 | cls.prefix = "__TestServiceInfoHandlerWithoutAuth_" 113 | super(TestServiceInfoHandlerWithoutAuth, cls).setUpClass() 114 | 115 | def test_server_with_no_auth_expect_correct_info_response(self): 116 | response = self.fetch("/info") 117 | self.assertEqual(response.code, 200) 118 | actual_response = json.loads(response.body) 119 | expected_response = _create_expected_info_response( 120 | self.app.settings, self.app.tabpy_state 121 | ) 122 | 123 | self.assertDictEqual(actual_response, expected_response) 124 | self.assertTrue("versions" in actual_response) 125 | versions = actual_response["versions"] 126 | self.assertTrue("v1" in versions) 127 | v1 = versions["v1"] 128 | self.assertTrue("features" in v1) 129 | features = v1["features"] 130 | self.assertDictEqual({'arrow_enabled': False, 'evaluate_enabled': True, 'gzip_enabled': True}, features) 131 | 132 | def test_given_server_with_no_auth_and_password_expect_correct_info_response(self): 133 | header = { 134 | "Content-Type": "application/json", 135 | "TabPy-Client": "Integration test for deploying models with auth", 136 | "Authorization": "Basic " + 137 | base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), 138 | } 139 | 140 | response = self.fetch("/info", headers=header) 141 | self.assertEqual(response.code, 406) 142 | -------------------------------------------------------------------------------- /tests/unit/tools_tests/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # Keep test cases logging quiet 4 | logging.basicConfig(level=logging.CRITICAL + 1) 5 | -------------------------------------------------------------------------------- /tests/unit/tools_tests/test_client.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import Mock 3 | 4 | from tabpy.tabpy_tools.client import Client 5 | from tabpy.tabpy_tools.client import _check_endpoint_name 6 | 7 | 8 | class TestClient(unittest.TestCase): 9 | def setUp(self): 10 | self.client = Client("http://example.com/") 11 | self.client._service = Mock() # TODO: should spec this 12 | 13 | def test_init(self): 14 | client = Client("http://example.com:9004") 15 | self.assertEqual(client._endpoint, "http://example.com:9004") 16 | self.assertEqual(client._remote_server, False) 17 | 18 | client = Client("http://example.com/", 10.0) 19 | self.assertEqual(client._endpoint, "http://example.com/") 20 | 21 | client = Client(endpoint="https://example.com/", query_timeout=-10.0) 22 | self.assertEqual(client._endpoint, "https://example.com/") 23 | self.assertEqual(client.query_timeout, 0.0) 24 | 25 | client = Client( 26 | "http://example.com:442/", 27 | remote_server=True, 28 | localhost_endpoint="http://localhost:9004/" 29 | ) 30 | self.assertEqual(client._endpoint, "http://example.com:442/") 31 | self.assertEqual(client._remote_server, True) 32 | self.assertEqual(client._localhost_endpoint, "http://localhost:9004/") 33 | 34 | # valid name tests 35 | with self.assertRaises(ValueError): 36 | Client("") 37 | with self.assertRaises(TypeError): 38 | Client(1.0) 39 | with self.assertRaises(ValueError): 40 | Client("*#") 41 | with self.assertRaises(TypeError): 42 | Client() 43 | with self.assertRaises(ValueError): 44 | Client("http:/www.example.com/") 45 | with self.assertRaises(ValueError): 46 | Client("httpx://www.example.com:9004") 47 | 48 | def test_get_status(self): 49 | self.client._service.get_status.return_value = "asdf" 50 | self.assertEqual(self.client.get_status(), "asdf") 51 | 52 | def test_query_timeout(self): 53 | self.client.query_timeout = 5.0 54 | self.assertEqual(self.client.query_timeout, 5.0) 55 | self.assertEqual(self.client._service.query_timeout, 5.0) 56 | 57 | def test_query(self): 58 | self.client._service.query.return_value = "ok" 59 | 60 | self.assertEqual(self.client.query("foo", 1, 2, 3), "ok") 61 | 62 | self.client._service.query.assert_called_once_with("foo", 1, 2, 3) 63 | 64 | self.client._service.query.reset_mock() 65 | 66 | self.assertEqual(self.client.query("foo", a=1, b=2, c=3), "ok") 67 | 68 | self.client._service.query.assert_called_once_with("foo", a=1, b=2, c=3) 69 | 70 | def test_get_endpoints(self): 71 | self.client._service.get_endpoints.return_value = "foo" 72 | 73 | self.assertEqual(self.client.get_endpoints("foo"), "foo") 74 | 75 | self.client._service.get_endpoints.assert_called_once_with("foo") 76 | 77 | def test_get_endpoint_upload_destination(self): 78 | self.client._service.get_endpoint_upload_destination.return_value = { 79 | "path": "foo" 80 | } 81 | 82 | self.assertEqual(self.client._get_endpoint_upload_destination(), "foo") 83 | 84 | def test_set_credentials(self): 85 | username, password = "username", "password" 86 | self.client.set_credentials(username, password) 87 | 88 | self.client._service.set_credentials.assert_called_once_with(username, password) 89 | 90 | def test_check_invalid_endpoint_name(self): 91 | endpoint_name = "Invalid:model:@name" 92 | with self.assertRaises(ValueError) as err: 93 | _check_endpoint_name(endpoint_name) 94 | 95 | self.assertEqual( 96 | err.exception.args[0], 97 | f"endpoint name {endpoint_name } can only contain: " 98 | "a-z, A-Z, 0-9, underscore, hyphens and spaces.", 99 | ) 100 | 101 | def test_deploy_with_remote_server(self): 102 | client = Client("http://example.com:9004/", remote_server=True) 103 | mock_evaluate_remote_script = Mock() 104 | client._evaluate_remote_script = mock_evaluate_remote_script 105 | client.deploy('name', lambda: True, 'description') 106 | mock_evaluate_remote_script.assert_called() 107 | 108 | def test_gen_remote_script(self): 109 | client = Client("http://example.com:9004/", remote_server=True) 110 | script = client._gen_remote_script() 111 | self.assertTrue("from tabpy.tabpy_tools.client import Client" in script) 112 | self.assertTrue("client = Client('http://example.com:9004/')" in script) 113 | self.assertFalse("client.set_credentials" in script) 114 | 115 | client.set_credentials("username", "password") 116 | script = client._gen_remote_script() 117 | self.assertTrue("client.set_credentials('username', 'password')" in script) 118 | -------------------------------------------------------------------------------- /tests/unit/tools_tests/test_rest_object.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | 4 | from tabpy.tabpy_tools.rest import RESTObject, RESTProperty, enum 5 | 6 | 7 | class TestRESTObject(unittest.TestCase): 8 | def test_new_class(self): 9 | class FooObject(RESTObject): 10 | f = RESTProperty(float) 11 | i = RESTProperty(int) 12 | s = RESTProperty(str) 13 | e = RESTProperty(enum("a", "b")) 14 | 15 | f = FooObject(f="6.0", i="3", s="hello!") 16 | self.assertEqual(f.f, 6.0) 17 | self.assertEqual(f.i, 3) 18 | self.assertEqual(f.s, "hello!") 19 | 20 | with self.assertRaises(AttributeError): 21 | f.e 22 | 23 | self.assertEqual(f["f"], 6.0) 24 | self.assertEqual(f["i"], 3) 25 | self.assertEqual(f["s"], "hello!") 26 | 27 | with self.assertRaises(KeyError): 28 | f["e"] 29 | with self.assertRaises(KeyError): 30 | f["cat"] 31 | with self.assertRaises(KeyError): 32 | f["cat"] = 5 33 | 34 | self.assertEqual(len(f), 3) 35 | self.assertEqual(set(f), set(["f", "i", "s"])) 36 | self.assertEqual(set(f.keys()), set(["f", "i", "s"])) 37 | self.assertEqual(set(f.values()), set([6.0, 3, "hello!"])) 38 | self.assertEqual(set(f.items()), set([("f", 6.0), ("i", 3), ("s", "hello!")])) 39 | 40 | f.e = "a" 41 | self.assertEqual(f.e, "a") 42 | self.assertEqual(f["e"], "a") 43 | f["e"] = "b" 44 | self.assertEqual(f.e, "b") 45 | 46 | with self.assertRaises(ValueError): 47 | f.e = "fubar" 48 | 49 | f.f = sys.float_info.max 50 | self.assertEqual(f.f, sys.float_info.max) 51 | f.f = float("inf") 52 | self.assertEqual(f.f, float("inf")) 53 | f.f = None 54 | self.assertEqual(f.f, None) 55 | 56 | class BarObject(FooObject): 57 | x = RESTProperty(str) 58 | 59 | f = BarObject(f="6.0", i="3", s="hello!", x="5") 60 | self.assertEqual(f.f, 6.0) 61 | self.assertEqual(f.i, 3) 62 | self.assertEqual(f.s, "hello!") 63 | self.assertEqual(f.x, "5") 64 | -------------------------------------------------------------------------------- /tests/unit/tools_tests/test_schema.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from tabpy.tabpy_tools.schema import generate_schema 4 | 5 | 6 | class TestSchema(unittest.TestCase): 7 | def test_schema(self): 8 | schema = generate_schema( 9 | input={"x": ["happy", "sad", "neutral"]}, 10 | input_description={"x": "text to analyze"}, 11 | output=[0.98, -0.99, 0], 12 | output_description="scores for input texts", 13 | ) 14 | expected = { 15 | "input": { 16 | "type": "object", 17 | "properties": { 18 | "x": { 19 | "type": "array", 20 | "items": {"type": "string"}, 21 | "description": "text to analyze", 22 | } 23 | }, 24 | "required": ["x"], 25 | }, 26 | "sample": {"x": ["happy", "sad", "neutral"]}, 27 | "output": { 28 | "type": "array", 29 | "items": {"type": "number"}, 30 | "description": "scores for input texts", 31 | }, 32 | } 33 | self.assertEqual(schema, expected) 34 | --------------------------------------------------------------------------------