├── .coveragerc ├── .github ├── CODEOWNERS └── workflows │ ├── End2EndTest.yml │ ├── publish-python.yaml │ ├── semgrep.yml │ ├── snyk-issue.yml │ └── snyk-pr.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── DESCRIPTION.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── deploy.sh ├── scripts ├── decrypt_secret.sh └── parameters.py.gpg ├── setup.py ├── snowflake └── ingest │ ├── __init__.py │ ├── error.py │ ├── errorcode.py │ ├── simple_ingest_manager.py │ ├── utils │ ├── __init__.py │ ├── network.py │ ├── tokentools.py │ └── uris.py │ └── version.py └── tests ├── __init__.py ├── conftest.py ├── data ├── test_file.csv └── test_rsa_key ├── parameters.py.gpg ├── sfctest0_private_key_1.p8 ├── test_security_manager.py ├── test_simple_ingest.py └── test_unit_tokens.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [xml] 2 | output=snowflake-ingest-python-coverage.xml 3 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @snowflakedb/snowpipe 2 | -------------------------------------------------------------------------------- /.github/workflows/End2EndTest.yml: -------------------------------------------------------------------------------- 1 | name: Snowpipe Python SDK Tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: '**' 8 | 9 | jobs: 10 | build: 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | os: [ ubuntu-latest ] 15 | python-version: [ '3.8', '3.9', '3.10' ] 16 | steps: 17 | - name: Checkout Code 18 | uses: actions/checkout@v3 19 | 20 | - name: Install Python 21 | uses: actions/setup-python@v3 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | architecture: 'x64' 25 | 26 | - name: Display Python version 27 | run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);" 28 | 29 | - name: Decrypt profile.json 30 | env: 31 | DECRYPTION_PASSPHRASE: ${{ secrets.PARAMETERS_PY_DECRYPT_PASSPHRASE }} 32 | run: | 33 | ./scripts/decrypt_secret.sh 34 | 35 | - name: Generate Pytest coverage report 36 | run: | 37 | pip install pytest 38 | pip install wheel 39 | pip install pytest-cov 40 | pip install . 41 | pip install snowflake-connector-python 42 | python -m pytest --cov=./ --cov-report=xml --tb=native tests/ 43 | # pytest --cov=./ --cov-report=xml --tb=native tests 44 | 45 | - name: Upload coverage to Codecov 46 | uses: codecov/codecov-action@v4 47 | with: 48 | files: ./snowflake-ingest-python-coverage.xml 49 | name: codecov-snowpipe-python-sdk 50 | fail_ci_if_error: true 51 | verbose: true 52 | token: ${{ secrets.CODECOV_TOKEN }} 53 | -------------------------------------------------------------------------------- /.github/workflows/publish-python.yaml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: write 17 | id-token: write 18 | 19 | jobs: 20 | deploy: 21 | 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - uses: actions/checkout@v3 26 | - name: Set up Python 27 | uses: actions/setup-python@v3 28 | with: 29 | python-version: '3.x' 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install build 34 | - name: Build package 35 | run: python -m build 36 | - name: List artifacts 37 | run: ls ./dist 38 | - name: Install sigstore 39 | run: python -m pip install sigstore 40 | - name: Signing 41 | run: | 42 | for dist in dist/*; do 43 | dist_base="$(basename "${dist}")" 44 | echo "dist: ${dist}" 45 | echo "dist_base: ${dist_base}" 46 | python -m \ 47 | sigstore sign "${dist}" \ 48 | --output-signature "${dist_base}.sig" \ 49 | --output-certificate "${dist_base}.crt" \ 50 | --bundle "${dist_base}.sigstore" 51 | 52 | # Verify using `.sig` `.crt` pair; 53 | python -m \ 54 | sigstore verify identity "${dist}" \ 55 | --signature "${dist_base}.sig" \ 56 | --cert "${dist_base}.crt" \ 57 | --cert-oidc-issuer https://token.actions.githubusercontent.com \ 58 | --cert-identity ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/.github/workflows/publish-python.yaml@${GITHUB_REF} 59 | 60 | # Verify using `.sigstore` bundle; 61 | python -m \ 62 | sigstore verify identity "${dist}" \ 63 | --bundle "${dist_base}.sigstore" \ 64 | --cert-oidc-issuer https://token.actions.githubusercontent.com \ 65 | --cert-identity ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/.github/workflows/publish-python.yaml@${GITHUB_REF} 66 | done 67 | - name: List artifacts after sign 68 | run: ls ./dist 69 | - name: Copy files to release 70 | run: | 71 | gh release upload ${{ github.event.release.tag_name }} *.sigstore 72 | gh release upload ${{ github.event.release.tag_name }} *.sig 73 | gh release upload ${{ github.event.release.tag_name }} *.crt 74 | env: 75 | GITHUB_TOKEN: ${{ github.TOKEN }} 76 | - name: Publish package 77 | uses: pypa/gh-action-pypi-publish@release/v1 78 | with: 79 | user: __token__ 80 | password: ${{ secrets.PYPI_API_TOKEN }} 81 | -------------------------------------------------------------------------------- /.github/workflows/semgrep.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Run semgrep checks 3 | 4 | on: 5 | pull_request: 6 | branches: [master] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | run-semgrep-reusable-workflow: 13 | uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main 14 | secrets: 15 | token: ${{ secrets.SEMGREP_APP_TOKEN }} 16 | 17 | -------------------------------------------------------------------------------- /.github/workflows/snyk-issue.yml: -------------------------------------------------------------------------------- 1 | name: Snyk Issue 2 | 3 | on: 4 | schedule: 5 | - cron: '* */12 * * *' 6 | 7 | permissions: 8 | contents: read 9 | issues: write 10 | pull-requests: write 11 | 12 | concurrency: snyk-issue 13 | 14 | jobs: 15 | snyk: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: checkout action 19 | uses: actions/checkout@v3 20 | with: 21 | repository: snowflakedb/whitesource-actions 22 | token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} 23 | path: whitesource-actions 24 | - name: set-env 25 | run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV 26 | - name: Jira Creation 27 | uses: ./whitesource-actions/snyk-issue 28 | with: 29 | snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }} 30 | snyk_token: ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }} 31 | jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }} 32 | env: 33 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | 35 | -------------------------------------------------------------------------------- /.github/workflows/snyk-pr.yml: -------------------------------------------------------------------------------- 1 | name: snyk-pr 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | 7 | permissions: 8 | contents: read 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | snyk: 14 | runs-on: ubuntu-latest 15 | if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} 16 | steps: 17 | - name: checkout 18 | uses: actions/checkout@v3 19 | with: 20 | ref: ${{ github.event.pull_request.head.ref }} 21 | fetch-depth: 0 22 | 23 | - name: checkout action 24 | uses: actions/checkout@v3 25 | with: 26 | repository: snowflakedb/whitesource-actions 27 | token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} 28 | path: whitesource-actions 29 | 30 | - name: PR 31 | uses: ./whitesource-actions/snyk-pr 32 | env: 33 | PR_TITLE: ${{ github.event.pull_request.title }} 34 | with: 35 | jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }} 36 | gh_token: ${{ secrets.GITHUB_TOKEN }} 37 | amend: false # true if you want the commit to be amended with the JIRA number 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | dist/ 3 | build/ 4 | __pycache__/ 5 | *.egg-info 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv/ 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | 95 | # Rope project settings 96 | 97 | tests/parameters.py 98 | *.xml 99 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - hooks: 3 | - id: secret-scanner 4 | repo: git@github.com:snowflakedb/casec_precommit.git 5 | rev: v1.5 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '3.5' 4 | - '3.6' 5 | - '3.7' 6 | before_install: 7 | - openssl aes-256-cbc -K $encrypted_94d8fbd70a3e_key -iv $encrypted_94d8fbd70a3e_iv -in tests/parameters.py.enc -out tests/parameters.py -d 8 | install: 9 | - "./scripts/travis_install.sh" 10 | script: 11 | - pytest --tb=native --cov-report xml --cov=snowflake.ingest tests 12 | after_success: 13 | - pip install codecov 14 | - codecov -f snowflake-ingest-python-coverage.xml -t 661d6c83-f010-4af1-bf7f-0df1ce3dbc8d -------------------------------------------------------------------------------- /DESCRIPTION.rst: -------------------------------------------------------------------------------- 1 | This package includes the Snowpipe python SDK 2 | 3 | Snowflake Documentation is available at: 4 | https://docs.snowflake.net/ 5 | 6 | Source code is also available at: https://github.com/snowflakedb/snowflake-ingest-python 7 | 8 | Release Notes 9 | ------------------------------------------------------------------------------- 10 | - v1.0.10 (November 14, 2024) 11 | 12 | - Update readme for artifact validation using cosign 13 | 14 | - v1.0.9 (September 10, 2024) 15 | 16 | - Fix casing for RFC-6750 conformity 17 | - Handle unexpected json structure in error payload 18 | 19 | - v1.0.8 (July 03, 2024) 20 | - Update dependency package to newer version (requests) 21 | 22 | - v1.0.7 (January 05, 2024) 23 | 24 | - Pin dependency package to newer version (snowflake-connector-python) 25 | 26 | - v1.0.6 (November 08, 2023) 27 | 28 | - Pin dependency package to newer version (requests) 29 | 30 | - v1.0.5 (November 04, 2022) 31 | 32 | - Pin dependency package to newer versions (requests, snowflake-connector-python) 33 | - Fix pyJwt logic to allow for version >= 2.0.0 34 | 35 | - v1.0.4 (May 11, 2021) 36 | 37 | - Support a special account name style. 38 | 39 | - v1.0.3 (January 11, 2021) 40 | 41 | - Use older version of pyJwt by pinning the version(<2.0.0) 42 | 43 | - v1.0.2 (March 09, 2020) 44 | 45 | - Stop logging JWT token 46 | 47 | - v1.0.1 (November 08, 2019) 48 | 49 | - Replaced Botocore's vendored requests with standalone requests 50 | 51 | - v1.0.0 (September 19, 2018) 52 | 53 | - Fix typing package issue in python 3.5.0 and 3.5.1 54 | - Support key rotation in key pair authentication 55 | 56 | - v0.9.1 (November 9, 2017) 57 | 58 | - Public preview release 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2012-2023 Snowflake Computing, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include LICENSE 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Snowflake Python Ingest Service SDK 2 | =================================== 3 | 4 | 5 | .. image:: http://img.shields.io/:license-Apache%202-brightgreen.svg 6 | :target: http://www.apache.org/licenses/LICENSE-2.0.txt 7 | 8 | .. image:: https://travis-ci.org/snowflakedb/snowflake-ingest-python.svg?branch=master 9 | :target: https://travis-ci.org/snowflakedb/snowflake-ingest-python 10 | 11 | .. image:: https://codecov.io/gh/snowflakedb/snowflake-ingest-python/branch/master/graph/badge.svg 12 | :target: https://codecov.io/gh/snowflakedb/snowflake-ingest-python 13 | 14 | .. image:: https://badge.fury.io/py/snowflake-ingest.svg 15 | :target: https://pypi.python.org/pypi/snowflake-ingest 16 | 17 | The Snowflake Ingest Service SDK allows users to ingest files into their Snowflake data warehouse in a programmatic 18 | fashion via key-pair authentication. Note that this is for Snowpipe only and does not support Snowpipe Streaming. 19 | 20 | Prerequisites 21 | ============= 22 | 23 | Python 3.4+ 24 | ----------- 25 | The Snowflake Ingest SDK requires Python 3.4 or above. Backwards compatibility with older versions of Python 3 26 | or any versions of Python 2 is not planned at this time. 27 | 28 | 29 | A 2048-bit RSA key pair 30 | ----------------------- 31 | Snowflake Authentication for the Ingest Service requires creating a 2048 bit 32 | RSA key pair and, registering the public key with Snowflake. For detailed instructions, 33 | please visit the relevant `Snowflake Documentation Page `_. 34 | 35 | 36 | Furl, PyJWT, Requests, and Cryptography 37 | --------------------------------------- 38 | 39 | Internally, the Snowflake Ingest SDK makes use of `Furl `_, 40 | `PyJWT `_, and `Requests `_. 41 | In addition, the `cryptography `_ is used with PyJWT to sign JWT tokens. 42 | 43 | 44 | Installation 45 | ============ 46 | If you would like to use this sdk, you can install it using python setuptools. 47 | 48 | .. code-block:: bash 49 | 50 | pip install snowflake-ingest 51 | 52 | Usage 53 | ===== 54 | Here is a simple "hello world" example for using ingest sdk. 55 | 56 | .. code-block:: python 57 | 58 | from logging import getLogger 59 | from snowflake.ingest import SimpleIngestManager 60 | from snowflake.ingest import StagedFile 61 | from snowflake.ingest.utils.uris import DEFAULT_SCHEME 62 | from datetime import timedelta 63 | from requests import HTTPError 64 | from cryptography.hazmat.primitives import serialization 65 | from cryptography.hazmat.primitives.serialization import load_pem_private_key 66 | from cryptography.hazmat.backends import default_backend 67 | from cryptography.hazmat.primitives.serialization import Encoding 68 | from cryptography.hazmat.primitives.serialization import PrivateFormat 69 | from cryptography.hazmat.primitives.serialization import NoEncryption 70 | import time 71 | import datetime 72 | import os 73 | import logging 74 | 75 | logging.basicConfig( 76 | filename='/tmp/ingest.log', 77 | level=logging.DEBUG) 78 | logger = getLogger(__name__) 79 | 80 | 81 | with open("./rsa_key.p8", 'rb') as pem_in: 82 | pemlines = pem_in.read() 83 | private_key_obj = load_pem_private_key(pemlines, 84 | os.environ['PRIVATE_KEY_PASSPHRASE'].encode(), 85 | default_backend()) 86 | 87 | private_key_text = private_key_obj.private_bytes( 88 | Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode('utf-8') 89 | # Assume the public key has been registered in Snowflake: 90 | # private key in PEM format 91 | 92 | # List of files in the stage specified in the pipe definition 93 | file_list=['a.csv'] 94 | ingest_manager = SimpleIngestManager(account='testaccount', 95 | host='testaccount.snowflakecomputing.com', 96 | user='ingest_user', 97 | pipe='TESTDB.TESTSCHEMA.TESTPIPE', 98 | private_key=private_key_text) 99 | # List of files, but wrapped into a class 100 | staged_file_list = [] 101 | 102 | for file_name in file_list: 103 | staged_file_list.append(StagedFile(file_name, None)) 104 | 105 | try: 106 | resp = ingest_manager.ingest_files(staged_file_list) 107 | except HTTPError as e: 108 | # HTTP error, may need to retry 109 | logger.error(e) 110 | exit(1) 111 | 112 | # This means Snowflake has received file and will start loading 113 | assert(resp['responseCode'] == 'SUCCESS') 114 | 115 | # Needs to wait for a while to get result in history 116 | while True: 117 | history_resp = ingest_manager.get_history() 118 | 119 | if len(history_resp['files']) > 0: 120 | print('Ingest Report:\n') 121 | print(history_resp) 122 | break 123 | else: 124 | # wait for 20 seconds 125 | time.sleep(20) 126 | 127 | hour = timedelta(hours=1) 128 | date = datetime.datetime.utcnow() - hour 129 | history_range_resp = ingest_manager.get_history_range(date.isoformat() + 'Z') 130 | 131 | print('\nHistory scan report: \n') 132 | print(history_range_resp) 133 | 134 | 135 | Artifact Validation 136 | ===== 137 | Artifacts produced in this repository are signed by Snowflake and can be validated on the client side with the following steps. 138 | 139 | 1. Install cosign following `these instructions `_. 140 | 2. Download the `.whl` from the repository like `pypi `_. 141 | 3. Download the `.crt` and `.sig` files for the version of artifact from the `release page `_. 142 | 4. Validate with cosign. The following command is an example to validate the `.whl` file of version 1.0.9. If valid, a message "Verified OK" should be printed out. 143 | 144 | .. code-block:: bash 145 | 146 | cosign verify-blob snowflake_ingest-1.0.9-py3-none-any.whl \ 147 | --certificate snowflake_ingest-1.0.9-py3-none-any.whl.crt \ 148 | --certificate-identity https://github.com/snowflakedb/snowflake-ingest-python/.github/workflows/publish-python.yaml@refs/tags/v1.0.9 \ 149 | --certificate-oidc-issuer https://token.actions.githubusercontent.com \ 150 | --signature snowflake_ingest-1.0.9-py3-none-any.whl.sig 151 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Upload snowflake-ingest-python Package to PyPI 4 | # 5 | # USAGE ./deploy.sh 6 | 7 | echo $WORKSPACE 8 | 9 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 10 | 11 | echo $THIS_DIR 12 | function upload_package() { 13 | local target_pkg=ingest 14 | 15 | rm -f $THIS_DIR/dist/snowflake_${target_pkg}*.whl || true 16 | rm -f $THIS_DIR/snowflake/${target_pkg}/generated_version.py || true 17 | rm -rf $THIS_DIR/snowflake/${target_pkg}/build || true 18 | rm -f $THIS_DIR/dist/snowflake{_,-}${target_pkg}*.{whl,tar.gz} || true 19 | 20 | python3 setup.py sdist bdist_wheel 21 | 22 | 23 | WHL=$(ls $WORKSPACE/dist/snowflake_${target_pkg}*.whl) 24 | TGZ=$(ls $WORKSPACE/dist/snowflake_${target_pkg}*.tar.gz) 25 | 26 | VIRTUAL_ENV_DIR=$THIS_DIR/upload 27 | echo "****** $WHL ******" 28 | echo 29 | unzip -l $WHL 30 | echo 31 | echo "****** $TGZ ******" 32 | echo 33 | tar tvfz $TGZ 34 | echo "Verify the package contents. DON'T include any test case or data!" 35 | if [[ -z "$JENKINS_URL" ]]; then 36 | # not-jenkins job 37 | read -n1 -p "Are you sure to upload $WHL (y/N)? " 38 | echo 39 | if [[ $REPLY != [yY] ]]; then 40 | log INFO "Good bye!" 41 | exit 0 42 | fi 43 | fi 44 | TWINE_OPTIONS=() 45 | if [[ -n "$TWINE_CONFIG_FILE" ]]; then 46 | TWINE_OPTIONS=("--config-file" "$TWINE_CONFIG_FILE") 47 | fi 48 | # twine register -r pypi $WHL # one time 49 | twine upload ${TWINE_OPTIONS[@]} -r pypi $WHL 50 | twine upload ${TWINE_OPTIONS[@]} -r pypi $TGZ 51 | } 52 | 53 | which python3 54 | python3 --version 55 | 56 | 57 | virtualenv -p python3 release 58 | 59 | source release/bin/activate 60 | pip3 install --upgrade pip 61 | pip3 install -U setuptools_rust setuptools twine 62 | 63 | touch pypirc 64 | #!/bin/bash -ex 65 | export TERM=vt100 66 | 67 | cat >$WORKSPACE/pypirc <=3.0.3", 14 | "furl", 15 | "cryptography", 16 | "requests<=2.32.3"] 17 | 18 | # If we're at version less than 3.4 - fail 19 | if version_info[0] < 3 or version_info[1] < 4: 20 | exit("Unsupported version of Python. Minimum version for the Ingest SDK is 3.4") 21 | 22 | # If we're at version 3.4, backfill the typing library 23 | elif version_info[1] == 4: 24 | DEPENDS.append("typing") 25 | 26 | # Python 3.5.0 and 3.5.1 have incompatible typing modules. Use typing_extensions instead. 27 | elif version_info[1] == 5 and version_info[2] < 2: 28 | DEPENDS.append("typing_extensions") 29 | 30 | here = os.path.abspath(os.path.dirname(__file__)) 31 | 32 | def test_suite(): 33 | """ 34 | Defines the test suite for the snowflake ingest SDK 35 | """ 36 | loader = unittest.TestLoader() 37 | return loader.discover("tests", pattern="test_*.py") 38 | 39 | about = {} 40 | with open(os.path.join(here, 'snowflake', 'ingest', 'version.py'), 41 | mode='r', encoding='utf-8') as f: 42 | exec(f.read(), about) 43 | 44 | __version__ = about['__version__'] 45 | 46 | if 'SF_BUILD_NUMBER' in os.environ: 47 | __version__ += ('.' + str(os.environ['SF_BUILD_NUMBER'])) 48 | 49 | with open(os.path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f: 50 | long_description = f.read() 51 | 52 | setup( 53 | name='snowflake_ingest', 54 | version=__version__, 55 | description='Official SnowflakeDB File Ingest SDK', 56 | long_description=long_description, 57 | author='Snowflake Computing', 58 | author_email='support@snowflake.net', 59 | url='https://www.snowflake.net', 60 | packages=['snowflake.ingest', 61 | 'snowflake.ingest.utils'], 62 | license='Apache', 63 | keywords="snowflake ingest sdk copy loading", 64 | 65 | package_data={ 66 | 'snowflake.ingest':['*.rst', 'LICENSE'] 67 | }, 68 | # From here we describe the package classifiers 69 | classifiers=[ 70 | "Development Status :: 2 - Pre-Alpha", 71 | "Intended Audience :: Developers", 72 | "License :: OSI Approved :: Apache Software License", 73 | "Topic :: Database" 74 | ], 75 | # Now we describe the dependencies 76 | install_requires=DEPENDS, 77 | # At last we set the test suite 78 | test_suite="setup.test_suite" 79 | ) 80 | -------------------------------------------------------------------------------- /snowflake/ingest/__init__.py: -------------------------------------------------------------------------------- 1 | from .simple_ingest_manager import SimpleIngestManager, StagedFile 2 | __all__ = [SimpleIngestManager, StagedFile] 3 | -------------------------------------------------------------------------------- /snowflake/ingest/error.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | from requests import Response 3 | 4 | class IngestResponseError(Exception): 5 | """ 6 | Error thrown when rest request failed 7 | """ 8 | def __init__(self, response: Response): 9 | self.http_error_code = response.status_code 10 | 11 | try: 12 | json_body = response.json() 13 | except ValueError: 14 | self.message = 'Http Error: {}, Message: {}'.format(self.http_error_code, 15 | response.reason) 16 | return 17 | 18 | try: 19 | self.code = json_body[u'code'] 20 | self.success = json_body[u'success'] 21 | self._raw_message = json_body[u'message'] 22 | self.data = json_body[u'data'] 23 | self.message = 'Http Error: {}, Vender Code: {}, Message: {}' \ 24 | .format(self.http_error_code, self.code, self._raw_message) 25 | except KeyError: 26 | self.message = 'Http Error: {}, Message: {}, Body: {}'.format(self.http_error_code, 27 | response.reason, 28 | json_body) 29 | return 30 | 31 | def __str__(self): 32 | return self.message 33 | 34 | 35 | class IngestClientError(Exception): 36 | """ 37 | Error thrown in the client side 38 | """ 39 | def __init__(self, **kwargs): 40 | self.code = kwargs.get('code') 41 | self.message = kwargs.get('message') 42 | 43 | def __str__(self): 44 | return 'Vendor Code: {}, Message: {}'\ 45 | .format(self.code, self.message) 46 | -------------------------------------------------------------------------------- /snowflake/ingest/errorcode.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | 3 | ERR_INVALID_PRIVATE_KEY = 290001 4 | -------------------------------------------------------------------------------- /snowflake/ingest/simple_ingest_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | """ 3 | simple_ingest_manager - Creates a basic ingest manager that synchronously sends 4 | requests to the Snowflake Ingest service 5 | """ 6 | 7 | # This class will manage our tokens 8 | from .utils import SecurityManager 9 | 10 | # This will manage our URL generation 11 | from .utils import URLGenerator 12 | from .utils.network import SnowflakeRestful 13 | from .utils.uris import DEFAULT_HOST_FMT 14 | from .utils.uris import DEFAULT_PORT 15 | from .utils.uris import DEFAULT_SCHEME 16 | from .version import __version__ 17 | 18 | # We use a named tuple to represent remote files 19 | from collections import namedtuple 20 | 21 | # UUID for typing formation 22 | from uuid import UUID 23 | 24 | import sys 25 | import platform 26 | 27 | from logging import getLogger 28 | logger = getLogger(__name__) 29 | 30 | from typing import Dict, Any 31 | try: 32 | from typing import Text 33 | except ImportError: 34 | logger.debug('# Python 3.5.0 and 3.5.1 have incompatible typing modules.', exc_info=True) 35 | from typing_extensions import Text 36 | 37 | # We just need a simple named tuple to represent remote files 38 | StagedFile = namedtuple("StagedFile", ["path", "size"]) 39 | 40 | AUTH_HEADER = "Authorization" # Authorization header name 41 | BEARER_FORMAT = "Bearer {0}" # The format of this bearer 42 | 43 | USER_AGENT_HEADER = "User-Agent" # User-Agent header name 44 | CLIENT_NAME = u"SnowpipePythonSDK" # Don't change! 45 | CLIENT_VERSION = __version__ 46 | PLATFORM = platform.platform() 47 | PYTHON_VERSION = u'.'.join(str(v) for v in sys.version_info[:3]) 48 | SNOWPIPE_SDK_USER_AGENT = \ 49 | u'{name}/{version}/{python_version}/{platform}'.format( 50 | name=CLIENT_NAME, 51 | version=CLIENT_VERSION, 52 | python_version=PYTHON_VERSION, 53 | platform=PLATFORM) 54 | 55 | OK = 200 # Is this Response OK? 56 | 57 | 58 | class SimpleIngestManager(object): 59 | """ 60 | SimpleIngestManager - this class is a simple wrapper around the Snowflake Ingest 61 | Service rest api. It is *synchronous* and as such we will block until we either totally fail to 62 | get a response *or* we successfully hear back from the server. 63 | """ 64 | 65 | def __init__(self, account: Text, user: Text, pipe: Text, private_key: Text, 66 | scheme: Text = DEFAULT_SCHEME, host: Text = None, port: int = DEFAULT_PORT): 67 | """ 68 | Simply instantiates all of our local state 69 | :param account: the name of the account who is loading 70 | :param user: the name of the user who is loading 71 | :param pipe: the name of the pipe which we want to use for ingesting 72 | :param private_key: the private key we use for token signature 73 | """ 74 | self.sec_manager = SecurityManager(account, user, private_key) # Create the token generator 75 | self.url_engine = URLGenerator(scheme=scheme, 76 | host=host if host is not None else DEFAULT_HOST_FMT.format(account), 77 | port=port) 78 | self.pipe = pipe 79 | self._next_begin_mark = None 80 | self.restful = SnowflakeRestful() 81 | 82 | def _get_auth_header(self) -> Dict[Text, Text]: 83 | """ 84 | _get_auth_header - simply method to generate the bearer header for our http requests 85 | :return: A singleton mapping from bearer to token 86 | """ 87 | 88 | token_bearer = BEARER_FORMAT.format(self.sec_manager.get_token()) 89 | return {AUTH_HEADER: token_bearer} 90 | 91 | def _get_user_agent_header(self) -> Dict[Text, Text]: 92 | """ 93 | _get_user_agent_header - method to generate the user-agent header for our http requests 94 | :return: A singleton mapping for user agent and sdk version 95 | """ 96 | return {USER_AGENT_HEADER: SNOWPIPE_SDK_USER_AGENT} 97 | 98 | def _get_headers(self) -> Dict[Text, Text]: 99 | """ 100 | _get_headers - get all required SDK headers to be sent to the service 101 | :return: Array of headers to be sent 102 | """ 103 | headers = self._get_auth_header() 104 | headers.update(self._get_user_agent_header()) 105 | return headers 106 | 107 | def ingest_files(self, staged_files: [StagedFile], request_id: UUID = None) -> Dict[Text, Any]: 108 | """ 109 | ingest_files - Informs Snowflake about the files to be ingested into a table through this pipe 110 | :param staged_files: a list of files we want to ingest 111 | :param request_id: an optional request uuid to label this request 112 | :return: the deserialized response from the service 113 | """ 114 | # Generate the target url 115 | target_url = self.url_engine.make_ingest_url(self.pipe, request_id) 116 | logger.info('Ingest file request url: %s', target_url) 117 | 118 | # Make our message payload 119 | payload = { 120 | "files": [x._asdict() for x in staged_files] 121 | } 122 | 123 | # Send our request! 124 | headers = self._get_headers() 125 | response_body = self.restful.post(target_url, json=payload, headers=headers) 126 | logger.debug('Ingest response: %s', str(response_body)) 127 | 128 | return response_body 129 | 130 | def get_history(self, recent_seconds: int = None, request_id: UUID = None) -> Dict[Text, Any]: 131 | """ 132 | get_history - returns the currently cached ingest history from the service 133 | :param request_id: an optional request UUID to label this 134 | :param recent_seconds: an optional argument that specify the time range that history can be seen 135 | :return: the deserialized response from the service 136 | """ 137 | # generate our history endpoint url 138 | target_url = self.url_engine.make_history_url(self.pipe, recent_seconds, self._next_begin_mark, request_id) 139 | logger.info('Get history request url: %s', target_url) 140 | 141 | # Send out our request! 142 | headers = self._get_headers() 143 | response_body = self.restful.get(target_url, headers=headers) 144 | 145 | self._next_begin_mark = response_body['nextBeginMark'] 146 | 147 | return response_body 148 | 149 | def get_history_range(self, start_time_inclusive: Text, end_time_exclusive: Text = None, 150 | request_id: UUID = None) -> Dict[Text, Any]: 151 | """ 152 | get_history_range - returns the ingest history between two points in time 153 | :param request_id: an optional request UUID to label this 154 | :param start_time_inclusive: Timestamp in ISO-8601 format. Start of the time range to retrieve load history data. 155 | :param end_time_exclusive: Timestamp in ISO-8601 format. End of the time range to retrieve load history data. 156 | If omitted, then CURRENT_TIMESTAMP() is used as the end of the range. 157 | :return: the deserialized response from the service 158 | """ 159 | # generate our history endpoint url 160 | target_url = self.url_engine.make_history_range_url(self.pipe, start_time_inclusive, end_time_exclusive, request_id) 161 | logger.info('Get history range request url: %s', target_url) 162 | 163 | # Send out our request! 164 | headers = self._get_headers() 165 | response = self.restful.get(target_url, headers=headers) 166 | 167 | return response 168 | 169 | -------------------------------------------------------------------------------- /snowflake/ingest/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .tokentools import SecurityManager 2 | from .uris import URLGenerator 3 | 4 | # Forward the Security Manager and URLGenerator 5 | __all__ = [SecurityManager, URLGenerator] -------------------------------------------------------------------------------- /snowflake/ingest/utils/network.py: -------------------------------------------------------------------------------- 1 | # import python connector to inject ocsp check method into requests library 2 | import snowflake.connector 3 | import requests 4 | from requests import Response 5 | import time 6 | from ..error import IngestResponseError 7 | 8 | from logging import getLogger 9 | logger = getLogger(__name__) 10 | 11 | from typing import Dict, Any 12 | try: 13 | from typing import Text 14 | except ImportError: 15 | logger.debug('# Python 3.5.0 and 3.5.1 have incompatible typing modules.', exc_info=True) 16 | from typing_extensions import Text 17 | 18 | # default timeout in seconds for a rest request 19 | DEFAULT_REQUEST_TIMEOUT = 1 * 60 20 | 21 | 22 | class SnowflakeRestful(object): 23 | """ 24 | A simple wrapper over python request library to handle retry 25 | """ 26 | def post(self, url: Text, json: Dict, headers: Dict) -> Dict[Text, Any]: 27 | """ 28 | Http POST request 29 | :param url: request url, 30 | :param json: post request body 31 | :param headers: request headers, authentication etc 32 | :return: response payload 33 | """ 34 | return self._exec_request_with_retry(url=url, method='POST', json=json, headers=headers) 35 | 36 | def get(self, url: Text, headers: Dict) -> Dict[Text, Any]: 37 | """ 38 | Http GET request 39 | :param url: 40 | :param headers: 41 | :return: 42 | """ 43 | return self._exec_request_with_retry(url=url, method='GET', headers=headers) 44 | 45 | def _exec_request_with_retry(self, url: Text, method: Text, headers: Dict = None, json: Dict = None) \ 46 | -> Dict[Text, Any]: 47 | 48 | class RetryCtx(object): 49 | def __init__(self, timeout=None): 50 | self.retry_count = 0 51 | self.total_timeout = timeout 52 | self.next_sleep_time = 1 53 | self._request_start_time = time.time() 54 | 55 | def sleep_time(self): 56 | """ 57 | :return: time in seconds to sleep next time, -1 if should not sleep 58 | """ 59 | if time.time() - self._request_start_time > self.total_timeout: 60 | logger.info('Request timeout reached.') 61 | return -1 62 | else: 63 | # exponential backoff 64 | this_sleep_time = self.next_sleep_time 65 | self.next_sleep_time = this_sleep_time * 2 66 | self.retry_count += 1 67 | logger.info('Retried request. Backoff time %d, Retry count %d', 68 | this_sleep_time, self.retry_count) 69 | return this_sleep_time 70 | 71 | retry_context = RetryCtx(DEFAULT_REQUEST_TIMEOUT) 72 | 73 | while True: 74 | response = self._exec_request(url=url, method=method, headers=headers, json=json) 75 | 76 | if response.ok: 77 | return response.json() 78 | elif self._can_retry(response.status_code): 79 | next_sleep_time = retry_context.sleep_time() 80 | if next_sleep_time > 0: 81 | time.sleep(next_sleep_time) 82 | continue 83 | 84 | raise IngestResponseError(response) 85 | 86 | def _exec_request(self, url: Text, method: Text, headers: Dict = None, json: Dict = None) -> Response: 87 | return requests.request(method=method, 88 | url=url, 89 | headers=headers, 90 | json=json) 91 | 92 | @staticmethod 93 | def _can_retry(http_code): 94 | return http_code == 408 or 500 <= http_code < 600 95 | -------------------------------------------------------------------------------- /snowflake/ingest/utils/tokentools.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | 3 | """ 4 | tokentools.py - provides services for automatically creating and renewing 5 | JWT tokens for authenticating to Snowflake 6 | """ 7 | 8 | from datetime import timedelta, datetime 9 | from logging import getLogger 10 | from cryptography.exceptions import UnsupportedAlgorithm 11 | from cryptography.hazmat.primitives.serialization import load_pem_private_key 12 | from cryptography.hazmat.primitives.serialization import Encoding 13 | from cryptography.hazmat.primitives.serialization import PublicFormat 14 | from cryptography.hazmat.backends import default_backend 15 | from snowflake.connector.util_text import parse_account 16 | from ..error import IngestClientError 17 | from ..errorcode import ERR_INVALID_PRIVATE_KEY 18 | import base64 19 | import hashlib 20 | 21 | import jwt 22 | 23 | logger = getLogger(__name__) 24 | 25 | try: 26 | from typing import Text 27 | except ImportError: 28 | logger.debug('# Python 3.5.0 and 3.5.1 have incompatible typing modules.', exc_info=True) 29 | from typing_extensions import Text 30 | 31 | ISSUER = "iss" 32 | EXPIRE_TIME = "exp" 33 | ISSUE_TIME = "iat" 34 | SUBJECT = "sub" 35 | 36 | 37 | class SecurityManager(object): 38 | """ 39 | Given a private key, username, and account, signs and creates tokens for use in Snowflake Ingest Requests 40 | """ 41 | LIFETIME = timedelta(minutes=59) # The tokens will have a 59 minute lifetime 42 | RENEWAL_DELTA = timedelta(minutes=54) # Tokens will be renewed after 54 minutes 43 | ALGORITHM = "RS256" # Tokens will be generated using RSA with SHA256 44 | 45 | def __init__(self, account: Text, user: Text, private_key: Text, 46 | lifetime: timedelta = LIFETIME, renewal_delay: timedelta = RENEWAL_DELTA): 47 | """ 48 | __init__ creates a security manager with the specified context arguments 49 | :param account: the account in which data is being loaded 50 | :param user: The user who is loading these files 51 | :param private_key: the private key we'll use for signing tokens 52 | :param lifetime: how long this key will live (in minutes) 53 | :param renewal_delay: how long until the security manager should renew the key 54 | """ 55 | 56 | logger.info( 57 | """Creating Security Manager with arguments 58 | account : %s, user : %s, lifetime : %s, renewal_delay : %s""", 59 | account, user, lifetime, renewal_delay) 60 | 61 | # Snowflake account names are canonically in all caps. Also trim account names if contain '.' 62 | self.account = parse_account(account.upper()) 63 | self.user = user.upper() # Snowflake user names are also in all caps by default 64 | self.qualified_username = self.account + "." + self.user # Generate the full user name 65 | self.lifetime = lifetime # the timedelta until our tokens expire 66 | self.renewal_delay = renewal_delay # the timedelta until we renew the token 67 | self.private_key = private_key # stash the private key 68 | self.renew_time = datetime.utcnow() # We need to renew the token NOW 69 | self.token = None # We initially have no token 70 | 71 | def get_token(self) -> Text: 72 | """ 73 | Regenerates the current token if and only if we have exceeded the renewal time 74 | bounds set 75 | :return: the new token 76 | """ 77 | now = datetime.utcnow() # Fetch the current time 78 | 79 | # If the token has expired, or doesn't exist, regenerate it 80 | if self.token is None or self.renew_time <= now: 81 | logger.info("Renewing token because renewal time (%s) is eclipsed by present time (%s)", 82 | self.renew_time, now) 83 | # Calculate the next time we need to renew the token 84 | self.renew_time = now + self.renewal_delay 85 | 86 | public_key_fp = self.calculate_public_key_fingerprint(self.private_key) 87 | 88 | # Create our payload 89 | payload = { 90 | 91 | # The issuer is the public key fingerprint 92 | ISSUER: self.qualified_username + '.' + public_key_fp, 93 | 94 | # subject is user's fully qualified username 95 | SUBJECT: self.qualified_username, 96 | 97 | # The payload was issued at this point it time 98 | ISSUE_TIME: now, 99 | 100 | # The token should no longer be accepted after our lifetime has elapsed 101 | EXPIRE_TIME: now + SecurityManager.LIFETIME 102 | } 103 | 104 | # Regenerate the actual token 105 | self.token = jwt.encode(payload, self.private_key, algorithm=SecurityManager.ALGORITHM) 106 | logger.info("New Token created") 107 | 108 | return self.token.decode('utf-8') if isinstance(self.token, bytes) else self.token 109 | 110 | def calculate_public_key_fingerprint(self, private_key: Text) -> Text: 111 | """ 112 | Given a private key in pem format, return the public key fingerprint 113 | :param private_key: private key string 114 | :return: public key fingerprint 115 | """ 116 | try: 117 | private_key = load_pem_private_key(private_key.encode(), None, default_backend()) 118 | except (ValueError, UnsupportedAlgorithm) as e: 119 | raise IngestClientError( 120 | code=ERR_INVALID_PRIVATE_KEY, 121 | message='Invalid private key. {}'.format(e)) 122 | 123 | # get the raw bytes of public key 124 | public_key_raw = private_key.public_key().public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo) 125 | 126 | # take sha256 on raw bytes and then do base64 encode 127 | sha256hash = hashlib.sha256() 128 | sha256hash.update(public_key_raw) 129 | 130 | public_key_fp = 'SHA256:' + base64.b64encode(sha256hash.digest()).decode('utf-8') 131 | logger.info("Public key fingerprint is %s", public_key_fp) 132 | 133 | return public_key_fp 134 | 135 | def get_account(self) -> Text: 136 | return self.account 137 | -------------------------------------------------------------------------------- /snowflake/ingest/utils/uris.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | 3 | """ 4 | request_builder.py - Builds the URIs and http requests that we send for ingest 5 | requests and history requests 6 | """ 7 | 8 | from uuid import uuid4, UUID 9 | from furl import furl 10 | from logging import getLogger 11 | 12 | # Create a logger for this module 13 | logger = getLogger(__name__) 14 | 15 | try: 16 | from typing import Text 17 | except ImportError: 18 | logger.debug('# Python 3.5.0 and 3.5.1 have incompatible typing modules.', exc_info=True) 19 | from typing_extensions import Text 20 | 21 | # URI Construction constants we use for making a target URL 22 | DEFAULT_HOST_FMT = "{}.snowflakecomputing.com" # by default we target the Snowflake US instance 23 | DEFAULT_PORT = 443 # we also target https by default implying 443 24 | DEFAULT_SCHEME = "https" # we also need to set the scheme to HTTPS 25 | 26 | # Our format strings for the endpoints we need target 27 | INGEST_ENDPOINT_FORMAT = "/v1/data/pipes/{0}/insertFiles" # The template for an ingest request endpoint 28 | HISTORY_ENDPOINT_FORMAT = "/v1/data/pipes/{0}/insertReport" # The template for an ingest history endpoint 29 | HISTORY_SCAN_ENDPOINT_FORMAT = "/v1/data/pipes/{0}/loadHistoryScan" # The template for ingest history range endpoint 30 | 31 | # Parameter used to pass along request UUIDs 32 | REQUEST_ID_PARAMETER = "requestId" 33 | # Parameter that we use for setting the stage for this url 34 | STAGE_PARAMETER = "stage" 35 | # Parameters for insertReport endpoint 36 | RECENT_HISTORY_IN_SECONDS_PARAMETER = 'recentSeconds' 37 | HISTORY_BEGIN_MARK = 'beginMark' 38 | # Parameters for loadHistoryScan endpoint 39 | HISTORY_RANGE_START_INCLUSIVE = 'startTimeInclusive' 40 | HISTORY_RANGE_END_EXCLUSIVE = 'endTimeExclusive' 41 | 42 | # Method to generate an the URL for an ingest request for a given table, and stage 43 | class URLGenerator(object): 44 | """ 45 | URLGenerator - this class handles creating the URLs for a requests to 46 | the Snowflake service 47 | """ 48 | def __init__(self, host: Text, scheme: Text = DEFAULT_SCHEME, port: int = DEFAULT_PORT): 49 | """ 50 | This constructor simply stashes the basic portions of the request URL such that the user 51 | doesn't have to repeatedly provide them 52 | """ 53 | self.scheme = scheme 54 | self.host = host 55 | self.port = port 56 | 57 | def _make_base_url(self, uuid: UUID = None) -> furl: 58 | """ 59 | _makeBaseURL - generates the common base URL for all of our requests 60 | :param uuid: a UUID we want to attach to the 61 | :return: the furl wrapper around our unfinished base url 62 | """ 63 | base = furl() # Create an uninitialized base URI object 64 | base.host = self.host # set the host name 65 | base.port = self.port # set the port number 66 | base.scheme = self.scheme # set the access scheme 67 | 68 | # if we have no uuid to attach to this request, generate one 69 | if uuid is None: 70 | uuid = uuid4() 71 | 72 | # Set the request id parameter uuid 73 | base.args[REQUEST_ID_PARAMETER] = str(uuid) 74 | 75 | return base 76 | 77 | def make_ingest_url(self, pipe: Text, uuid: UUID = None) -> Text: 78 | """ 79 | make_ingest_url - creates a textual representation of the target url we need to hit for 80 | ingesting files 81 | :param pipe: the pipe which we want to use to ingest files (fully qualified) 82 | :param uuid: an optional UUID argument to tag this request 83 | :return: the completed URL 84 | """ 85 | 86 | # Compute the base url 87 | builder = self._make_base_url(uuid) 88 | 89 | # Set the path for the ingest url 90 | builder.path = INGEST_ENDPOINT_FORMAT.format(pipe) 91 | 92 | return builder.url 93 | 94 | def make_history_url(self, pipe: Text, recent_seconds: int = None, 95 | begin_mark: Text = None, uuid: UUID = None) -> Text: 96 | """ 97 | make_history_url - creates a textual representation of the target url we need to hit for 98 | history requests 99 | :param pipe: the pipe for which we want to see the see history 100 | :param uuid: an optional UUID argument to tag this request 101 | :param recent_seconds: an optional argument to specify recent seconds 102 | :param begin_mark: an optional argument used to indicate from which record should next reponse 103 | return 104 | :return: the completed URL 105 | """ 106 | 107 | # Compute the base url 108 | builder = self._make_base_url(uuid) 109 | 110 | # Set the path for the history url 111 | builder.path = HISTORY_ENDPOINT_FORMAT.format(pipe) 112 | 113 | if recent_seconds is not None: 114 | builder.args[RECENT_HISTORY_IN_SECONDS_PARAMETER] = str(recent_seconds) 115 | 116 | if begin_mark is not None: 117 | builder.args[HISTORY_BEGIN_MARK] = str(begin_mark) 118 | 119 | return builder.url 120 | 121 | def make_history_range_url(self, pipe: Text, start_time_inclusive: Text, 122 | end_time_exclusive: Text = None, uuid: UUID = None) -> Text: 123 | """ 124 | make_history_url - creates [M#Ka textual representation of the target url we need to hit for 125 | history range scan requests 126 | :param pipe: the pipe for which we want to see the see history 127 | :param uuid: an optional UUID argument to tag this request 128 | :param start_time_inclusive: Timestamp in ISO-8601 format. Start of the time range to retrieve load history data. 129 | :param end_time_exclusive: Timestamp in ISO-8601 format. End of the time range to retrieve load history data. 130 | If omitted, then CURRENT_TIMESTAMP() is used as the end of the range. 131 | :return: the completed URL 132 | """ 133 | 134 | # Compute the base url 135 | builder = self._make_base_url(uuid) 136 | 137 | # Set the path for the history url 138 | builder.path = HISTORY_SCAN_ENDPOINT_FORMAT.format(pipe) 139 | 140 | builder.args[HISTORY_RANGE_START_INCLUSIVE] = start_time_inclusive 141 | 142 | if end_time_exclusive is not None: 143 | builder.args[HISTORY_RANGE_END_EXCLUSIVE] = end_time_exclusive 144 | 145 | return builder.url 146 | -------------------------------------------------------------------------------- /snowflake/ingest/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. 2 | # Update this for the versions 3 | __version__ = '1.0.10' 4 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowflakedb/snowflake-ingest-python/3fe4db6ffdf3f9310c8c83f84c8335d327db0f0d/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | import pytest 3 | import os 4 | from cryptography.hazmat.backends import default_backend 5 | from cryptography.hazmat.primitives.asymmetric import rsa 6 | from cryptography.hazmat.primitives import serialization 7 | import snowflake.connector 8 | import uuid 9 | from .parameters import CONNECTION_PARAMETERS 10 | from .parameters import PRIVATE_KEY_1_PASSPHRASE 11 | from snowflake.connector.compat import TO_UNICODE 12 | 13 | 14 | if os.getenv('TRAVIS') == 'true': 15 | TEST_SCHEMA = 'TRAVIS_JOB_{0}'.format(os.getenv('TRAVIS_JOB_ID')) 16 | else: 17 | TEST_SCHEMA = 'python_connector_tests_' + TO_UNICODE(uuid.uuid4()).replace( 18 | '-', '_') 19 | 20 | 21 | class TestUtil: 22 | 23 | @staticmethod 24 | def get_data_dir(): 25 | tests_dir = os.path.dirname(os.path.realpath(__file__)) 26 | return os.path.join(tests_dir, "data") 27 | 28 | @staticmethod 29 | def generate_key_pair(): 30 | private_key = rsa.generate_private_key(backend=default_backend(), 31 | public_exponent=65537, 32 | key_size=2048) 33 | 34 | private_key_pem = private_key.private_bytes( 35 | encoding=serialization.Encoding.PEM, 36 | format=serialization.PrivateFormat.PKCS8, 37 | encryption_algorithm=serialization.NoEncryption() 38 | ).decode('utf-8') 39 | 40 | public_key_in_pem = private_key.public_key().public_bytes( 41 | serialization.Encoding.PEM, 42 | serialization.PublicFormat.SubjectPublicKeyInfo).decode('utf-8') 43 | 44 | return private_key_pem, public_key_in_pem 45 | 46 | @staticmethod 47 | def read_private_key(): 48 | """ 49 | Add your own rsa private key(for testing only) in tests directory 50 | Follow instructions here 51 | https://docs.snowflake.net/manuals/user-guide/data-load-snowpipe-rest-gs.html#using-key-pair-authentication 52 | :return: private key pem encoded 53 | """ 54 | tests_dir = os.path.dirname(os.path.realpath(__file__)) 55 | private_key_filename = os.path.join(tests_dir, "sfctest0_private_key_1.p8") 56 | 57 | with open(private_key_filename, "rb") as private_key_file: 58 | private_key = serialization.load_pem_private_key( 59 | private_key_file.read(), 60 | password=PRIVATE_KEY_1_PASSPHRASE.encode('utf-8'), 61 | backend=default_backend() 62 | ) 63 | 64 | private_key_pem = private_key.private_bytes( 65 | encoding=serialization.Encoding.PEM, 66 | format=serialization.PrivateFormat.PKCS8, 67 | encryption_algorithm=serialization.NoEncryption()).decode('utf-8') 68 | 69 | return private_key_pem 70 | 71 | 72 | @pytest.fixture(scope='session', autouse=True) 73 | def init_test_schema(request): 74 | """ 75 | Initializes and Deinitializes the test schema 76 | This is automatically called per test session. 77 | """ 78 | param = get_cnx_param() 79 | with snowflake.connector.connect(**param) as con: 80 | # Uncomment below two lines to test it locally, travis user account 81 | # doesnt have permissions to create db and schema 82 | # con.cursor().execute("CREATE OR REPLACE DATABASE {0}".format(TEST_DB)) 83 | # con.cursor().execute("USE DATABASE {0}".format(TEST_DB)) 84 | con.cursor().execute( 85 | "CREATE SCHEMA IF NOT EXISTS {0}".format(TEST_SCHEMA)) 86 | # con.cursor().execute("USE SCHEMA {0}".format(TEST_SCHEMA)) 87 | 88 | def fin(): 89 | param1 = get_cnx_param() 90 | with snowflake.connector.connect(**param1) as con1: 91 | con1.cursor().execute( 92 | "DROP SCHEMA IF EXISTS {0}".format(TEST_SCHEMA)) 93 | request.addfinalizer(fin) 94 | 95 | 96 | @pytest.fixture() 97 | def test_util(): 98 | return TestUtil 99 | 100 | 101 | @pytest.fixture() 102 | def connection_ctx(request): 103 | param = get_cnx_param() 104 | cnx = snowflake.connector.connect(**param) 105 | 106 | def fin(): 107 | cnx.close() 108 | request.addfinalizer(fin) 109 | 110 | return {'cnx': cnx, 'param': param} 111 | 112 | 113 | def get_cnx_param(): 114 | param = CONNECTION_PARAMETERS 115 | param['schema'] = TEST_SCHEMA 116 | return param 117 | -------------------------------------------------------------------------------- /tests/data/test_file.csv: -------------------------------------------------------------------------------- 1 | 1, hello 2 | 2, hello2 3 | 3, hello3 4 | 4, hello4 -------------------------------------------------------------------------------- /tests/data/test_rsa_key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDP3CkiWRD4gJqz 3 | 977fVzCaofo/49/BUYU3rt8MVJCMpSnHT7aeM95KDbQYiY4s6mPf9cO/6JjHvbmQ 4 | 7xPNTnnhRruA0BWL768kXw5rETijmlA/WIJMbkLgwxtGafZYn3b1o2mqSWKxK690 5 | KynynUv7sydxabjnk4nJVxC4nHAp1VI8X/DawIXbBTzu4ktPmBXKzriRNZjtk+D1 6 | ZIkaIQonP1Wpzse6kg/6wW3OhO5BHwA4fCINwGjBzqr0mmBN5ry9nBIpff48uXNV 7 | 9gVIIgQRzwLgNgWFfnXOHS77+0n0z0BtmUatyeAhREwqGcMoA5aivUatHf5rEbaC 8 | Kjr3WSGNAgMBAAECggEAaEFN5Gt15p5iedfORakuuLB7myYJaaYgwlAUkdOseM6y 9 | iMSDnQ/4832yEgiUZhTeKUvUdeINF0oi0/4GGZi96u8WRsKXvAto9j4zNiJ6HRze 10 | GRekqv82zhMuEAA/zi2VdhkTe5S5SpHVu9eWf5jDrqgqJWlYk9Zdar4fpejZHTF4 11 | M+6EE6eUX76CWoqwUMLAEj/aCEM2qeYHWk1VQyX83HB1jy+BX2cEDeZvxmV7P7mp 12 | ZlZ6gEOdVfhlewZJbC+SWRwm5B+dawscaBRE4pJtiaMV7qOmz17GpfgE/RpJlqVA 13 | fe790HdQdCIKIBeKTpqCyP7iWCrDu3cjoSd44LrpBQKBgQD+0ti3m9fsuNkNDuC+ 14 | dgAXkR8YGvAQe8M9bcmACBFGGiLStCrpSgZwagDuta2mZLZT31lG4KGb5MrOV0zz 15 | Q64YhglRaSHPtqi2ITJHxX9RdHTSj9w6axFWUvfXHbEZDI7E4daMcgQZ6u9dr7VF 16 | DFHchW7OPJjozeokqMG1mpQ3rwKBgQDQ0c/exV8Vu0M7Iteu01rhvJxtWSx0xWv6 17 | B1bgyYOkyWpYVKdIk1LcVAZhgvtbiikJTlXKgwttQMgdygPpTUiwPInB/NCF0oAs 18 | orfpGeBexLFJFvbN3CoNajGmDSHx6RQi9K1V842hjPBA+b6uvEbfUlpyH4AP21PM 19 | EfjNMo9NgwKBgDOO3aJouct/qwrlU7u1jFc4WZ469Q+guuQW7oolF7mjWCBhq7z/ 20 | 6UWdbQrfX38nKWzW5+1bTdeI9y/AoiUmMHdtxzzdlKW+Q2x2UwIKh7QnZ+uih+Ca 21 | ASwCJXs02rxCujBDsXFBMGs+CahfAMIzt+xyYvT/dcDEyPcZ3fesiwipAoGBAIRS 22 | v++BkJxbuuGpVZVSFz/+Xf2oyVQBmkepCPOOnp34iCwLEKobuSEnGZgHATLjnNdp 23 | zVFzsvT7XRQLZGkdcRdEdWL4ykZSuqgOQI40uIo1B8ayB5kxj3BKv8VigwUhVoJE 24 | G+bgW/poLgJuf9eINTzkma3BqkviBvrE1K1rAYXzAoGBAL+98/XzKDyhw0ncJASv 25 | SQ+3+WOoeotDX6AHS5PVDNj/r2wJspA6D82/GIfVi58DexhzgjVYrFi0Fi+00DyG 26 | 3fVnuAeJCdVuqNEpW7N8QZGvvavPOBPJj20m0bcjZ7w0+4fnpT6fjF8Q3JOQTdTa 27 | iLVX9N4wZxsVH9pPVjNWWKLy 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /tests/parameters.py.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowflakedb/snowflake-ingest-python/3fe4db6ffdf3f9310c8c83f84c8335d327db0f0d/tests/parameters.py.gpg -------------------------------------------------------------------------------- /tests/sfctest0_private_key_1.p8: -------------------------------------------------------------------------------- 1 | -----BEGIN ENCRYPTED PRIVATE KEY----- 2 | MIIE6TAbBgkqhkiG9w0BBQMwDgQIznLt1jsMp7sCAggABIIEyJQnQHOL3GzbzBHt 3 | 0dTiVF5Ih+X2on0JoIPb2uMwjG/GRujWLvye8eNdV4N2invavwwu3e2r/hd9w+4Z 4 | pVC73g8z7mfSiNJ+yo6hmoJlSxeM2LEIXukb2QCsoYWufXxD0N/H35aX5lga46cG 5 | MRkIHOlq6Xal8jwEHrv+oKv7bUx3oulcwJ1pe5/xSOBSvZcK7tMfEIo+XehgMCuh 6 | Uzzu62cf/vAX84HiGVOMZUsVvO0A0LB6KPig9ePr4Z25AsZV+/Hq4dIpHzFXe/AY 7 | hUzt5nLzllMGsZOQBWsCW4Ufw3ztPyOGDeoYXJVS1cwVajsg+dBotGlnPbmASvPG 8 | VvRRgyVcKiRHFd7b1MosmuGGp+fuVf74Q7HoBsUsB+mkntJgdF5JXseSney2hpr8 9 | b73apmxwHPAsuWOAqrXHXIVe3IUxo4npuaWxj7ZvVSHqmIjuOyGA5rbeKJkJ6Vzo 10 | ePH7A2TAAr6eQJ8Hhf/GVWrQTy2XJvT6i8TjiRqBwDVk4SDuilT2wHgXUQ/SWphZ 11 | 0ryFh8Vweb+DbjnRfNUNo+EkH+J/aa9SRMr7OyBCVD1hfNXNJbNDH5/K4FIl0JpX 12 | Hz7Y/gwFahEE9GYe+Pi7zETuZqZSQCFORtJV1rFxUQo8BcY6q81QUTmeXR7OhWW7 13 | cc8j1D+JrTwTYf5u2yn6sKHkImY8igEQCD4xtcwX208Y7vD9R7Cuu1jKf902Ickg 14 | ipDwuuVYX5gp/d2rcLe13xMSZXvd5Ps1j2U4PeSX9HwE0yQ1pYe2klfE0t+YUvfd 15 | IBikIIWW4szJgujrXWRQ1zF7UOWQp82/3QkVdty5/9Fn2oK0xmtV9gfLKS0Z5PN3 16 | 0YYb5doJdjYcFly4v4fyTjJ6/tXT6fdrB+8wEqBefVheesac+OngRxd/NWZt4Hij 17 | clika1KZhGKd5HFw5nIvBCqofufhzztd12ryxkM/gtTDsbCNIDtpoXeN+7HjaQ8Y 18 | VXMFCavnQAZ0IcebreNcFXwLsNrmqxd13zXJTWYpyCVnFd8xV9+CEl8AfWlzHgHg 19 | LiicsT+mpkGrpYrwMFEWh0zMR398kA9CC2qjg6cRBYU3/KcpddG6yKwxhEN2ypyV 20 | rsCcIjL900A2rUZzRX7ejw7M1zwG9wta04htQibaM9jImEVxZO+19ZZloty30jCB 21 | imVG9ViHiADAEk67ciB4N7Xtx+aJRGiVFiQ/5Rt9JzaRev1GDtFSkt5eC3AQTUqo 22 | ZCRro/iV015c/lPftlY9jftwXqmWuBMdYrsocZGtFjwsq5jsO4Id/nV+Gboa4bs6 23 | uUcMqlsDVzaExu2ADtkSQAO5lVO8GYHwk0WssRIO3VfCVKXIVylr1M9vp/f7+AIG 24 | YLX6aHRayQmnLwq+YpKTF7PCY20psq4gYXdxyPcUMUKG8o9jyee+Ly0NVYXwfTLu 25 | MGnu41I09kgMgIC/xvMiw4wElGCx3SKYEbVneobqFOex0OjXdvhKZn1A5qkOuQp0 26 | 1vRNAZsAxp824YGhaqAHC56KwbGUtThUVA3eLXJKa5G4RnSW0TgxxGjY3tOHE0GM 27 | F5nx6RQzmpF2EfiCzPptwKaZK6GfypMU2jCRaxcGCaBuSMFh/p7zMvA+B9FldZI2 28 | UVTP0P/Wny2P7ebvqw== 29 | -----END ENCRYPTED PRIVATE KEY----- 30 | -------------------------------------------------------------------------------- /tests/test_security_manager.py: -------------------------------------------------------------------------------- 1 | from snowflake.ingest.utils import SecurityManager 2 | 3 | 4 | def test_same_token(test_util): 5 | """ 6 | Tests that accounts are parsed correctly 7 | """ 8 | expected_account = 'TESTACCOUNT' 9 | 10 | actual_account = 'testaccount' 11 | user = 'testuser' 12 | private_key = '' 13 | sec_manager = SecurityManager(actual_account, user, private_key) 14 | assert sec_manager.get_account() == expected_account 15 | 16 | actual_account = 'testaccount.something' 17 | sec_manager = SecurityManager(actual_account, user, private_key) 18 | assert sec_manager.get_account() == expected_account 19 | -------------------------------------------------------------------------------- /tests/test_simple_ingest.py: -------------------------------------------------------------------------------- 1 | from snowflake.ingest import SimpleIngestManager 2 | from snowflake.ingest import StagedFile 3 | import time 4 | import os 5 | 6 | 7 | def test_simple_ingest(connection_ctx, test_util): 8 | param = connection_ctx['param'] 9 | 10 | pipe_name = '{}.{}.TEST_SIMPLE_INGEST_PIPE'.format( 11 | param['database'], 12 | param['schema']) 13 | 14 | private_key = test_util.read_private_key() 15 | 16 | cur = connection_ctx['cnx'].cursor() 17 | 18 | test_file = os.path.join(test_util.get_data_dir(), 'test_file.csv') 19 | cur.execute('create or replace table TEST_SIMPLE_INGEST_TABLE(c1 number, c2 string)') 20 | cur.execute('create or replace stage TEST_SIMPLE_INGEST_STAGE') 21 | cur.execute('put file://{} @TEST_SIMPLE_INGEST_STAGE'.format(test_file)) 22 | cur.execute('create or replace pipe {0} as copy into TEST_SIMPLE_INGEST_TABLE ' 23 | 'from @TEST_SIMPLE_INGEST_STAGE'.format(pipe_name)) 24 | 25 | ingest_manager = SimpleIngestManager(account=param['account'], 26 | user=param['user'], 27 | private_key=private_key, 28 | pipe=pipe_name, 29 | scheme=param['protocol'], 30 | host=param['host'], 31 | port=param['port']) 32 | 33 | staged_files = [StagedFile('test_file.csv.gz', None)] 34 | 35 | resp = ingest_manager.ingest_files(staged_files) 36 | 37 | assert resp['responseCode'] == 'SUCCESS' 38 | 39 | start_polling_time = time.time() 40 | 41 | while time.time() - start_polling_time < 120: 42 | history_resp = ingest_manager.get_history() 43 | 44 | if len(history_resp['files']) == 1: 45 | assert history_resp['files'][0]['path'] == 'test_file.csv.gz' 46 | return 47 | else: 48 | # wait for 20 seconds 49 | time.sleep(20) 50 | 51 | assert False 52 | 53 | -------------------------------------------------------------------------------- /tests/test_unit_tokens.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. 2 | """ 3 | test_tokens.py - This defines a series of tests to ascertain that we are 4 | capable of renewing JWT tokens 5 | """ 6 | 7 | from snowflake.ingest.utils import SecurityManager 8 | from snowflake.ingest.error import IngestClientError 9 | from snowflake.ingest.errorcode import ERR_INVALID_PRIVATE_KEY 10 | from datetime import timedelta 11 | from time import sleep 12 | import os 13 | import pytest 14 | 15 | 16 | def test_same_token(test_util): 17 | """ 18 | Tests that we aren't immediately regenerating the key after each request 19 | """ 20 | private_key, _ = test_util.generate_key_pair() 21 | sec_man = SecurityManager("testaccount", "snowman", private_key, 22 | renewal_delay=timedelta(seconds=3)) 23 | assert sec_man.get_token() == sec_man.get_token() 24 | 25 | 26 | def test_regenerate_token(test_util): 27 | """ 28 | Tests that the security manager generates new tokens after we 29 | cross the set renewal threshold 30 | """ 31 | private_key, _ = test_util.generate_key_pair() 32 | sec_man = SecurityManager("testaccount", "snowman", private_key, 33 | renewal_delay=timedelta(seconds=3)) 34 | old_token = sec_man.get_token() 35 | sleep(5) 36 | assert old_token != sec_man.get_token() 37 | 38 | 39 | def test_calculate_public_key_fingerprint(test_util): 40 | with open(os.path.join(test_util.get_data_dir(), 'test_rsa_key'), 'r') as key_file: 41 | private_key = key_file.read() 42 | sec_man = SecurityManager("testaccount", "snowman", private_key, 43 | renewal_delay=timedelta(minutes=3)) 44 | public_key_fingerprint = sec_man.calculate_public_key_fingerprint(private_key) 45 | 46 | assert public_key_fingerprint == 'SHA256:QKX8hnXHVAVXp7mLdCAF+vjU2A8RBuRSpgdRjPHhVWY=' 47 | 48 | 49 | def test_invalid_private_key(): 50 | sec_man = SecurityManager("testaccount", "snowman", 'invalid_private_key', 51 | renewal_delay=timedelta(minutes=3)) 52 | with pytest.raises(IngestClientError) as client_error: 53 | sec_man.get_token() 54 | 55 | assert client_error.value.code == ERR_INVALID_PRIVATE_KEY 56 | --------------------------------------------------------------------------------