├── .coveragerc ├── .flake8 ├── .github └── workflows │ ├── publish_pypi.yml │ └── run_tests.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── doc └── images │ ├── switch_versions.png │ └── transfer_accounts.png ├── requirements.dev.txt ├── requirements.test.txt ├── requirements.txt ├── setup.py ├── sf_git.test.conf ├── sf_git ├── __init__.py ├── cache.py ├── cli.py ├── commands.py ├── config.py ├── git_utils.py ├── models.py ├── rest_utils.py ├── sf_git.conf ├── snowsight_auth.py └── worksheets_utils.py └── tests ├── conftest.py ├── data ├── Benchmarking_Tutorials │ ├── .[Tutorial]_Sample_queries_on_TPC-DS_data_metadata.json │ ├── .[Tutorial]_Sample_queries_on_TPC-H_data_metadata.json │ ├── .[Tutorial]_TPC-DS_100TB_Complete_Query_Test_metadata.json │ ├── .[Tutorial]_TPC-DS_10TB_Complete_Query_Test_metadata.json │ ├── [Tutorial]_Sample_queries_on_TPC-DS_data.sql │ ├── [Tutorial]_Sample_queries_on_TPC-H_data.sql │ ├── [Tutorial]_TPC-DS_100TB_Complete_Query_Test.sql │ └── [Tutorial]_TPC-DS_10TB_Complete_Query_Test.sql └── Getting_Started_Tutorials │ ├── .[Template]_Adding_a_user_and_granting_roles_metadata.json │ ├── .[Tutorial]_Using_Python_to_load_and_query_sample_data_metadata.json │ ├── .[Tutorial]_Using_SQL_to_load_and_query_sample_data_metadata.json │ ├── [Template]_Adding_a_user_and_granting_roles.sql │ ├── [Tutorial]_Using_Python_to_load_and_query_sample_data.py │ └── [Tutorial]_Using_SQL_to_load_and_query_sample_data.sql ├── fixtures ├── contents.json ├── entities.json ├── folders.json ├── worksheets.json └── worksheets_update.json ├── test_cache.py ├── test_commands.py ├── test_config.py └── test_worksheets_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = sf_git 3 | omit = 4 | *tests* 5 | sf_git/cli.py 6 | sf_git/snowsight_auth.py 7 | sf_git/rest_utils.py 8 | disable_warnings = no-data-collected 9 | branch = True 10 | 11 | [html] 12 | directory = coverage_html_report -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = D203 3 | exclude = 4 | .git, 5 | __pycache__, 6 | old, 7 | build, 8 | dist 9 | max-complexity = 10 10 | max-line-length = 80 11 | -------------------------------------------------------------------------------- /.github/workflows/publish_pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI and TestPyPi 2 | 3 | on: 4 | push: 5 | branches: 6 | - releases 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | name: Build distribution 12 | if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes 13 | runs-on: ubuntu-latest 14 | environment: production 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: "3.10" 22 | - name: Install pypa/build 23 | run: >- 24 | python3 -m 25 | pip install 26 | setuptools-scm wheel 27 | - name: Build a binary wheel and a source tarball 28 | run: python3 setup.py sdist bdist_wheel 29 | - name: Store the distribution packages 30 | uses: actions/upload-artifact@v3 31 | with: 32 | name: python-package-distributions 33 | path: dist/ 34 | 35 | # publish-to-pypi: 36 | # name: >- 37 | # Publish to PyPI 38 | # if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes 39 | # needs: 40 | # - build 41 | # runs-on: ubuntu-latest 42 | # environment: 43 | # name: production 44 | # url: https://pypi.org/p/sf_git 45 | # permissions: 46 | # id-token: write # IMPORTANT: mandatory for trusted publishing 47 | # 48 | # steps: 49 | # - name: Download all the dists 50 | # uses: actions/download-artifact@v3 51 | # with: 52 | # name: python-package-distributions 53 | # path: dist/ 54 | # - name: Publish to PyPI 55 | # uses: pypa/gh-action-pypi-publish@release/v1 56 | 57 | # github-release: 58 | # name: >- 59 | # Sign the with Sigstore 60 | # and upload them to GitHub Release 61 | # if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' # only publish to PyPI on tag pushes 62 | # needs: 63 | # - publish-to-pypi 64 | # runs-on: ubuntu-latest 65 | # environment: production 66 | # 67 | # permissions: 68 | # contents: write # IMPORTANT: mandatory for making GitHub Releases 69 | # id-token: write # IMPORTANT: mandatory for sigstore 70 | # 71 | # steps: 72 | # - name: Download all the dists 73 | # uses: actions/download-artifact@v3 74 | # with: 75 | # name: python-package-distributions 76 | # path: dist/ 77 | # - name: Sign the dists with Sigstore 78 | # uses: sigstore/gh-action-sigstore-python@v1.2.3 79 | # with: 80 | # inputs: >- 81 | # ./dist/*.tar.gz 82 | # ./dist/*.whl 83 | # - name: Create GitHub Release 84 | # env: 85 | # GITHUB_TOKEN: ${{ github.token }} 86 | # run: >- 87 | # gh release create 88 | # '${{ github.ref_name }}' 89 | # --repo '${{ github.repository }}' 90 | # --notes "" 91 | # - name: Upload artifact signatures to GitHub Release 92 | # env: 93 | # GITHUB_TOKEN: ${{ github.token }} 94 | # # Upload to GitHub Release using the `gh` CLI. 95 | # # `dist/` contains the built packages, and the 96 | # # sigstore-produced signatures and certificates. 97 | # run: >- 98 | # gh release upload 99 | # '${{ github.ref_name }}' dist/** 100 | # --repo '${{ github.repository }}' 101 | 102 | publish-to-testpypi: 103 | name: Publish to TestPyPI 104 | needs: 105 | - build 106 | runs-on: ubuntu-latest 107 | 108 | environment: 109 | name: test 110 | url: https://test.pypi.org/p/sf_git 111 | 112 | permissions: 113 | id-token: write # IMPORTANT: mandatory for trusted publishing 114 | 115 | steps: 116 | - name: Download all the dists 117 | uses: actions/download-artifact@v3 118 | with: 119 | name: python-package-distributions 120 | path: dist/ 121 | - name: Publish to TestPyPI 122 | uses: pypa/gh-action-pypi-publish@release/v1 123 | with: 124 | repository-url: https://test.pypi.org/legacy/ 125 | 126 | -------------------------------------------------------------------------------- /.github/workflows/run_tests.yml: -------------------------------------------------------------------------------- 1 | name: Run Unit Test via Pytest 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'sf_git/**.py' 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | environment: pytest 15 | strategy: 16 | matrix: 17 | python-version: ["3.10"] 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 29 | if [ -f requirements.test.txt ]; then pip install -r requirements.test.txt; fi 30 | - name: set git default branch to main 31 | run: | 32 | git config --global init.defaultBranch main 33 | - name: Lint with Ruff 34 | run: | 35 | pip install flake8 36 | flake8 sf_git 37 | continue-on-error: true 38 | - name: Test with pytest 39 | run: | 40 | coverage run -m pytest tests -v -s 41 | - name: Generate Coverage Report 42 | run: | 43 | REPORT="$(coverage report -m | tail -1)" 44 | COV_TOTAL="$(echo $REPORT | tail -c 5 | sed 's/ //g' | sed 's/%//g')" 45 | echo "COVERAGE=$COV_TOTAL" >> $GITHUB_ENV 46 | # - name: Upload coverage data 47 | # uses: actions/upload-artifact@v3 48 | # with: 49 | # name: "coverage-data" 50 | # path: .coverage.* 51 | # if-no-files-found: ignore 52 | # - name: Extract branch name 53 | # shell: bash 54 | # run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT 55 | # id: extract_branch 56 | - name: Format branch name for cov badge 57 | shell: bash 58 | run: | 59 | BRANCH_NAME="$(echo ${GITHUB_REF_NAME} | sed 's/\//_/g')" 60 | echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_ENV 61 | - name: Create badge 62 | uses: schneegans/dynamic-badges-action@v1.7.0 63 | with: 64 | auth: ${{ secrets.GIST_TOKEN }} 65 | gistID: e2c293d7db07bee70d2845387cb133ff 66 | filename: sf_git_${{ env.BRANCH_NAME }}_cov_badge.json 67 | label: sf_git coverage 68 | message: ${{ env.COVERAGE }}% 69 | minColorRange: 50 70 | maxColorRange: 90 71 | valColorRange: ${{ env.COVERAGE }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | version.txt 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | .idea/ 162 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 tdambrin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include sf_git.conf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | sfgit logo 3 |

4 | 5 | ![image](https://img.shields.io/badge/Python-FFD43B?style=for-the-badge&logo=python&logoColor=blue) 6 | [![image](https://img.shields.io/badge/Gmail-D14836?style=for-the-badge&logo=gmail&logoColor=white)](mailto:thomas.dambrin@gmail.com?subject=[GitHub]%20Snowflake%20Git%20Versioning) 7 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 8 | ![Coverage Badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/tdambrin/e2c293d7db07bee70d2845387cb133ff/raw/sf_git_main_cov_badge.json) 9 | 10 | 11 | 12 | # Snowflake Git 13 | 14 | 🆕 Git now supported in Snowflake, see the [official documentation](https://docs.snowflake.com/en/developer-guide/git/git-overview). 15 | 16 | # Worksheet Versioning 17 | 18 | Inspired by a snowflake developers maintained [repository](https://github.com/Snowflake-Labs/sfsnowsightextensions). 19 | 20 | ### Git integration 21 | 22 | The extension is designed to **apply git versioning on worksheets while developing on Snowsight, fully taking advantage of its functionalities**.\ 23 | The following workflow is advised : 24 | 1. [Start session] Upload worksheet from local branch to a user Snowsight workspace 25 | 2. Test, update and validate on Snowsight 26 | 3. [End session] Update local branch with modified Snowsight worksheets 27 | 28 | ## Install 29 | 30 | Entry points are accessible through a CLI once the package is installed. 31 | To install it, please follow the following steps : 32 | 33 | ```bash 34 | # [Optional: Python virtual environement] 35 | $ pyenv virtualenv 3.10.4 sf 36 | $ pyenv activate sf 37 | 38 | # [Mandatory: Pip install] 39 | $ pip install -U pip 40 | $ pip install sf_git==1.4.2 41 | 42 | # [Check your installation] 43 | $ sfgit --version 44 | # [Should result in:] 45 | # sfgit, version 1.4.2 46 | 47 | 48 | # [Check your installation] 49 | $ sfgit --help 50 | 51 | # [Should result in:] 52 | # Usage: sfgit [OPTIONS] COMMAND [ARGS]... 53 | # 54 | # Options: 55 | # --help Show this message and exit. 56 | # 57 | # Commands: 58 | # commit Commit Snowsight worksheets to Git repository. 59 | # config Configure sfgit for easier version control. 60 | # fetch Fetch worksheets from user Snowsight account and store them in... 61 | # init Initialize a git repository and set it as the sfgit versioning... 62 | # push Upload locally stored worksheets to Snowsight user workspace. 63 | ``` 64 | 65 | Commands have been created to **import/export (respectively fetch/push) snowsight worksheets to/from local**. 66 | 67 | ## Configure your git 68 | 69 | > **Warning** 70 | > A git repository is necessary to manage worksheets. You can either use an existing one 71 | > or create a new one. 72 | 73 | To apply versioning to your worksheets, you need to **configure Git information** 74 | through the config command. 75 | 76 | First, set git repository to be used: 77 | 78 | ```bash 79 | # if you want to use an existing git repository 80 | $ sfgit config --git-repo 81 | 82 | # if you want to create a new one 83 | $ sfgit init -p 84 | ``` 85 | 86 | Then, set a location to save your worksheets within this git repository: 87 | ```bash 88 | $ sfgit config --save-dir 89 | ``` 90 | 91 | ## Authenticate 92 | Currently, only authentication mode supported is the credentials (PWD) mode. 93 | 94 | > :warning: The single sign-on (SSO) will be fixed. 95 | 96 | Commands requiring Snowsight authentication all have options to provide at command time. 97 | If you don't want to manually input them everytime, you can set them at Python/Virtual environement level with : 98 | 99 | 100 | ```bash 101 | $ sfgit config --account 102 | $ sfgit config --username 103 | $ sfgit config --password # unnecessary for SSO authentication mode 104 | ``` 105 | 106 | ### Account ID 107 | 108 | > [!WARNING] 109 | > `The account ID to be configured is in the .. format.` 110 | 111 | If you are unsure about how to retrieve it for your snowflake account, you can run this query: 112 | 113 | ```sql 114 | SHOW REGIONS; 115 | 116 | WITH 117 | SF_REGIONS AS (SELECT * FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()))), 118 | INFOS AS (SELECT CURRENT_REGION() AS CR, CURRENT_ACCOUNT() AS CA) 119 | SELECT CONCAT( 120 | LOWER(INFOS.CA), 121 | '.', 122 | SF_REGIONS."region", 123 | '.', 124 | SF_REGIONS."cloud" 125 | ) AS account_id 126 | FROM INFOS LEFT JOIN SF_REGIONS ON INFOS.CR = SF_REGIONS."snowflake_region"; 127 | 128 | ``` 129 | 130 | Unfortunately, the _region_ is not always the same in the _SHOW REGIONS_ ouput. Please check and adapt the format comforming to the official [documentation](https://docs.snowflake.com/en/user-guide/admin-account-identifier#non-vps-account-locator-formats-by-cloud-platform-and-region). 131 | For example, eastus2 for Azure should actually be east-us-2. 132 | 133 | 134 | ## Use 135 | 136 | **Import user worksheet locally** : 137 | ```bash 138 | $ sfgit fetch --auth-mode PWD 139 | ``` 140 | 141 | **Import user worksheet locally (using command time args)** : 142 | ```bash 143 | $ sfgit fetch --username tdambrin --account-id my_account.west-europe.azure -p mysecret -am PWD 144 | ``` 145 | 146 | **See what changed for only your worksheets in the git** : 147 | ```bash 148 | $ sfgit diff 149 | ``` 150 | 151 | **Commit you worksheets** (or through git commands for more flexibility) : 152 | ```bash 153 | $ sfgit commit --branch master -m "Initial worksheet commit" 154 | ``` 155 | 156 | **Export user worksheets to Snowsight** 157 | ```bash 158 | $ sfgit push --auth-mode PWD --branch master 159 | ``` 160 | 161 | ## Be creative 162 | 163 | Use the package to fit your use case, versioning is a way to do many things. 164 | 165 | ### Switch versions 166 | 167 |
168 | Switch versions 169 |
170 | 171 | 172 | ### Transfer worksheets to another account 173 | 174 | ![Transfer accounts](./doc/images/transfer_accounts.png) 175 | 176 | ## Policies 177 | Feedbacks and contributions are greatly appreciated. This package was made to ease every day life for Snowflake 178 | developers and promote version control as much as possible. 179 | 180 | For questions, please feel free to reach out [by email](mailto:thomas.dambrin@gmail.com?subject=[GitHub]%20Snowflake%20Git%20Versioning). 181 | -------------------------------------------------------------------------------- /doc/images/switch_versions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tdambrin/sf_git/36ab52b6b1451817badfdbe8a8de47f7ecd4c41b/doc/images/switch_versions.png -------------------------------------------------------------------------------- /doc/images/transfer_accounts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tdambrin/sf_git/36ab52b6b1451817badfdbe8a8de47f7ecd4c41b/doc/images/transfer_accounts.png -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | black 2 | isort 3 | flake8> -------------------------------------------------------------------------------- /requirements.test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-mock 3 | pytest-ordering 4 | requests-mock -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pip>=21.3.1 2 | setuptools>=56 3 | pandas 4 | coverage 5 | python-dotenv 6 | click==7.1.2 7 | deepdiff 8 | gitpython>=3.1 9 | requests 10 | urllib3 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | description = """ 4 | Python package to manipulate Snowsight worksheets 5 | and easily apply Git versioning on it. 6 | """ 7 | 8 | with open("README.md", "r") as f: 9 | long_description = f.read() 10 | 11 | with open("requirements.txt", "r") as f: 12 | requirements = f.read() 13 | 14 | setup( 15 | name="sf_git", 16 | version="1.4.2", 17 | setup_requires=["setuptools_scm"], 18 | author="Thomas Dambrin", 19 | author_email="thomas.dambrin@gmail.com", 20 | url="https://github.com/tdambrin/sf_git", 21 | license="MIT", 22 | description=description, 23 | packages=find_packages(), 24 | install_requires=requirements, 25 | keywords=["python", "snowflake", "git"], 26 | classifiers=[ 27 | "Intended Audience :: Developers", 28 | "Programming Language :: Python :: 3", 29 | "Operating System :: MacOS :: MacOS X", 30 | "Operating System :: Unix", 31 | "Operating System :: Microsoft :: Windows", 32 | ], 33 | # include_package_data: to install data from MANIFEST.in 34 | include_package_data=True, 35 | zip_safe=False, 36 | # all functions @cli.command() decorated in sf_git/cli.py 37 | entry_points={"console_scripts": ["sfgit = sf_git.cli:cli"]}, 38 | scripts=[], 39 | long_description=long_description, 40 | long_description_content_type="text/markdown", 41 | ) 42 | -------------------------------------------------------------------------------- /sf_git.test.conf: -------------------------------------------------------------------------------- 1 | SNOWFLAKE_VERSIONING_REPO='./tmp/sf_git_test' 2 | WORKSHEETS_PATH='./tmp/sf_git_test/data' 3 | SF_ACCOUNT_ID='fake.west-europe.azure' 4 | SF_LOGIN_NAME='fake' 5 | SF_PWD='fake' 6 | -------------------------------------------------------------------------------- /sf_git/__init__.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from dotenv import load_dotenv 4 | 5 | __version__ = "1.4.2" 6 | 7 | HERE = Path(__file__).parent 8 | DOTENV_PATH: Path = HERE / "sf_git.conf" 9 | 10 | if DOTENV_PATH.is_file(): 11 | print(f"loading dotenv from {DOTENV_PATH}") 12 | load_dotenv(dotenv_path=DOTENV_PATH) 13 | -------------------------------------------------------------------------------- /sf_git/cache.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | from pathlib import Path 5 | from typing import List, Optional, Union 6 | import git 7 | 8 | import sf_git.config as config 9 | from sf_git.models import Worksheet, WorksheetError 10 | from sf_git.git_utils import get_tracked_files, get_blobs_content 11 | 12 | 13 | def save_worksheets_to_cache(worksheets: List[Worksheet]): 14 | """ 15 | Save worksheets to cache. Git is not involved here. 16 | 17 | For each worksheet, two files are created/overriden: 18 | - ._metadata.json (worksheet info) 19 | - .sql or .py (worksheet content) 20 | 21 | :param worksheets: list of worksheets to save 22 | """ 23 | 24 | print(f"[Worksheets] Saving to {config.GLOBAL_CONFIG.worksheets_path}") 25 | if not os.path.exists(config.GLOBAL_CONFIG.worksheets_path): 26 | os.makedirs(config.GLOBAL_CONFIG.worksheets_path, exist_ok=True) 27 | 28 | for ws in worksheets: 29 | ws_name = re.sub(r"[ :/]", "_", ws.name) 30 | extension = "py" if ws.content_type == "python" else "sql" 31 | if ws.folder_name: 32 | folder_name = re.sub(r"[ :/]", "_", ws.folder_name) 33 | file_name = f"{folder_name}/{ws_name}.{extension}" 34 | worksheet_metadata_file_name = ( 35 | f"{folder_name}/.{ws_name}_metadata.json" 36 | ) 37 | 38 | # create folder if not exists 39 | if not os.path.exists( 40 | config.GLOBAL_CONFIG.worksheets_path / folder_name 41 | ): 42 | os.mkdir(config.GLOBAL_CONFIG.worksheets_path / folder_name) 43 | else: 44 | file_name = f"{ws_name}.{extension}" 45 | worksheet_metadata_file_name = f".{ws_name}_metadata.json" 46 | 47 | with open(config.GLOBAL_CONFIG.worksheets_path / file_name, "w", encoding="utf-8") as f: 48 | f.write(ws.content) 49 | ws_metadata = { 50 | "name": ws.name, 51 | "_id": ws._id, 52 | "folder_name": ws.folder_name, 53 | "folder_id": ws.folder_id, 54 | "content_type": ws.content_type, 55 | } 56 | with open( 57 | config.GLOBAL_CONFIG.worksheets_path 58 | / worksheet_metadata_file_name, 59 | "w", 60 | encoding="utf-8", 61 | ) as f: 62 | f.write(json.dumps(ws_metadata)) 63 | print("[Worksheets] Saved") 64 | 65 | 66 | def load_worksheets_from_cache( 67 | repo: git.Repo, 68 | branch_name: Optional[str] = None, 69 | only_folder: Optional[Union[str, Path]] = None, 70 | ) -> List[Worksheet]: 71 | """ 72 | Load worksheets from cache. 73 | 74 | :param repo: Git repository as it only considers tracked files 75 | :param branch_name: name of git branch to get files from 76 | :param only_folder: to get only worksheets in that folder 77 | 78 | :return: list of tracked worksheet objects 79 | """ 80 | 81 | print(f"[Worksheets] Loading from {config.GLOBAL_CONFIG.worksheets_path}") 82 | if not os.path.exists(config.GLOBAL_CONFIG.worksheets_path): 83 | raise WorksheetError( 84 | "Could not retrieve worksheets from cache. " 85 | f"The folder {config.GLOBAL_CONFIG.worksheets_path} does not exist" 86 | ) 87 | 88 | tracked_files = [ 89 | f 90 | for f in get_tracked_files( 91 | repo, config.GLOBAL_CONFIG.worksheets_path, branch_name 92 | ) 93 | ] 94 | 95 | # filter on worksheet files 96 | ws_metadata_files = [ 97 | f for f in tracked_files if f.name.endswith("_metadata.json") 98 | ] 99 | if len(ws_metadata_files) == 0: 100 | return [] 101 | 102 | # map to worksheet objects 103 | worksheets = [] 104 | metadata_contents = get_blobs_content(ws_metadata_files) 105 | 106 | for wsf in metadata_contents.values(): 107 | ws_metadata = json.loads(wsf) 108 | if only_folder and ws_metadata["folder_name"] != only_folder: 109 | continue 110 | current_ws = Worksheet( 111 | ws_metadata["_id"], 112 | ws_metadata["name"], 113 | ws_metadata["folder_id"], 114 | ws_metadata["folder_name"], 115 | content_type=ws_metadata.get("content_type", "sql"), 116 | ) 117 | extension = "py" if current_ws.content_type == "python" else "sql" 118 | content_filename = re.sub( 119 | r"[ :/]", "_", f"{ws_metadata['name']}.{extension}" 120 | ) 121 | 122 | try: 123 | content_blob = next( 124 | f for f in tracked_files if f.name == content_filename 125 | ) 126 | except StopIteration: 127 | tracked_files = [f.name for f in tracked_files] 128 | print( 129 | f"{content_filename} not found in {tracked_files}" 130 | ) 131 | return [] 132 | 133 | ws_content_as_dict = get_blobs_content([content_blob]) 134 | ws_content = list(ws_content_as_dict.values())[0] 135 | current_ws.content = ws_content 136 | worksheets.append(current_ws) 137 | 138 | return worksheets 139 | -------------------------------------------------------------------------------- /sf_git/cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | import sf_git.config as config 4 | import sf_git.commands 5 | 6 | 7 | @click.group() 8 | def cli(): 9 | pass 10 | 11 | 12 | @click.group() 13 | @click.version_option(sf_git.__version__) 14 | @click.pass_context 15 | def cli(ctx): 16 | pass 17 | 18 | 19 | @click.command("init") 20 | @click.option( 21 | "--path", 22 | "-p", 23 | type=str, 24 | help="Absolute path of Git repository to be created.", 25 | default=".", 26 | show_default=True, 27 | ) 28 | @click.option( 29 | "--mkdir", 30 | help="""(Flag) If provided, the repository directory will be created 31 | if it doesn't exist""", 32 | is_flag=True, 33 | default=False, 34 | show_default=True, 35 | ) 36 | def init(path: str, mkdir: bool): 37 | """ 38 | Initialize a git repository and set it as the sfgit versioning repository. 39 | """ 40 | 41 | sf_git.commands.init_repo_procedure(path=path, mkdir=mkdir) 42 | 43 | 44 | @click.command("config") 45 | @click.option( 46 | "--get", 47 | help="If provided, print the current value for following config option", 48 | type=str, 49 | 50 | ) 51 | @click.option( 52 | "--git-repo", 53 | "-r", 54 | type=str, 55 | help="Absolute path of Git repository used for versioning", 56 | ) 57 | @click.option( 58 | "--save-dir", 59 | "-d", 60 | type=str, 61 | help="""Absolute path of directory to save worksheets in. 62 | Must be included in fit repository. Created if does not exist. 63 | """, 64 | ) 65 | @click.option( 66 | "--account", 67 | "-a", 68 | type=str, 69 | help=""" 70 | Default account for Snowsight authentication. 71 | """, 72 | ) 73 | @click.option( 74 | "--username", 75 | "-u", 76 | type=str, 77 | help=""" 78 | Default username for Snowsight authentication. 79 | """, 80 | ) 81 | @click.option( 82 | "--password", 83 | "-p", 84 | type=str, 85 | help=""" 86 | Default password for Snowsight authentication. 87 | """, 88 | ) 89 | def config_repo( 90 | get: str, 91 | git_repo: str, 92 | save_dir: str, 93 | account: str, 94 | username: str, 95 | password: str, 96 | ): 97 | """ 98 | Configure sfgit for easier version control. 99 | Git_repo configuration is mandatory. 100 | """ 101 | 102 | if get: 103 | sf_git.commands.get_config_repo_procedure(get, click.echo) 104 | else: 105 | sf_git.commands.set_config_repo_procedure( 106 | git_repo=git_repo, 107 | save_dir=save_dir, 108 | account=account, 109 | username=username, 110 | password=password, 111 | logger=click.echo, 112 | ) 113 | 114 | 115 | @click.command("fetch") 116 | @click.option( 117 | "--username", 118 | "-u", 119 | type=str, 120 | help="Snowflake user", 121 | ) 122 | @click.option("--account-id", "-a", type=str, help="Snowflake Account Id") 123 | @click.option( 124 | "--auth-mode", 125 | "-am", 126 | type=str, 127 | help="Authentication Mode. Currently supports PWD (Default) and SSO.", 128 | default="PWD", 129 | show_default=True, 130 | ) 131 | @click.option("--password", "-p", type=str, help="Snowflake password") 132 | @click.option( 133 | "--store/--no-store", 134 | "-l", 135 | help="(Flag) Whether to store the worksheets or just display them.", 136 | default=True, 137 | show_default=True, 138 | ) 139 | @click.option( 140 | "--only-folder", 141 | "-only", 142 | type=str, 143 | help="Only fetch worksheets with given folder name", 144 | ) 145 | def fetch_worksheets( 146 | username: str, 147 | account_id: str, 148 | auth_mode: str, 149 | password: str, 150 | store: bool, 151 | only_folder: str, 152 | ): 153 | """ 154 | Fetch worksheets from user Snowsight account and store them in cache. 155 | """ 156 | 157 | username = username or config.GLOBAL_CONFIG.sf_login_name 158 | account_id = account_id or config.GLOBAL_CONFIG.sf_account_id 159 | password = password or config.GLOBAL_CONFIG.sf_pwd 160 | 161 | sf_git.commands.fetch_worksheets_procedure( 162 | username=username, 163 | account_id=account_id, 164 | auth_mode=auth_mode, 165 | password=password, 166 | store=store, 167 | only_folder=only_folder, 168 | logger=click.echo, 169 | ) 170 | 171 | 172 | @click.command("commit") 173 | @click.option( 174 | "--branch", 175 | "-b", 176 | type=str, 177 | help="Branch to commit to. Default is current.", 178 | ) 179 | @click.option("--message", "-m", type=str, help="Commit message") 180 | def commit( 181 | branch: str, 182 | message: str, 183 | ): 184 | """ 185 | Commit Snowsight worksheets to Git repository. 186 | """ 187 | 188 | sf_git.commands.commit_procedure( 189 | branch=branch, message=message, logger=click.echo 190 | ) 191 | 192 | 193 | @click.command("push") 194 | @click.option("--username", "-u", type=str, help="Snowflake user") 195 | @click.option( 196 | "--account-id", 197 | "-a", 198 | type=str, 199 | help="Snowflake Account Id", 200 | ) 201 | @click.option( 202 | "--auth-mode", 203 | "-am", 204 | type=str, 205 | help="Authentication Mode. Currently supports PWD (Default) and SSO.", 206 | default="PWD", 207 | show_default=True, 208 | ) 209 | @click.option("--password", "-p", type=str, help="Snowflake password") 210 | @click.option( 211 | "--branch", 212 | "-b", 213 | type=str, 214 | help="Branch to commit to. Default is current.", 215 | ) 216 | @click.option( 217 | "--only-folder", 218 | "-only", 219 | type=str, 220 | help="Only push worksheets with given folder name", 221 | ) 222 | def push_worksheets( 223 | username: str, 224 | account_id: str, 225 | auth_mode: str, 226 | password: str, 227 | branch: str, 228 | only_folder: str, 229 | ): 230 | """ 231 | Upload locally stored worksheets to Snowsight user workspace. 232 | The local directory containing worksheets data is named {username}. 233 | More flexibility to come. 234 | """ 235 | 236 | username = username or config.GLOBAL_CONFIG.sf_login_name 237 | account_id = account_id or config.GLOBAL_CONFIG.sf_account_id 238 | password = password or config.GLOBAL_CONFIG.sf_pwd 239 | 240 | sf_git.commands.push_worksheets_procedure( 241 | username=username, 242 | account_id=account_id, 243 | auth_mode=auth_mode, 244 | password=password, 245 | branch=branch, 246 | only_folder=only_folder, 247 | logger=click.echo, 248 | ) 249 | 250 | 251 | @click.command("diff") 252 | def diff(): 253 | """ 254 | Displays unstaged changes on worksheets 255 | """ 256 | 257 | sf_git.commands.diff_procedure(logger=click.echo) 258 | 259 | 260 | @click.group() 261 | @click.version_option(sf_git.__version__) 262 | @click.pass_context 263 | def cli(ctx): 264 | pass 265 | 266 | 267 | cli.add_command(init) 268 | cli.add_command(config_repo) 269 | cli.add_command(fetch_worksheets) 270 | cli.add_command(commit) 271 | cli.add_command(push_worksheets) 272 | cli.add_command(diff) 273 | 274 | if __name__ == "__main__": 275 | cli() 276 | -------------------------------------------------------------------------------- /sf_git/commands.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | from typing import Callable, List 4 | from pathlib import Path, WindowsPath 5 | import git 6 | import dotenv 7 | 8 | from click import UsageError 9 | 10 | import sf_git.config as config 11 | from sf_git.cache import load_worksheets_from_cache 12 | from sf_git.snowsight_auth import authenticate_to_snowsight 13 | from sf_git.models import AuthenticationMode, SnowflakeGitError, Worksheet 14 | from sf_git.worksheets_utils import get_worksheets as sf_get_worksheets 15 | from sf_git.worksheets_utils import ( 16 | print_worksheets, 17 | upload_to_snowsight, 18 | ) 19 | from sf_git.git_utils import diff 20 | from sf_git import DOTENV_PATH 21 | 22 | 23 | def init_repo_procedure(path: str, mkdir: bool = False) -> str: 24 | """ 25 | Initialize a git repository. 26 | 27 | :param path: absolute or relative path to git repository root 28 | :param mkdir: create the repository if it doesn't exist 29 | 30 | :return: new value of versioning repo in sf_git config 31 | """ 32 | p_constructor = WindowsPath if platform.system() == "Windows" else Path 33 | abs_path = p_constructor(path).absolute() 34 | if not os.path.exists(abs_path) and not mkdir: 35 | raise UsageError( 36 | f"[Init] {abs_path} does not exist." 37 | "\nPlease provide a path towards an existing directory" 38 | " or add the --mkdir option to create it." 39 | ) 40 | 41 | # create repository 42 | git.Repo.init(path, mkdir=mkdir) 43 | 44 | # set as sf_git versioning repository 45 | dotenv.set_key( 46 | DOTENV_PATH, 47 | key_to_set="SNOWFLAKE_VERSIONING_REPO", 48 | value_to_set=str(abs_path), 49 | ) 50 | 51 | return dotenv.get_key(DOTENV_PATH, "SNOWFLAKE_VERSIONING_REPO") 52 | 53 | 54 | def get_config_repo_procedure(key: str, logger: Callable = print) -> str: 55 | """ 56 | Get sf_git config value. 57 | 58 | :param key: key to retrieve value for 59 | :param logger: logging function e.g. print 60 | 61 | :returns: sf_git config value 62 | """ 63 | 64 | dotenv_config = dotenv.dotenv_values(DOTENV_PATH) 65 | if key not in dotenv_config.keys(): 66 | raise UsageError( 67 | f"[Config] {key} does not exist.\n" 68 | "Config values to be retrieved are " 69 | f"{list(dotenv_config.keys())}" 70 | ) 71 | 72 | logger(dotenv_config[key]) 73 | 74 | return dotenv_config[key] 75 | 76 | 77 | def set_config_repo_procedure( 78 | git_repo: str = None, 79 | save_dir: str = None, 80 | account: str = None, 81 | username: str = None, 82 | password: str = None, 83 | logger: Callable = print, 84 | ) -> dict: 85 | """ 86 | Set sf_git config values. 87 | 88 | :param git_repo: if provided, set the git repository path 89 | :param save_dir: if provided, set the worksheet directory path 90 | :param account: if provided, set the account id 91 | :param username: if provided, set the user login name 92 | :param password: if provided, set the user password 93 | :param logger: logging function e.g. print 94 | 95 | :returns: dict with newly set keys and their values 96 | """ 97 | 98 | updates = {} 99 | 100 | # check repositories 101 | if git_repo: 102 | repo_path = Path(git_repo).absolute() 103 | if not os.path.exists(repo_path): 104 | raise UsageError( 105 | f"[Config] {git_repo} does not exist." 106 | "Please provide path towards an existing git repository" 107 | " or create a new one with sfgit init." 108 | ) 109 | 110 | if save_dir: 111 | save_dir = Path(save_dir).absolute() 112 | # get git_repo 113 | repo_path = ( 114 | Path(git_repo).absolute() 115 | if git_repo 116 | else config.GLOBAL_CONFIG.repo_path 117 | ) 118 | if repo_path not in save_dir.parents and repo_path != save_dir: 119 | raise UsageError( 120 | "[Config] " 121 | f"{save_dir} is not a subdirectory of {repo_path}.\n" 122 | "Please provide a saving directory within the git repository." 123 | ) 124 | 125 | if git_repo: 126 | dotenv.set_key( 127 | DOTENV_PATH, 128 | key_to_set="SNOWFLAKE_VERSIONING_REPO", 129 | value_to_set=str(git_repo), 130 | ) 131 | logger(f"Set SNOWFLAKE_VERSIONING_REPO to {str(git_repo)}") 132 | updates["SNOWFLAKE_VERSIONING_REPO"] = dotenv.get_key( 133 | DOTENV_PATH, "SNOWFLAKE_VERSIONING_REPO" 134 | ) 135 | 136 | if save_dir: 137 | dotenv.set_key( 138 | DOTENV_PATH, 139 | key_to_set="WORKSHEETS_PATH", 140 | value_to_set=str(save_dir), 141 | ) 142 | logger(f"Set WORKSHEETS_PATH to {str(save_dir)}") 143 | updates["WORKSHEETS_PATH"] = dotenv.get_key( 144 | DOTENV_PATH, "WORKSHEETS_PATH" 145 | ) 146 | 147 | if account: 148 | dotenv.set_key( 149 | DOTENV_PATH, 150 | key_to_set="SF_ACCOUNT_ID", 151 | value_to_set=account, 152 | ) 153 | logger(f"Set SF_ACCOUNT_ID to {account}") 154 | updates["SF_ACCOUNT_ID"] = dotenv.get_key(DOTENV_PATH, "SF_ACCOUNT_ID") 155 | 156 | if username: 157 | dotenv.set_key( 158 | DOTENV_PATH, 159 | key_to_set="SF_LOGIN_NAME", 160 | value_to_set=username, 161 | ) 162 | logger(f"Set SF_LOGIN_NAME to {username}") 163 | updates["SF_LOGIN_NAME"] = dotenv.get_key(DOTENV_PATH, "SF_LOGIN_NAME") 164 | 165 | if password: 166 | dotenv.set_key( 167 | DOTENV_PATH, 168 | key_to_set="SF_PWD", 169 | value_to_set=password, 170 | ) 171 | logger("Set SF_PWD to provided password.") 172 | updates["SF_PWD"] = "*" * len(password) 173 | 174 | return updates 175 | 176 | 177 | def fetch_worksheets_procedure( 178 | username: str, 179 | account_id: str, 180 | auth_mode: str = None, 181 | password: str = None, 182 | only_folder: str = None, 183 | store: bool = True, 184 | logger: Callable = print, 185 | ) -> List[Worksheet]: 186 | """ 187 | Fetch worksheets from Snowsight. 188 | 189 | :param username: username to authenticate 190 | :param account_id: account id to authenticate 191 | :param auth_mode: authentication mode, supported are PWD (default) and SSO 192 | :param password: password to authenticate (not required for SSO) 193 | :param only_folder: name of folder if only fetch a specific folder from Snowsight 194 | :param store: (flag) save worksheets locally in configured worksheet directory 195 | :param logger: logging function e.g. print 196 | 197 | :returns: list of fetched worksheets 198 | """ # noqa: E501 199 | 200 | # Get auth parameters 201 | logger(" ## Authenticating to Snowsight ##") 202 | 203 | if auth_mode: 204 | if auth_mode == "SSO": 205 | auth_mode = AuthenticationMode.SSO 206 | password = None 207 | elif auth_mode == "PWD": 208 | auth_mode = AuthenticationMode.PWD 209 | if not password: 210 | raise UsageError( 211 | "No password provided for PWD authentication mode." 212 | "Please provide one." 213 | ) 214 | else: 215 | raise UsageError(f"{auth_mode} is not supported.") 216 | else: # default 217 | auth_mode = AuthenticationMode.PWD 218 | if not password: 219 | raise UsageError( 220 | "No password provided for PWD authentication mode." 221 | "Please provide one." 222 | ) 223 | logger(f" ## Password authentication with username={username} ##") 224 | auth_context = authenticate_to_snowsight( 225 | account_id, username, password, auth_mode=auth_mode 226 | ) 227 | 228 | if auth_context.snowsight_token != "": 229 | logger(f" ## Authenticated as {auth_context.username}##") 230 | else: 231 | logger(" ## Authentication failed ##") 232 | exit(1) 233 | 234 | logger(" ## Getting worksheets ##") 235 | worksheets = sf_get_worksheets( 236 | auth_context, store_to_cache=store, only_folder=only_folder 237 | ) 238 | 239 | logger("## Got worksheets ##") 240 | print_worksheets(worksheets, logger=logger) 241 | 242 | if store and worksheets: 243 | worksheet_path = dotenv.get_key(DOTENV_PATH, "WORKSHEETS_PATH") 244 | logger(f"## Worksheets saved to {worksheet_path} ##") 245 | 246 | return worksheets 247 | 248 | 249 | def commit_procedure(branch: str, message: str, logger: Callable) -> str: 250 | """ 251 | Commits all worksheets in worksheet directory 252 | 253 | :param branch: name of branch to commit to, defaults to active one 254 | :param message: commit message 255 | :param logger: logging function e.g. print 256 | 257 | :returns: commit sha 258 | """ 259 | # Get git repo 260 | try: 261 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path) 262 | except git.InvalidGitRepositoryError as exc: 263 | raise SnowflakeGitError( 264 | "Could not find Git Repository here : " 265 | f"{config.GLOBAL_CONFIG.repo_path}" 266 | ) from exc 267 | 268 | # Get git branch 269 | if branch: 270 | available_branches = {b.name: b for b in repo.branches} 271 | if branch not in available_branches.keys(): 272 | raise UsageError(f"Could not find branch {branch}") 273 | branch = available_branches[branch] 274 | 275 | # and checkout if necessary 276 | if repo.active_branch.name != branch.name: 277 | repo.head.reference = branch 278 | else: 279 | branch = repo.active_branch 280 | 281 | # Add worksheets to staged files 282 | repo.index.add(config.GLOBAL_CONFIG.worksheets_path) 283 | 284 | # Commit 285 | if message: 286 | commit_message = message 287 | else: 288 | commit_message = "[UPDATE] Snowflake worksheets" 289 | 290 | c = repo.index.commit(message=commit_message) 291 | 292 | logger(f"## Committed worksheets to branch {branch.name}") 293 | 294 | return c.hexsha 295 | 296 | 297 | def push_worksheets_procedure( 298 | username: str, 299 | account_id: str, 300 | auth_mode: str = None, 301 | password: str = None, 302 | branch: str = None, 303 | only_folder: str = None, 304 | logger: Callable = print, 305 | ) -> dict: 306 | """ 307 | Push committed worksheet to Snowsight. 308 | 309 | :param username: username to authenticate 310 | :param account_id: account id to authenticate 311 | :param auth_mode: authentication mode, supported are PWD (default) and SSO 312 | :param password: password to authenticate (not required for SSO) 313 | :param only_folder: name of folder if only push a specific folder to Snowsight 314 | :param branch: branch to get worksheets from 315 | :param logger: logging function e.g. print 316 | 317 | :returns: upload report with success and errors per worksheet 318 | """ # noqa: E501 319 | 320 | # Get auth parameters 321 | logger(" ## Authenticating to Snowsight ##") 322 | if not username: 323 | raise SnowflakeGitError("No username to authenticate with.") 324 | if not account_id: 325 | raise SnowflakeGitError("No account to authenticate with.") 326 | 327 | if auth_mode: 328 | if auth_mode == "SSO": 329 | auth_mode = AuthenticationMode.SSO 330 | password = None 331 | elif auth_mode == "PWD": 332 | auth_mode = AuthenticationMode.PWD 333 | if not password: 334 | raise UsageError( 335 | "No password provided for PWD authentication mode." 336 | "Please provide one." 337 | ) 338 | else: 339 | raise UsageError(f"{auth_mode} is not supported.") 340 | else: # default 341 | auth_mode = AuthenticationMode.PWD 342 | if not password: 343 | raise UsageError( 344 | "No password provided for PWD authentication mode." 345 | "Please provide one." 346 | ) 347 | 348 | auth_context = authenticate_to_snowsight( 349 | account_id, username, password, auth_mode=auth_mode 350 | ) 351 | 352 | if auth_context.snowsight_token != "": 353 | logger(f" ## Authenticated as {auth_context.login_name}##") 354 | else: 355 | logger(" ## Authentication failed ##") 356 | exit(1) 357 | 358 | # Get file content from git utils 359 | try: 360 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path) 361 | except git.InvalidGitRepositoryError as exc: 362 | raise SnowflakeGitError( 363 | "Could not find Git Repository here : " 364 | f"{config.GLOBAL_CONFIG.repo_path}" 365 | ) from exc 366 | 367 | logger(f" ## Getting worksheets from cache for user {username} ##") 368 | worksheets = load_worksheets_from_cache( 369 | repo=repo, 370 | branch_name=branch, 371 | only_folder=only_folder, 372 | ) 373 | 374 | logger("## Got worksheets ##") 375 | print_worksheets(worksheets, logger=logger) 376 | 377 | logger("## Uploading to SnowSight ##") 378 | upload_report = upload_to_snowsight(auth_context, worksheets) 379 | worksheet_errors = upload_report["errors"] 380 | logger("## Uploaded to SnowSight ##") 381 | 382 | if worksheet_errors: 383 | logger("Errors happened for the following worksheets :") 384 | for err in worksheet_errors: 385 | logger( 386 | f"Name : {err['name']} " 387 | f"| Error type : {err['error'].snowsight_error}" 388 | ) 389 | 390 | return upload_report 391 | 392 | 393 | def diff_procedure(logger: Callable = print) -> str: 394 | """ 395 | Displays unstaged changes on worksheets for configured repository and worksheets path. 396 | 397 | :param logger: logging function e.g. print 398 | 399 | :returns: diff output as a string 400 | """ # noqa: E501 401 | 402 | # Get configuration 403 | try: 404 | repo = git.Repo(config.GLOBAL_CONFIG.repo_path) 405 | except git.InvalidGitRepositoryError as exc: 406 | raise SnowflakeGitError( 407 | "Could not find Git Repository here : " 408 | f"{config.GLOBAL_CONFIG.repo_path}" 409 | ) from exc 410 | 411 | worksheets_path = config.GLOBAL_CONFIG.worksheets_path 412 | if not os.path.exists(worksheets_path): 413 | raise SnowflakeGitError( 414 | "Worksheets path is not set or the folder doesn't exist. " 415 | "Please set it or create it (manually or with sfgit fetch" 416 | ) 417 | 418 | diff_output = diff( 419 | repo, subdirectory=worksheets_path, file_extensions=["py", "sql"] 420 | ) 421 | logger(diff_output) 422 | 423 | return diff_output 424 | -------------------------------------------------------------------------------- /sf_git/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path, PosixPath, WindowsPath 3 | import platform 4 | from dataclasses import dataclass 5 | from typing import Union 6 | 7 | 8 | @dataclass 9 | class Config: 10 | """Package configuration, git, Snowflake and Filesystem""" 11 | 12 | sf_main_app_url: str = "https://app.snowflake.com" 13 | repo_path: Union[PosixPath, WindowsPath] = None 14 | worksheets_path: Union[PosixPath, WindowsPath] = None 15 | sf_account_id: str = None 16 | sf_login_name: str = None 17 | sf_pwd: str = None 18 | 19 | def __post_init__(self): 20 | # make paths windows if necessary 21 | if platform.system() == "Windows": 22 | if self.repo_path: 23 | self.repo_path = WindowsPath(self.repo_path) 24 | if self.worksheets_path: 25 | self.worksheets_path = WindowsPath(self.worksheets_path) 26 | self.repo_path = self.repo_path.absolute() 27 | self.worksheets_path = self.worksheets_path.absolute() 28 | 29 | 30 | GLOBAL_CONFIG = Config( 31 | repo_path=Path(os.environ["SNOWFLAKE_VERSIONING_REPO"]).absolute(), 32 | worksheets_path=Path( 33 | os.environ.get("WORKSHEETS_PATH") or "/tmp/snowflake_worksheets" 34 | ).absolute(), 35 | sf_account_id=os.environ.get("SF_ACCOUNT_ID"), 36 | sf_login_name=os.environ.get("SF_LOGIN_NAME"), 37 | sf_pwd=os.environ.get("SF_PWD"), 38 | ) 39 | -------------------------------------------------------------------------------- /sf_git/git_utils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import List, Type, Union, Dict, Optional 3 | import re 4 | 5 | import git 6 | from git.objects.blob import Blob 7 | from git.objects.tree import Tree 8 | from git.repo.base import Repo 9 | 10 | from sf_git.models import SnowflakeGitError 11 | 12 | 13 | def get_tracked_files( 14 | repo: Repo, folder: Path, branch_name: Optional[str] = None 15 | ) -> List[Union[Type[Blob], Type[Tree]]]: 16 | """ 17 | Get all git tracked files in a folder inside a git repo 18 | 19 | :param repo: git repository tracking files 20 | :param folder: name of folder inside git repo to only load from 21 | :param branch_name: branch to consider, default is active branch 22 | 23 | :returns: list of blobs (file) and tree (dir) 24 | """ 25 | 26 | # check that folder is in git 27 | repo_wd = Path(repo.working_dir) 28 | if repo_wd not in folder.parents: 29 | return [] 30 | 31 | # retrieve branch, active one by default 32 | if branch_name is not None: 33 | available_branches = {b.name: b for b in repo.branches} 34 | if branch_name not in available_branches.keys(): 35 | raise SnowflakeGitError( 36 | f"Unable to retrieve branch {branch_name}" 37 | f" in Repository {repo.working_dir}." 38 | "Please check that the branch name is correct" 39 | ) 40 | branch = available_branches[branch_name] 41 | else: 42 | branch = repo.active_branch 43 | 44 | # get to folder by folder names 45 | repo_objects = branch.commit.tree.traverse() 46 | try: 47 | folder_tree = next( 48 | obj for obj in repo_objects if obj.abspath == str(folder) 49 | ) 50 | except StopIteration as exc: 51 | raise SnowflakeGitError( 52 | f"Unable to retrieve folder {str(folder)}" 53 | f" in Repository {repo.working_dir} and branch {branch.name}." 54 | "Please check that the files you are looking for are committed" 55 | ) from exc 56 | 57 | # get files 58 | tracked_files = folder_tree.traverse() 59 | return tracked_files 60 | 61 | 62 | def get_blobs_content(blobs: List[Blob]) -> Dict[str, bytes]: 63 | """ 64 | Get blob contents and return as {blob_name: content}. 65 | Allow to get file content from git trees easily. 66 | """ 67 | 68 | contents = { 69 | b.name: b.data_stream.read() for b in blobs if isinstance(b, Blob) 70 | } 71 | return contents 72 | 73 | 74 | def diff( 75 | repo: git.Repo, 76 | subdirectory: Union[str, Path] = None, 77 | file_extensions: Union[str, List[str]] = None, 78 | ) -> str: 79 | """ 80 | Get git diff output with subdirectory and file extension filters 81 | 82 | :param repo: git repository 83 | :param subdirectory: only on files within this subdirectory 84 | :param file_extensions: only match files with these extensions 85 | 86 | :returns: str, git diff output 87 | """ 88 | # Check input 89 | if subdirectory and isinstance(subdirectory, str): 90 | subdirectory = Path(subdirectory) 91 | if file_extensions and isinstance(file_extensions, str): 92 | file_extensions = [file_extensions] 93 | 94 | # Get blobs 95 | search_path = ( 96 | subdirectory 97 | if subdirectory 98 | else Path(repo.git.rev_parse("--show-toplevel")) 99 | ) 100 | globs = [] 101 | if not file_extensions: 102 | globs.extend(search_path.glob("**/*")) 103 | else: 104 | for extension in file_extensions: 105 | globs.extend(list(search_path.glob(f"**/*.{extension}"))) 106 | 107 | # Get git diff output 108 | if not globs: 109 | return "" 110 | 111 | diff_output = repo.git.diff(globs) 112 | 113 | # Add coloring 114 | diff_output = re.sub( 115 | r"^(\++)(.*?)$", 116 | "\033[32m\\1\\2\033[0m", 117 | diff_output, 118 | flags=re.MULTILINE, 119 | ) 120 | diff_output = re.sub( 121 | r"^(-+)(.*?)$", 122 | "\033[31m\\1\\2\033[0m", 123 | diff_output, 124 | flags=re.MULTILINE, 125 | ) 126 | 127 | return diff_output 128 | -------------------------------------------------------------------------------- /sf_git/models.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from enum import Enum 3 | from typing import Any 4 | 5 | 6 | class AuthenticationMode(Enum): 7 | SSO = "SSO" 8 | PWD = "PWD" 9 | 10 | 11 | class AuthenticationError(Exception): 12 | def __init__(self, *args): 13 | if args: 14 | self.message = args[0] 15 | else: 16 | self.message = None 17 | 18 | def __str__(self): 19 | if self.message: 20 | return f"AuthenticationError: {self.message}" 21 | else: 22 | return "AuthenticationError: failed with no more information." 23 | 24 | 25 | @dataclass 26 | class AuthenticationContext: 27 | """To store authentication result information""" 28 | 29 | account: str = "" 30 | account_name: str = "" 31 | account_url: str = "" 32 | auth_code_challenge: str = "" 33 | auth_code_challenge_method: str = "" 34 | auth_originator: str = "" 35 | auth_redirect_uri: str = "" 36 | auth_session_token: str = "" 37 | app_server_url: str = "" 38 | client_id: str = "" 39 | cookies: Any = None 40 | csrf: str = "" 41 | login_name: str = "" 42 | main_app_url: str = "" 43 | master_token: str = "" 44 | oauth_nonce: str = "" 45 | organization_id: str = "" 46 | region: str = "" 47 | server_version: str = "" 48 | snowsight_token: dict = field(default_factory=dict) 49 | username: str = "" 50 | window_id: str = "" 51 | 52 | 53 | class SnowsightError(Enum): 54 | PERMISSION = "PERMISSION" 55 | UNKNOWN = "UNKNOWN" 56 | 57 | 58 | class WorksheetError(Exception): 59 | def __init__(self, *args): 60 | if args: 61 | self.message = args[0] 62 | if len(args) > 1: 63 | self.snowsight_error = SnowsightError.PERMISSION 64 | else: 65 | self.snowsight_error = SnowsightError.UNKNOWN 66 | else: 67 | self.message = None 68 | self.snowsight_error = SnowsightError.UNKNOWN 69 | 70 | def __str__(self): 71 | if self.message: 72 | return ( 73 | f"WorksheetError: {self.message}." 74 | f" Error type is {self.snowsight_error}" 75 | ) 76 | else: 77 | return "WorksheetError: no more information provided" 78 | 79 | 80 | class Worksheet: 81 | def __init__( 82 | self, 83 | _id: str, 84 | name: str, 85 | folder_id: str, 86 | folder_name: str, 87 | content: str = "To be filled", 88 | content_type: str = "sql", 89 | ): 90 | self._id = _id 91 | self.name = name 92 | self.folder_id = folder_id 93 | self.folder_name = folder_name 94 | self.content = content 95 | self.content_type = content_type 96 | 97 | def to_dict(self): 98 | return { 99 | "_id": self._id, 100 | "name": self.name, 101 | "folder_id": self.folder_id, 102 | "folder_name": self.folder_name, 103 | "content": self.content, 104 | "content_type": self.content_type, 105 | } 106 | 107 | 108 | class Folder: 109 | def __init__(self, _id: str, name: str, worksheets=None): 110 | if worksheets is None: 111 | worksheets = [] 112 | self._id = _id 113 | self.name = name 114 | self.worksheets = worksheets 115 | 116 | def to_dict(self): 117 | return { 118 | "_id": self._id, 119 | "name": self.name, 120 | "worksheets": self.worksheets, 121 | } 122 | 123 | 124 | class SnowflakeGitError(Exception): 125 | """ 126 | Custom Exception for sf_git specific raised exceptions 127 | """ 128 | 129 | def __init__(self, *args): 130 | if args: 131 | self.message = args[0] 132 | else: 133 | self.message = "Not provided" 134 | 135 | def __str__(self): 136 | if self.message: 137 | return f"SnowflakeGitError: {self.message}." 138 | 139 | return "SnowflakeGitError: no more information provided" 140 | -------------------------------------------------------------------------------- /sf_git/rest_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import platform 3 | import re 4 | import socket 5 | import subprocess 6 | import time 7 | 8 | import requests 9 | 10 | 11 | def api_post( 12 | base_url: str, 13 | rest_api_url: str, 14 | accept_header: str, 15 | request_body: str = None, 16 | request_type_header: str = None, 17 | snowflake_context: str = None, 18 | referer: str = None, 19 | csrf: str = None, 20 | cookies: requests.cookies.RequestsCookieJar = None, # noqa 21 | allow_redirect: bool = False, 22 | as_obj: bool = False, 23 | ): 24 | start_time = time.time() 25 | 26 | if not base_url.endswith("/"): 27 | base_url += "/" 28 | 29 | try: 30 | session = requests.Session() 31 | session.verify = False 32 | session.max_redirects = 20 33 | 34 | headers = requests.models.CaseInsensitiveDict() 35 | if referer: 36 | headers["Referer"] = referer 37 | if snowflake_context: 38 | headers["x-snowflake-context"] = snowflake_context 39 | if cookies: 40 | session.cookies.update(cookies) 41 | if csrf: 42 | headers["X-CSRF-Token"] = csrf 43 | 44 | headers["Accept"] = accept_header 45 | 46 | content_type = request_type_header 47 | headers["Content-Type"] = content_type 48 | 49 | response = session.post( 50 | base_url + rest_api_url, 51 | headers=headers, 52 | data=request_body, 53 | timeout=60, 54 | allow_redirects=allow_redirect, 55 | ) 56 | 57 | if response.status_code < 400: 58 | result_string = response.text or "" 59 | logging.info( 60 | f"POST {base_url}/{rest_api_url} returned " 61 | f"{response.status_code}" 62 | f"({response.reason})\nRequest Headers:\n{headers}\nCookies:\n" 63 | f"{len(result_string)}:\n{result_string}" 64 | ) 65 | if as_obj: 66 | return response 67 | return result_string 68 | else: 69 | result_string = response.text or "" 70 | cookies_list = response.headers.get("Set-Cookie") 71 | if result_string: 72 | logging.error( 73 | f"POST {base_url}/{rest_api_url}" 74 | f" returned {response.status_code}" 75 | f" ({response.reason})\n" 76 | f"Request Headers:\n{headers}\n" 77 | f"Cookies:\n{cookies_list}\n" 78 | f"Request:\n{request_body}\n" 79 | f"Response Length {len(result_string)}" 80 | ) 81 | else: 82 | logging.error( 83 | f"POST {base_url}/{rest_api_url} " 84 | f"returned {response.status_code}\n" 85 | f"Request Headers:\n{headers}\nCookies:\n{cookies_list}\n" 86 | f"Request:\n{request_body}" 87 | ) 88 | if response.status_code in (401, 403): 89 | logging.warning( 90 | f"POST {base_url}/{rest_api_url} " 91 | f"returned {response.status_code} ({response.reason})," 92 | f" Request:\n{request_body}" 93 | ) 94 | 95 | response.raise_for_status() 96 | except requests.exceptions.RequestException as ex: 97 | logging.error( 98 | f"POST {base_url}/{rest_api_url} " 99 | f"threw {ex.__class__.__name__} ({ex})" 100 | ) 101 | logging.error(ex) 102 | 103 | logging.error( 104 | f"POST {base_url}/{rest_api_url} " 105 | f"threw {ex.__class__.__name__} ({ex})" 106 | ) 107 | 108 | return "" 109 | finally: 110 | elapsed_time = time.time() - start_time 111 | logging.info(f"POST {base_url}/{rest_api_url} took {elapsed_time:.2f}") 112 | 113 | 114 | def api_get( 115 | base_url: str, 116 | rest_api_url: str, 117 | accept_header: str, 118 | snowflake_context: str = None, 119 | referer: str = None, 120 | csrf: str = None, 121 | cookies: requests.cookies.RequestsCookieJar = None, # noqa 122 | allow_redirect: bool = True, 123 | as_obj: bool = False, 124 | ): 125 | start_time = time.time() 126 | 127 | if not base_url.endswith("/"): 128 | base_url += "/" 129 | try: 130 | session = requests.Session() 131 | session.verify = False 132 | session.max_redirects = 20 133 | 134 | headers = requests.models.CaseInsensitiveDict() 135 | if referer: 136 | headers["Referer"] = referer 137 | if snowflake_context: 138 | headers["x-snowflake-context"] = snowflake_context 139 | if cookies: 140 | session.cookies.update(cookies) 141 | if csrf: 142 | headers["X-CSRF-Token"] = csrf 143 | 144 | headers["Accept"] = accept_header 145 | 146 | response = session.get( 147 | base_url + rest_api_url, 148 | headers=headers, 149 | timeout=60, 150 | allow_redirects=allow_redirect, 151 | ) 152 | 153 | if response.status_code < 400: 154 | result_string = response.text or "" 155 | logging.info( 156 | f"POST {base_url}/{rest_api_url} returned " 157 | f"{response.status_code}" 158 | f"({response.reason})\nRequest Headers:\n{headers}\nCookies:\n" 159 | f"{len(result_string)}:\n{result_string}" 160 | ) 161 | if as_obj: 162 | return response 163 | return result_string 164 | else: 165 | result_string = response.text or "" 166 | cookies_list = response.headers.get("Set-Cookie") 167 | if result_string: 168 | logging.error( 169 | f"GET {base_url}/{rest_api_url}" 170 | f" returned {response.status_code}" 171 | f" ({response.reason})\n" 172 | f"Request Headers:\n{headers}\n" 173 | f"Cookies:\n{cookies_list}\n" 174 | f"Response Length {len(result_string)}" 175 | ) 176 | else: 177 | logging.error( 178 | f"GET {base_url}/{rest_api_url} " 179 | f"returned {response.status_code}\n" 180 | f"Request Headers:\n{headers}\nCookies:\n{cookies_list}\n" 181 | ) 182 | if response.status_code in (401, 403): 183 | logging.warning( 184 | f"GET {base_url}/{rest_api_url} " 185 | f"returned {response.status_code} ({response.reason})," 186 | ) 187 | 188 | response.raise_for_status() 189 | except requests.exceptions.RequestException as ex: 190 | logging.error( 191 | f"GET {base_url}/{rest_api_url} " 192 | f"threw {ex.__class__.__name__} ({ex})" 193 | ) 194 | logging.error(ex) 195 | 196 | logging.error( 197 | f"GET {base_url}/{rest_api_url} " 198 | f"threw {ex.__class__.__name__} ({ex})" 199 | ) 200 | 201 | return "" 202 | finally: 203 | elapsed_time = time.time() - start_time 204 | logging.info(f"GET {base_url}/{rest_api_url} took {elapsed_time:.2f}") 205 | 206 | 207 | def random_unused_port() -> int: 208 | sock = socket.socket() 209 | sock.bind(("", 0)) 210 | return sock.getsockname()[1] 211 | 212 | 213 | def start_browser(url: str): 214 | if platform.system() == "Windows": 215 | subprocess.Popen( 216 | ["cmd", "/c", "start", url.replace("&", "^&")], 217 | creationflags=subprocess.CREATE_NO_WINDOW, 218 | ) 219 | elif platform.system() == "Linux": 220 | subprocess.Popen(["xdg-open", url]) 221 | elif platform.system() == "Darwin": 222 | subprocess.Popen(["open", url]) 223 | else: 224 | raise NotImplementedError(f"Unsupported platform: {platform.system()}") 225 | 226 | 227 | def agg_cookies_from_responses( 228 | response_flow: requests.Response, 229 | ) -> requests.cookies.RequestsCookieJar: 230 | if not response_flow.history: 231 | return response_flow.cookies 232 | 233 | agg_cookies: requests.cookies.RequestsCookieJar = response_flow.history[ 234 | 0 235 | ].cookies 236 | for resp in response_flow.history[1:]: 237 | agg_cookies.update(resp.cookies) 238 | agg_cookies.update(response_flow.cookies) 239 | return agg_cookies 240 | -------------------------------------------------------------------------------- /sf_git/sf_git.conf: -------------------------------------------------------------------------------- 1 | SNOWFLAKE_VERSIONING_REPO='/tmp/trash' 2 | WORKSHEETS_PATH='/tmp/trash/worksheets' 3 | SF_ACCOUNT_ID='tofill.west-europe.azure' 4 | SF_LOGIN_NAME='TOFILL' 5 | -------------------------------------------------------------------------------- /sf_git/snowsight_auth.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | import socket 5 | import uuid 6 | from typing import Any, Dict 7 | from urllib import parse 8 | from urllib.parse import urlparse 9 | 10 | import requests 11 | import urllib3 12 | 13 | import sf_git.config as config 14 | from sf_git.models import ( 15 | AuthenticationContext, 16 | AuthenticationError, 17 | AuthenticationMode, 18 | ) 19 | from sf_git.rest_utils import ( 20 | agg_cookies_from_responses, 21 | api_get, 22 | api_post, 23 | random_unused_port, 24 | start_browser, 25 | ) 26 | 27 | urllib3.disable_warnings() 28 | 29 | 30 | def authenticate_to_snowsight( 31 | account_name: str, 32 | login_name: str, 33 | password: str, 34 | auth_mode: AuthenticationMode = AuthenticationMode.PWD, 35 | ) -> AuthenticationContext: 36 | auth_context = AuthenticationContext() 37 | auth_context.main_app_url = config.GLOBAL_CONFIG.sf_main_app_url 38 | auth_context.account_name = account_name 39 | auth_context.login_name = login_name 40 | 41 | # Get App Server Url and Account Url 42 | app_endpoint = get_account_app_endpoint(account_name) 43 | if not app_endpoint["valid"]: 44 | raise AuthenticationError( 45 | f"No valid endpoint for account {account_name}" 46 | ) 47 | 48 | auth_context.account = app_endpoint["account"] 49 | auth_context.account_url = app_endpoint["url"] 50 | auth_context.app_server_url = app_endpoint["appServerUrl"] 51 | auth_context.region = app_endpoint["region"] 52 | 53 | # Get unverified csrf 54 | bootstrap_response = get_csrf_from_boostrap_cookie(auth_context) 55 | csrf_cookie = next( 56 | (c for c in bootstrap_response.cookies if c.name.startswith("csrf-")), 57 | None, 58 | ) 59 | if not csrf_cookie: 60 | raise AuthenticationError("Could not get csrf from bootstrap endpoint") 61 | auth_context.csrf = csrf_cookie.value 62 | auth_context.cookies = bootstrap_response.cookies 63 | 64 | # Get client ID 65 | client_id_responses = oauth_start_get_snowsight_client_id_in_deployment( 66 | auth_context=auth_context 67 | ) 68 | parsed = urlparse(client_id_responses.url) 69 | parsed_query = parse.parse_qs(parsed.query) 70 | auth_context.cookies = agg_cookies_from_responses( 71 | client_id_responses 72 | ) # client_id_responses.history[-1].cookies 73 | state = json.loads(parsed_query["state"][0]) 74 | auth_context.oauth_nonce = state["oauthNonce"] 75 | auth_context.auth_originator = state.get("originator") 76 | auth_context.window_id = state.get("windowId") 77 | auth_context.auth_code_challenge = parsed_query["code_challenge"][0] 78 | auth_context.auth_code_challenge_method = parsed_query[ 79 | "code_challenge_method" 80 | ][0] 81 | auth_context.auth_redirect_uri = parsed_query["redirect_uri"][0] 82 | auth_context.client_id = parsed_query["client_id"][0] 83 | 84 | if not auth_context.organization_id: 85 | auth_context.organization_id = os.environ.get("SF_ORGANIZATION_ID") 86 | 87 | # Get master token 88 | if auth_mode == AuthenticationMode.PWD: 89 | auth_response = get_token_from_credentials( 90 | auth_context=auth_context, 91 | login_name=login_name, 92 | password=password, 93 | ) 94 | auth_response_details = json.loads(auth_response.text) 95 | if not auth_response_details["success"]: 96 | raise AuthenticationError("Invalid credentials") 97 | 98 | auth_context.master_token = auth_response_details["data"][ 99 | "masterToken" 100 | ] 101 | auth_context.auth_session_token = auth_response_details["data"][ 102 | "token" 103 | ] 104 | auth_context.server_version = auth_response_details["data"].get( 105 | "serverVersion" 106 | ) 107 | 108 | oauth_response = oauth_get_master_token_from_credentials( 109 | auth_context=auth_context, login_name=login_name, password=password 110 | ) 111 | oauth_response_details = json.loads(oauth_response.text) 112 | if not oauth_response_details["success"]: 113 | raise AuthenticationError("Invalid credentials") 114 | auth_context.master_token = oauth_response_details["data"][ 115 | "masterToken" 116 | ] 117 | 118 | elif auth_mode == AuthenticationMode.SSO: 119 | redirect_port = random_unused_port() 120 | sso_link_response = json.loads( 121 | get_sso_login_link( 122 | auth_context.account_url, 123 | account_name.split(".")[0], 124 | login_name, 125 | redirect_port, 126 | ) 127 | ) 128 | idp_url = sso_link_response["data"]["ssoUrl"] 129 | proof_key = sso_link_response["data"]["proofKey"] 130 | with socket.socket() as s: 131 | s.bind(("localhost", redirect_port)) 132 | start_browser(idp_url) 133 | s.listen(5) 134 | conn, addr = s.accept() 135 | sso_token = "" 136 | while True: 137 | data = conn.recv(8188) 138 | if not data: 139 | break 140 | str_data = data.decode("utf-8") 141 | if "token=" in str_data: 142 | str_data = str_data.split("token=")[1] 143 | if " HTTP" in str_data: # end of token 144 | str_data = str_data.split(" HTTP")[0] 145 | sso_token += str_data 146 | break 147 | sso_token += str_data 148 | s.close() 149 | if sso_token == "": 150 | raise AuthenticationError("Could not retrieve SSO token") 151 | 152 | auth_context.master_token = get_master_token_from_sso_token( 153 | auth_context.account_url, 154 | auth_context.account_name, 155 | auth_context.login_name, 156 | sso_token, 157 | proof_key, 158 | ) 159 | 160 | # Get oauth redirect 161 | oauth_response = oauth_authorize_get_oauth_redirect_from_oauth_token( 162 | auth_context=auth_context 163 | ) 164 | if ( 165 | oauth_response["code"] == "390302" 166 | or oauth_response["message"] == "Invalid consent request." 167 | ): 168 | print(oauth_response) 169 | raise AuthenticationError( 170 | "Could not get redirect from master token " 171 | f"for account url {auth_context.account_url} " 172 | f"and client_id {auth_context.client_id}. " 173 | "Please check your credentials." 174 | ) 175 | 176 | # Finalize authentication 177 | finalized_response = oauth_complete_get_auth_token_from_redirect( 178 | auth_context=auth_context, url=oauth_response["data"]["redirectUrl"] 179 | ) 180 | if finalized_response.status_code != 200: 181 | raise AuthenticationError( 182 | "Could not finalize authentication " 183 | f"for account url {auth_context.account_url} " 184 | f"with oauth_nonce_cookie {auth_context.oauth_nonce}" 185 | ) 186 | auth_context.cookies = agg_cookies_from_responses(finalized_response) 187 | for cookie in finalized_response.cookies: 188 | if cookie.name.startswith("user"): 189 | auth_context.snowsight_token[cookie.name] = cookie.value 190 | if not auth_context.snowsight_token: 191 | raise AuthenticationError( 192 | "Finalized Authentication but didnt find a token" 193 | " in final response : {status_code : " 194 | f"{finalized_response.status_code}, url : " 195 | f"{finalized_response.url}, cookies : " 196 | f"{finalized_response.cookies}" 197 | ) 198 | 199 | # handle different username 200 | params_page = finalized_response.content.decode("utf-8") 201 | match = re.search("(?i)var params = ({.*})", params_page, re.IGNORECASE) 202 | if match is not None: 203 | params = match.group(0).replace("var params = ", "") 204 | params = json.loads(params) 205 | username = params["user"].get("username") 206 | if username: 207 | auth_context.username = username 208 | else: 209 | auth_context.username = login_name 210 | else: 211 | auth_context.username = login_name 212 | 213 | return auth_context 214 | 215 | 216 | def get_account_app_endpoint(account_name: str) -> Dict[str, Any]: 217 | main_app_url = config.GLOBAL_CONFIG.sf_main_app_url 218 | response = api_post( 219 | main_app_url, 220 | f"v0/validate-snowflake-url?url={account_name}", 221 | "*/*", 222 | ) 223 | 224 | return json.loads(response) 225 | 226 | 227 | def get_csrf_from_boostrap_cookie( 228 | auth_context: AuthenticationContext, 229 | ) -> requests.Response: 230 | response = api_get( 231 | base_url=auth_context.main_app_url, 232 | rest_api_url="bootstrap", 233 | accept_header="*/*", 234 | allow_redirect=True, 235 | as_obj=True, 236 | cookies=auth_context.cookies, 237 | ) 238 | return response 239 | 240 | 241 | def oauth_start_get_snowsight_client_id_in_deployment( 242 | auth_context: AuthenticationContext, 243 | ) -> requests.Response: 244 | state_params = ( 245 | '{"csrf":' 246 | f'"{auth_context.csrf}","url":"{auth_context.account_url}","windowId":"{uuid.uuid4()}","browserUrl":"{auth_context.main_app_url}"}}' # noqa: E501 247 | ) 248 | 249 | rest_api_url = ( 250 | "start-oauth/snowflake?accountUrl=" 251 | f"{auth_context.account_url}" 252 | f"&&state={parse.quote_plus(state_params)}" 253 | ) 254 | 255 | response = api_get( 256 | base_url=auth_context.app_server_url, 257 | rest_api_url=rest_api_url, 258 | accept_header="text/html", 259 | csrf=auth_context.csrf, 260 | allow_redirect=True, 261 | cookies=auth_context.cookies, 262 | as_obj=True, 263 | ) 264 | 265 | if response.status_code == 200: 266 | return response 267 | 268 | raise AuthenticationError( 269 | f"No client id could be retrieved for account {auth_context.account_url}" 270 | ) 271 | 272 | 273 | def get_token_from_credentials( 274 | auth_context: AuthenticationContext, 275 | login_name: str, 276 | password: str, 277 | ) -> requests.Response: 278 | request_body = { 279 | "data": { 280 | "ACCOUNT_NAME": auth_context.account_name.split(".")[0], 281 | "LOGIN_NAME": login_name, 282 | "PASSWORD": password, 283 | } 284 | } 285 | 286 | auth_response = api_post( 287 | base_url=auth_context.account_url, 288 | rest_api_url="session/v1/login-request", 289 | request_type_header="application/json", 290 | accept_header="application/json", 291 | request_body=json.dumps(request_body), 292 | cookies=auth_context.cookies, 293 | as_obj=True, 294 | ) 295 | return auth_response 296 | 297 | 298 | def oauth_get_master_token_from_credentials( 299 | auth_context: AuthenticationContext, 300 | login_name: str, 301 | password: str, 302 | ) -> requests.Response: 303 | state_params = ( 304 | f'{{"csrf":"{auth_context.csrf}"' 305 | f',"url":"{auth_context.account_url}","browserUrl":"{auth_context.main_app_url}"' 306 | f',"originator":"{auth_context.auth_originator}","oauthNonce":"{auth_context.oauth_nonce}"}}' 307 | ) 308 | 309 | request_body = { 310 | "data": { 311 | "ACCOUNT_NAME": auth_context.account_name.split(".")[0], 312 | "LOGIN_NAME": login_name, 313 | "clientId": auth_context.client_id, 314 | "redirectURI": auth_context.auth_redirect_uri, 315 | "responseType": "code", 316 | "state": state_params, 317 | "scope": "refresh_token", 318 | "codeChallenge": auth_context.auth_code_challenge, 319 | "codeChallengeMethod": auth_context.auth_code_challenge_method 320 | or "S256", 321 | "CLIENT_APP_ID": "Snowflake UI", 322 | "CLIENT_APP_VERSION": 20241007103851, 323 | "PASSWORD": password, 324 | } 325 | } 326 | return api_post( 327 | base_url=auth_context.account_url, 328 | rest_api_url="session/authenticate-request", 329 | accept_header="application/json", 330 | request_body=json.dumps(request_body), 331 | request_type_header="application/json", 332 | cookies=auth_context.cookies, 333 | as_obj=True, 334 | allow_redirect=True, 335 | ) 336 | 337 | 338 | def oauth_authorize_get_oauth_redirect_from_oauth_token( 339 | auth_context: AuthenticationContext, 340 | ) -> Dict[str, Any]: 341 | state_params = ( 342 | f'{{"csrf":"{auth_context.csrf}"' 343 | f',"url":"{auth_context.account_url}","windowId":"{auth_context.window_id}","browserUrl":"{auth_context.main_app_url}"' # noqa: E501 344 | f',"originator":"{auth_context.auth_originator}","oauthNonce":"{auth_context.oauth_nonce}"}}' 345 | ) 346 | request_body = { 347 | "masterToken": auth_context.master_token, 348 | "clientId": auth_context.client_id, 349 | "redirectURI": auth_context.auth_redirect_uri, 350 | "responseType": "code", 351 | "state": state_params, 352 | "scope": "refresh_token", 353 | "codeChallenge": auth_context.auth_code_challenge, 354 | "codeChallengeMethod": auth_context.auth_code_challenge_method 355 | or "S256", 356 | } 357 | 358 | response_content = api_post( 359 | base_url=auth_context.account_url, 360 | rest_api_url="oauth/authorization-request", 361 | accept_header="application/json", 362 | request_body=json.dumps(request_body), 363 | request_type_header="application/json", 364 | csrf=auth_context.csrf, 365 | cookies=auth_context.cookies, 366 | allow_redirect=True, 367 | ) 368 | 369 | return json.loads(response_content) 370 | 371 | 372 | def oauth_complete_get_auth_token_from_redirect( 373 | auth_context: AuthenticationContext, 374 | url: str, 375 | ) -> requests.Response: 376 | headers = { 377 | "Accept": "*/*", 378 | "Referer": f"{auth_context.main_app_url}/", 379 | } 380 | 381 | response = requests.get( 382 | url, 383 | headers=headers, 384 | allow_redirects=True, 385 | timeout=10, 386 | cookies=auth_context.cookies, 387 | ) 388 | 389 | return response 390 | 391 | 392 | def get_sso_login_link( 393 | account_url: str, 394 | account_name: str, 395 | login_name: str, 396 | redirect_port: int, 397 | ): 398 | request_json_template = { 399 | "data": { 400 | "ACCOUNT_NAME": account_name, 401 | "LOGIN_NAME": login_name, 402 | "AUTHENTICATOR": "externalbrowser", 403 | "BROWSER_MODE_REDIRECT_PORT": redirect_port, 404 | } 405 | } 406 | 407 | request_body = json.dumps(request_json_template) 408 | 409 | response = api_post( 410 | account_url, 411 | "session/authenticator-request", 412 | "application/json", 413 | request_body, 414 | "application/json", 415 | ) 416 | 417 | return response 418 | 419 | 420 | def get_master_token_from_sso_token( 421 | account_url: str, 422 | account_name: str, 423 | login_name: str, 424 | sso_token: str, 425 | proof_key: str, 426 | ) -> str: 427 | request_json_template = { 428 | "data": { 429 | "ACCOUNT_NAME": account_name, 430 | "LOGIN_NAME": login_name, 431 | "AUTHENTICATOR": "externalbrowser", 432 | "TOKEN": sso_token, 433 | "PROOF_KEY": proof_key, 434 | } 435 | } 436 | 437 | request_body = json.dumps(request_json_template) 438 | 439 | response = api_post( 440 | account_url, 441 | "session/v1/login-request", 442 | "application/json", 443 | request_body, 444 | "application/json", 445 | ) 446 | 447 | parsed = json.loads(response) 448 | if not parsed["success"]: 449 | raise AuthenticationError( 450 | "Could not retrieve master token from sso_token for user" 451 | f"{login_name} and account {account_name}" 452 | ) 453 | return parsed["data"]["masterToken"] 454 | -------------------------------------------------------------------------------- /sf_git/worksheets_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from urllib import parse 3 | from typing import Callable, List, Optional 4 | import pandas as pd 5 | import requests 6 | 7 | from sf_git.cache import save_worksheets_to_cache 8 | from sf_git.models import ( 9 | AuthenticationContext, 10 | Folder, 11 | SnowsightError, 12 | Worksheet, 13 | WorksheetError, 14 | ) 15 | 16 | 17 | def get_worksheets( 18 | auth_context: AuthenticationContext, 19 | store_to_cache: Optional[bool] = False, 20 | only_folder: Optional[str] = None, 21 | ) -> List[Worksheet]: 22 | """ 23 | Get list of worksheets available for authenticated user 24 | """ 25 | 26 | optionsparams = ( 27 | '{"sort":{"col":"viewed","dir":"desc"},"limit":500,"owner":null,' 28 | '"types":["query", "folder"],"showNeverViewed":"if-invited"}' 29 | ) 30 | 31 | request_json_template = { 32 | "options": optionsparams, 33 | "location": "worksheets", 34 | } 35 | req_body = parse.urlencode(request_json_template) 36 | 37 | sf_context_name = auth_context.username 38 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}" 39 | 40 | res = requests.post( 41 | f"{auth_context.app_server_url}/v0/organizations/" 42 | f"{auth_context.organization_id}/entities/list", 43 | data=req_body, 44 | headers={ 45 | "Accept": "application/json", 46 | "Content-Type": "application/x-www-form-urlencoded", 47 | "X-Snowflake-Context": snowflake_context, 48 | "Referer": auth_context.main_app_url, 49 | }, 50 | cookies=auth_context.snowsight_token, 51 | timeout=90, 52 | ) 53 | 54 | if res.status_code != 200: 55 | raise WorksheetError( 56 | "Failed to get worksheet list\n" f"\t Reason is {res.text}" 57 | ) 58 | res_data = json.loads(res.text) 59 | entities = res_data["entities"] 60 | contents = res_data["models"].get("queries") 61 | if contents is None: 62 | return [] 63 | 64 | worksheets = [] 65 | for worksheet in entities: 66 | if ( 67 | only_folder is not None 68 | and worksheet["info"]["folderName"] != only_folder 69 | ): 70 | continue 71 | if worksheet["entityType"] == "query": 72 | current_ws = Worksheet( 73 | worksheet["entityId"], 74 | worksheet["info"]["name"], 75 | worksheet["info"]["folderId"], 76 | worksheet["info"]["folderName"], 77 | content_type=worksheet["info"]["queryLanguage"], 78 | ) 79 | worksheets.append(current_ws) 80 | 81 | for i, _ in enumerate(worksheets): 82 | content = contents[worksheets[i]._id] 83 | if "query" in content.keys(): 84 | worksheets[i].content = content["query"] 85 | elif "drafts" in content.keys(): 86 | draft_ids = [draft_id for draft_id in content["drafts"].keys()] 87 | if len(draft_ids) > 0: 88 | if len(draft_ids) > 1: # keys are timestamps 89 | last_update_id = str( 90 | max([int(draft_id) for draft_id in draft_ids]) 91 | ) 92 | else: 93 | last_update_id = draft_ids[0] 94 | worksheets[i].content = content["drafts"][last_update_id][ 95 | "query" 96 | ] 97 | else: 98 | worksheets[i].content = "" 99 | else: 100 | worksheets[i].content = "" 101 | 102 | if store_to_cache: 103 | save_worksheets_to_cache(worksheets) 104 | 105 | return worksheets 106 | 107 | 108 | def print_worksheets( 109 | worksheets: List[Worksheet], n=10, logger: Callable = print 110 | ): 111 | """ 112 | Log worksheets as a dataframe. 113 | 114 | :param worksheets: worksheets to log 115 | :param n: maximum number of worksheets to print (from head) 116 | :param logger: logging function e.g. print 117 | """ 118 | worksheets_df = pd.DataFrame([ws.to_dict() for ws in worksheets]) 119 | if worksheets_df.empty: 120 | logger("No worksheet") 121 | else: 122 | logger(worksheets_df.head(n)) 123 | 124 | 125 | def write_worksheet( 126 | auth_context: AuthenticationContext, worksheet: Worksheet 127 | ) -> Optional[WorksheetError]: 128 | """Write local worksheet to Snowsight worksheet.""" 129 | 130 | version = 1 131 | request_json_template = { 132 | "action": "saveDraft", 133 | "id": worksheet._id, 134 | "projectId": f"{worksheet._id}@{version}", 135 | "query": worksheet.content, 136 | "version": version, 137 | "modifiedTime": "2024-10-11T20:15:28.558Z", 138 | "appSessionId": "9054529735682", 139 | } 140 | 141 | req_body = parse.urlencode(request_json_template) 142 | 143 | sf_context_name = auth_context.username 144 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}" 145 | res = requests.post( 146 | f"{auth_context.app_server_url}/v0/queries", 147 | data=req_body, 148 | headers={ 149 | "Accept": "application/json", 150 | "Content-Type": "application/x-www-form-urlencoded", 151 | "X-Snowflake-Context": snowflake_context, 152 | "Referer": auth_context.main_app_url, 153 | "X-CSRF-Token": auth_context.csrf, 154 | "X-Snowflake-Role": "ACCOUNTADMIN", 155 | "X-Snowflake-Page-Source": "worksheet" 156 | }, 157 | cookies=auth_context.cookies, 158 | timeout=90, 159 | ) 160 | 161 | if res.status_code != 200: 162 | error_type = ( 163 | SnowsightError.PERMISSION 164 | if res.status_code == 403 165 | else SnowsightError.UNKNOWN 166 | ) 167 | return WorksheetError( 168 | f"Failed to write worksheet {worksheet.name}\n" 169 | f"\t Reason is {res.text}", 170 | error_type, 171 | ) 172 | 173 | 174 | def create_worksheet( 175 | auth_context: AuthenticationContext, 176 | worksheet_name: str, 177 | folder_id: str = None, 178 | ) -> str: 179 | """Create empty worksheet on Snowsight user workspace. 180 | 181 | :returns: str new worksheet id. 182 | """ 183 | 184 | request_json_template = { 185 | "action": "create", 186 | "orgId": auth_context.organization_id, 187 | "name": worksheet_name, 188 | } 189 | 190 | if folder_id: 191 | request_json_template["folderId"] = folder_id 192 | 193 | req_body = parse.urlencode(request_json_template) 194 | 195 | sf_context_name = auth_context.username 196 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}" 197 | 198 | res = requests.post( 199 | f"{auth_context.app_server_url}/v0/queries", 200 | data=req_body, 201 | headers={ 202 | "Accept": "application/json", 203 | "Content-Type": "application/x-www-form-urlencoded", 204 | "X-Snowflake-Context": snowflake_context, 205 | "Referer": auth_context.main_app_url, 206 | }, 207 | cookies=auth_context.snowsight_token, 208 | timeout=90, 209 | ) 210 | 211 | if res.status_code != 200: 212 | raise WorksheetError( 213 | f"Failed to create worksheet {worksheet_name}\n" 214 | f"\t Reason is {res.text}" 215 | ) 216 | response_data = json.loads(res.text) 217 | return response_data["pid"] 218 | 219 | 220 | def get_folders(auth_context: AuthenticationContext) -> List[Folder]: 221 | """ 222 | Get list of folders on authenticated user workspace 223 | """ 224 | 225 | optionsparams = ( 226 | '{"sort":{"col":"viewed","dir":"desc"},"limit":500,"owner":null,' 227 | '"types":["query", "folder"],"showNeverViewed":"if-invited"}' 228 | ) 229 | 230 | request_json_template = { 231 | "options": optionsparams, 232 | "location": "worksheets", 233 | } 234 | req_body = parse.urlencode(request_json_template) 235 | 236 | sf_context_name = auth_context.username 237 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}" 238 | 239 | res = requests.post( 240 | f"{auth_context.app_server_url}/v0/organizations/" 241 | f"{auth_context.organization_id}/entities/list", 242 | data=req_body, 243 | headers={ 244 | "Accept": "application/json", 245 | "Content-Type": "application/x-www-form-urlencoded", 246 | "X-Snowflake-Context": snowflake_context, 247 | "Referer": auth_context.main_app_url, 248 | }, 249 | cookies=auth_context.snowsight_token, 250 | timeout=90, 251 | ) 252 | 253 | if res.status_code != 200: 254 | raise WorksheetError( 255 | "Failed to get folder list\n" f"\t Reason is {res.text}" 256 | ) 257 | res_data = json.loads(res.text) 258 | entities = res_data["entities"] 259 | 260 | folders = [] 261 | for entity in entities: 262 | if entity["entityType"] == "folder": 263 | current_folder = Folder( 264 | entity["entityId"], 265 | entity["info"]["name"], 266 | ) 267 | folders.append(current_folder) 268 | return folders 269 | 270 | 271 | def print_folders(folders: List[Folder], n=10): 272 | folders_df = pd.DataFrame([f.to_dict() for f in folders]) 273 | print(folders_df.head(n)) 274 | 275 | 276 | def create_folder( 277 | auth_context: AuthenticationContext, folder_name: str 278 | ) -> str: 279 | """Create empty folder on Snowsight user workspace. 280 | 281 | :returns: str new folder id. 282 | """ 283 | 284 | request_json_template = { 285 | "orgId": auth_context.organization_id, 286 | "name": folder_name, 287 | "type": "list", 288 | "visibility": "private", 289 | } 290 | 291 | req_body = parse.urlencode(request_json_template) 292 | 293 | sf_context_name = auth_context.username 294 | snowflake_context = f"{sf_context_name}::{auth_context.account_url}" 295 | 296 | res = requests.post( 297 | f"{auth_context.app_server_url}/v0/folders", 298 | data=req_body, 299 | headers={ 300 | "Accept": "*/*", 301 | "Content-Type": "application/x-www-form-urlencoded", 302 | "X-Snowflake-Context": snowflake_context, 303 | "Referer": auth_context.main_app_url, 304 | }, 305 | cookies=auth_context.snowsight_token, 306 | timeout=90, 307 | ) 308 | 309 | if res.status_code != 200: 310 | raise WorksheetError( 311 | f"Failed to create folder {folder_name}\n" 312 | f"\t Reason is {res.text}" 313 | ) 314 | response_data = json.loads(res.text) 315 | return response_data["createdFolderId"] 316 | 317 | 318 | def upload_to_snowsight( 319 | auth_context: AuthenticationContext, worksheets: List[Worksheet] 320 | ) -> dict[str, List[dict]]: 321 | """ 322 | Upload worksheets to Snowsight user workspace 323 | keeping folder architecture. 324 | 325 | :param auth_context: Authentication info for Snowsight 326 | :param worksheets: list of worksheets to upload 327 | 328 | :returns: upload report with {'completed': list, 'errors': list} 329 | """ 330 | 331 | upload_report = {"completed": [], "errors": []} 332 | 333 | ss_folders = get_folders(auth_context) 334 | ss_folders = {folder.name: folder for folder in ss_folders} 335 | 336 | ss_worksheets = get_worksheets(auth_context) 337 | ss_worksheets = {ws.name: ws for ws in ss_worksheets} 338 | 339 | print( 340 | " ## Writing local worksheet to SnowSight" 341 | f" for user {auth_context.username} ##" 342 | ) 343 | for ws in worksheets: 344 | # folder management 345 | if ws.folder_name: 346 | if ws.folder_name in ss_folders.keys(): 347 | folder_id = ss_folders[ws.folder_name]._id 348 | else: 349 | print(f"creating folder {ws.folder_name}") 350 | folder_id = create_folder(auth_context, ws.folder_name) 351 | new_folder = Folder( 352 | folder_id, 353 | ws.folder_name, 354 | ) 355 | ss_folders[ws.folder_name] = new_folder 356 | else: 357 | folder_id = None 358 | 359 | # worksheet management 360 | if ws.name not in ss_worksheets.keys(): 361 | print(f"creating worksheet {ws.name}") 362 | worksheet_id = create_worksheet(auth_context, ws.name, folder_id) 363 | update_content = True 364 | else: 365 | worksheet_id = ss_worksheets[ws.name]._id 366 | update_content = ws.content != ss_worksheets[ws.name].content 367 | 368 | # content management 369 | if ws.content and update_content: 370 | print(f"updating worksheet {ws.name}") 371 | err = write_worksheet( 372 | auth_context, 373 | Worksheet( 374 | worksheet_id, 375 | ws.name, 376 | folder_id, 377 | ws.folder_name, 378 | ws.content, 379 | ), 380 | ) 381 | if err is not None: 382 | upload_report["errors"].append({"name": ws.name, "error": err}) 383 | else: 384 | upload_report["completed"].append({"name": ws.name}) 385 | 386 | print(" ## SnowSight updated ##") 387 | return upload_report 388 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import shutil 3 | from pathlib import Path 4 | import datetime 5 | from zoneinfo import ZoneInfo 6 | from git import Repo, Actor 7 | from dotenv import dotenv_values 8 | 9 | import sf_git.models 10 | from sf_git.config import Config 11 | 12 | PACKAGE_ROOT = Path(__file__).parent.parent 13 | TEST_CONF = dotenv_values(PACKAGE_ROOT / "sf_git.test.conf") 14 | 15 | TESTING_FOLDER = Path(__file__).parent.absolute() 16 | REPO_ROOT_PATH = Path(TEST_CONF["SNOWFLAKE_VERSIONING_REPO"]).absolute() 17 | REPO_DATA_PATH = Path(TEST_CONF["WORKSHEETS_PATH"]).absolute() 18 | 19 | 20 | @pytest.fixture(scope="session", name="testing_folder") 21 | def testing_folder(): 22 | return TESTING_FOLDER 23 | 24 | 25 | @pytest.fixture(scope="session", name="fixture_dotenv_path") 26 | def dotenv_path(): 27 | return REPO_ROOT_PATH / "sf_git.test.conf" 28 | 29 | 30 | @pytest.fixture(scope="session", name="repo_root_path") 31 | def repo_root_path(): 32 | return REPO_ROOT_PATH 33 | 34 | 35 | @pytest.fixture(scope="session", autouse=True) 36 | def setup_and_teardown(): 37 | """ 38 | Setup test envs and teardown 39 | """ 40 | 41 | # ---- SETUP ----- 42 | # Copy the content from `tests/assets/test_data`. 43 | if REPO_ROOT_PATH.is_dir(): 44 | shutil.rmtree(REPO_ROOT_PATH) 45 | 46 | shutil.copytree( 47 | TESTING_FOLDER / "data", REPO_DATA_PATH, dirs_exist_ok=True 48 | ) 49 | shutil.copy(PACKAGE_ROOT / "sf_git.test.conf", REPO_ROOT_PATH) 50 | 51 | # ---- RUN TESTS ----- 52 | yield 53 | 54 | # ---- TEARDOWN ----- 55 | shutil.rmtree(REPO_ROOT_PATH) 56 | 57 | 58 | @pytest.fixture(autouse=True, name="test_config") 59 | def mock_config(monkeypatch): 60 | import sf_git.config as config 61 | 62 | global_config = Config( 63 | repo_path=REPO_ROOT_PATH, 64 | worksheets_path=REPO_DATA_PATH, 65 | sf_account_id=TEST_CONF["SF_ACCOUNT_ID"], 66 | sf_login_name=TEST_CONF["SF_LOGIN_NAME"], 67 | sf_pwd=TEST_CONF["SF_PWD"], 68 | ) 69 | monkeypatch.setattr(config, "GLOBAL_CONFIG", global_config) 70 | 71 | return global_config 72 | 73 | 74 | @pytest.fixture(scope="session", name="repo") 75 | def repo() -> Repo: 76 | """ 77 | Create a git repository with sample data and history. 78 | """ 79 | 80 | author = Actor("An author", "author@example.com") 81 | committer = Actor("A committer", "committer@example.com") 82 | commit_date = datetime.datetime( 83 | 2024, 1, 16, tzinfo=ZoneInfo("Europe/Paris") 84 | ) 85 | 86 | # Init repo 87 | repo = Repo.init(REPO_ROOT_PATH) 88 | 89 | # Add some commits 90 | repo.index.add([REPO_DATA_PATH]) 91 | repo.index.commit( 92 | "Initial skeleton", 93 | author=author, 94 | committer=committer, 95 | author_date=commit_date, 96 | commit_date=commit_date, 97 | ) 98 | 99 | return repo 100 | 101 | 102 | @pytest.fixture(scope="session", name="auth_context") 103 | def auth_context() -> sf_git.models.AuthenticationContext: 104 | import sf_git.config as config 105 | 106 | context = sf_git.models.AuthenticationContext( 107 | app_server_url="https://test_snowflake.com", 108 | account_url="https://account.test_snowflake.com", 109 | account="fake.snowflake.account", 110 | account_name=config.GLOBAL_CONFIG.sf_account_id, 111 | client_id="fake", 112 | main_app_url="https://test_snowflake.com", 113 | master_token="fake", 114 | organization_id="fake.organization", 115 | region="west-europe", 116 | snowsight_token={"fake": "fake"}, 117 | login_name=config.GLOBAL_CONFIG.sf_login_name, 118 | username=config.GLOBAL_CONFIG.sf_login_name, 119 | ) 120 | 121 | return context 122 | -------------------------------------------------------------------------------- /tests/data/Benchmarking_Tutorials/.[Tutorial]_Sample_queries_on_TPC-DS_data_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] Sample queries on TPC-DS data", "_id": "5QwzKJlvL21", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Benchmarking_Tutorials/.[Tutorial]_Sample_queries_on_TPC-H_data_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] Sample queries on TPC-H data", "_id": "1AX6lOXnDRj", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Benchmarking_Tutorials/.[Tutorial]_TPC-DS_100TB_Complete_Query_Test_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] TPC-DS 100TB Complete Query Test", "_id": "1vN2lXM6eHu", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Benchmarking_Tutorials/.[Tutorial]_TPC-DS_10TB_Complete_Query_Test_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] TPC-DS 10TB Complete Query Test", "_id": "5S4d9X4vnRH", "folder_name": "Benchmarking Tutorials", "folder_id": "3nHNavVT", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Benchmarking_Tutorials/[Tutorial]_Sample_queries_on_TPC-H_data.sql: -------------------------------------------------------------------------------- 1 | /* Tutorial 1: Sample queries on TPC-H data 2 | 3 | Prerequisites 4 | This tutorial requires the Snowflake provided 5 | snowflake_sample_data database. If you don't 6 | have this database already in your account 7 | please add it by following these instructions: 8 | https://docs.snowflake.net/manuals/user-guide/sample-data-using.html 9 | 10 | Pricing Summary Report Query (Q1) 11 | This query demonstrates basic SQL 12 | functionality using the included TPC-H sample 13 | data. The results are the amount of business 14 | that was billed, shipped, and returned. 15 | 16 | Business Question: The Pricing Summary Report 17 | Query provides a summary pricing report for 18 | all line items shipped as of a given date. The 19 | date is within 60 - 120 days of the greatest 20 | ship date contained in the database. The query 21 | lists totals for extended price, discounted 22 | extended price, discounted extended price plus 23 | tax, average quantity, average extended price, 24 | and average discount. These aggregates are 25 | grouped by RETURNFLAG and LINESTATUS, and 26 | listed in ascending order of RETURNFLAG and 27 | LINESTATUS. A count of the number of line items 28 | in each group is included. 29 | */ 30 | 31 | use schema snowflake_sample_data.tpch_sf1; 32 | -- or tpch_sf100, tpch_sf1000 33 | 34 | SELECT 35 | l_returnflag, 36 | l_linestatus, 37 | sum(l_quantity) as sum_qty, 38 | sum(l_extendedprice) as sum_base_price, 39 | sum(l_extendedprice * (1-l_discount)) 40 | as sum_disc_price, 41 | sum(l_extendedprice * (1-l_discount) * 42 | (1+l_tax)) as sum_charge, 43 | avg(l_quantity) as avg_qty, 44 | avg(l_extendedprice) as avg_price, 45 | avg(l_discount) as avg_disc, 46 | count(*) as count_order 47 | FROM 48 | lineitem 49 | WHERE 50 | l_shipdate <= dateadd(day, -90, to_date('1998-12-01')) 51 | GROUP BY 52 | l_returnflag, 53 | l_linestatus 54 | ORDER BY 55 | l_returnflag, 56 | l_linestatus; 57 | 58 | -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/.[Template]_Adding_a_user_and_granting_roles_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Template] Adding a user and granting roles", "_id": "4ux48jcFhEw", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/.[Tutorial]_Using_Python_to_load_and_query_sample_data_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] Using Python to load and query sample data", "_id": "vMjWswGje9", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "python"} -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/.[Tutorial]_Using_SQL_to_load_and_query_sample_data_metadata.json: -------------------------------------------------------------------------------- 1 | {"name": "[Tutorial] Using SQL to load and query sample data", "_id": "Je8VP5w1VD", "folder_name": "Getting Started Tutorials", "folder_id": "Ga8xhnsR", "content_type": "sql"} -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/[Template]_Adding_a_user_and_granting_roles.sql: -------------------------------------------------------------------------------- 1 | /*-- 2 | In this Worksheet we will walk through creating a User in Snowflake. 3 | 4 | For the User we will provide grants to a defined default role and default warehouse 5 | and then walk through viewing all other users and roles in our account. 6 | 7 | To conclude, we will drop the created User. 8 | --*/ 9 | 10 | 11 | ------------------------------------------------------------------------------------------- 12 | -- Step 1: To start, we first must set our Role context 13 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role 14 | -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles 15 | ------------------------------------------------------------------------------------------- 16 | 17 | --> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return). 18 | --> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return). 19 | 20 | ---> set our Role context 21 | USE ROLE USERADMIN; 22 | 23 | ------------------------------------------------------------------------------------------- 24 | -- Step 2: Create our User 25 | -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user 26 | ------------------------------------------------------------------------------------------- 27 | 28 | ---> now let's create a User using various available parameters. 29 | -- NOTE: please fill out each section below before executing the query 30 | 31 | CREATE OR REPLACE USER -- adjust user name 32 | PASSWORD = '' -- add a secure password 33 | LOGIN_NAME = '' -- add a login name 34 | FIRST_NAME = '' -- add user's first name 35 | LAST_NAME = '' -- add user's last name 36 | EMAIL = '' -- add user's email 37 | MUST_CHANGE_PASSWORD = true -- ensures a password reset on first login 38 | DEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH 39 | 40 | 41 | /*-- 42 | With the User created, send the following information in a secure manner 43 | to whomever the User is created for, so that they can access this Snowflake account: 44 | --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/) 45 | --> LOGIN_NAME: from above 46 | --> PASSWORD: from above 47 | --*/ 48 | 49 | ------------------------------------------------------------------------------------------- 50 | -- Step 3: Grant access to a Role and Warehouse for our User 51 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role 52 | -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role 53 | -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege 54 | ------------------------------------------------------------------------------------------- 55 | 56 | ---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it 57 | USE ROLE SECURITYADMIN; 58 | 59 | /*-- 60 | • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy). 61 | • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role). 62 | 63 | NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles. 64 | Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below: 65 | • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system 66 | --*/ 67 | 68 | -- grant role SYSADMIN to our User 69 | GRANT ROLE SYSADMIN TO USER ; 70 | 71 | 72 | -- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role 73 | GRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN; 74 | 75 | 76 | ------------------------------------------------------------------------------------------- 77 | -- Step 4: Explore all Users and Roles in our Account 78 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role 79 | -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users 80 | -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles 81 | ------------------------------------------------------------------------------------------- 82 | 83 | ---> let's now explore all users and roles in our account using our ACCOUNTADMIN role 84 | USE ROLE ACCOUNTADMIN; 85 | 86 | -- show all users in account 87 | SHOW USERS; 88 | 89 | -- show all roles in account 90 | SHOW ROLES; 91 | 92 | ------------------------------------------------------------------------------------------- 93 | -- Step 5: Drop our created Users 94 | -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user 95 | ------------------------------------------------------------------------------------------- 96 | 97 | ---> to drop the user, we could execute the following command 98 | DROP USER ; 99 | -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/[Tutorial]_Using_Python_to_load_and_query_sample_data.py: -------------------------------------------------------------------------------- 1 | ################################################################################################### 2 | # Tasty Bytes is a fictitious, global food truck network, that is on a mission to serve unique food 3 | # options with high quality items in a safe, convenient and cost effective way. In order to drive 4 | # forward on their mission, Tasty Bytes is beginning to leverage the Snowflake Data Cloud. 5 | 6 | # In this Python Worksheet, we will walk through the end to end process required to load a CSV file 7 | # containing Menu specific data that is currently hosted in Blob Storage. 8 | 9 | # Please click Run and see details below on what each step it doing. The final output will return 10 | # a Dataframe 11 | ################################################################################################### 12 | 13 | ################################################################################################### 14 | # Step 1 - To start, we must first import our Snowpark Package and a few Functions and Types 15 | ################################################################################################### 16 | ## Note: You can add more packages by selecting them using the Packages control and then importing them. 17 | 18 | import snowflake.snowpark as snowpark 19 | from snowflake.snowpark.functions import col 20 | from snowflake.snowpark.types import StructField, StructType, IntegerType, StringType, VariantType 21 | 22 | ################################################################################################### 23 | # Step 2 - Let's now define our Session and Create our Database, Schema and Blob Stage 24 | ################################################################################################### 25 | 26 | def main(session: snowpark.Session): 27 | # Use SQL to create our Tasty Bytes Database 28 | session.sql('CREATE OR REPLACE DATABASE tasty_bytes_sample_data;').collect() 29 | 30 | # Use SQL to create our Raw POS (Point-of-Sale) Schema 31 | session.sql('CREATE OR REPLACE SCHEMA tasty_bytes_sample_data.raw_pos;').collect() 32 | 33 | # Use SQL to create our Blob Stage 34 | session.sql('CREATE OR REPLACE STAGE tasty_bytes_sample_data.public.blob_stage url = "s3://sfquickstarts/tastybytes/" file_format = (type = csv);').collect() 35 | 36 | ################################################################################################### 37 | # Step 3 - Now, we will define the Schema for our CSV file 38 | ################################################################################################### 39 | 40 | # Define our Menu Schema 41 | menu_schema = StructType([StructField("menu_id",IntegerType()),\ 42 | StructField("menu_type_id",IntegerType()),\ 43 | StructField("menu_type",StringType()),\ 44 | StructField("truck_brand_name",StringType()),\ 45 | StructField("menu_item_id",IntegerType()),\ 46 | StructField("menu_item_name",StringType()),\ 47 | StructField("item_category",StringType()),\ 48 | StructField("item_subcategory",StringType()),\ 49 | StructField("cost_of_goods_usd",IntegerType()),\ 50 | StructField("sale_price_usd",IntegerType()),\ 51 | StructField("menu_item_health_metrics_obj",VariantType())]) 52 | 53 | ################################################################################################### 54 | # Step 4 - Using the Schema, let's Create a Dataframe from the Menu file and Save it as a Table 55 | ################################################################################################### 56 | 57 | # Create a Dataframe from our Menu file from our Blob Stage 58 | df_blob_stage_read = session.read.schema(menu_schema).csv('@tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/') 59 | 60 | # Save our Dataframe as a Menu table in our Tasty Bytes Database and Raw POS Schema 61 | df_blob_stage_read.write.mode("overwrite").save_as_table("tasty_bytes_sample_data.raw_pos.menu") 62 | 63 | ################################################################################################### 64 | # Step 5 - With the table saved, let's create a new, filtered Dataframe and Return the results 65 | ################################################################################################### 66 | 67 | # Create a new Dataframe reading from our Menu table and filtering for the Freezing Point brand 68 | df_menu_freezing_point = session.table("tasty_bytes_sample_data.raw_pos.menu").filter(col("truck_brand_name") == 'Freezing Point') 69 | 70 | # return our Dataframe 71 | return df_menu_freezing_point -------------------------------------------------------------------------------- /tests/data/Getting_Started_Tutorials/[Tutorial]_Using_SQL_to_load_and_query_sample_data.sql: -------------------------------------------------------------------------------- 1 | /*-- 2 | Tasty Bytes is a fictitious, global food truck network, that is on a mission to serve unique food options with high 3 | quality items in a safe, convenient and cost effective way. In order to drive forward on their mission, Tasty Bytes 4 | is beginning to leverage the Snowflake Data Cloud. 5 | 6 | Within this Worksheet, we will walk through the end to end process required to load a CSV file containing Menu specific data 7 | that is currently hosted in Blob Storage. 8 | --*/ 9 | 10 | ------------------------------------------------------------------------------------------- 11 | -- Step 1: To start, let's set the Role and Warehouse context 12 | -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role 13 | -- USE WAREHOUSE: https://docs.snowflake.com/en/sql-reference/sql/use-warehouse 14 | ------------------------------------------------------------------------------------------- 15 | 16 | /*-- 17 | - To run a single query, place your cursor in the query editor and select the Run button (⌘-Return). 18 | - To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return). 19 | --*/ 20 | 21 | ---> set the Role 22 | USE ROLE accountadmin; 23 | 24 | ---> set the Warehouse 25 | USE WAREHOUSE compute_wh; 26 | 27 | ------------------------------------------------------------------------------------------- 28 | -- Step 2: With context in place, let's now create a Database, Schema, and Table 29 | -- CREATE DATABASE: https://docs.snowflake.com/en/sql-reference/sql/create-database 30 | -- CREATE SCHEMA: https://docs.snowflake.com/en/sql-reference/sql/create-schema 31 | -- CREATE TABLE: https://docs.snowflake.com/en/sql-reference/sql/create-table 32 | ------------------------------------------------------------------------------------------- 33 | 34 | ---> create the Tasty Bytes Database 35 | CREATE OR REPLACE DATABASE tasty_bytes_sample_data; 36 | 37 | ---> create the Raw POS (Point-of-Sale) Schema 38 | CREATE OR REPLACE SCHEMA tasty_bytes_sample_data.raw_pos; 39 | 40 | ---> create the Raw Menu Table 41 | CREATE OR REPLACE TABLE tasty_bytes_sample_data.raw_pos.menu 42 | ( 43 | menu_id NUMBER(19,0), 44 | menu_type_id NUMBER(38,0), 45 | menu_type VARCHAR(16777216), 46 | truck_brand_name VARCHAR(16777216), 47 | menu_item_id NUMBER(38,0), 48 | menu_item_name VARCHAR(16777216), 49 | item_category VARCHAR(16777216), 50 | item_subcategory VARCHAR(16777216), 51 | cost_of_goods_usd NUMBER(38,4), 52 | sale_price_usd NUMBER(38,4), 53 | menu_item_health_metrics_obj VARIANT 54 | ); 55 | 56 | ---> confirm the empty Menu table exists 57 | SELECT * FROM tasty_bytes_sample_data.raw_pos.menu; 58 | 59 | 60 | ------------------------------------------------------------------------------------------- 61 | -- Step 3: To connect to the Blob Storage, let's create a Stage 62 | -- Creating an S3 Stage: https://docs.snowflake.com/en/user-guide/data-load-s3-create-stage 63 | ------------------------------------------------------------------------------------------- 64 | 65 | ---> create the Stage referencing the Blob location and CSV File Format 66 | CREATE OR REPLACE STAGE tasty_bytes_sample_data.public.blob_stage 67 | url = 's3://sfquickstarts/tastybytes/' 68 | file_format = (type = csv); 69 | 70 | ---> query the Stage to find the Menu CSV file 71 | LIST @tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/; 72 | 73 | 74 | ------------------------------------------------------------------------------------------- 75 | -- Step 4: Now let's Load the Menu CSV file from the Stage 76 | -- COPY INTO : https://docs.snowflake.com/en/sql-reference/sql/copy-into-table 77 | ------------------------------------------------------------------------------------------- 78 | 79 | ---> copy the Menu file into the Menu table 80 | COPY INTO tasty_bytes_sample_data.raw_pos.menu 81 | FROM @tasty_bytes_sample_data.public.blob_stage/raw_pos/menu/; 82 | 83 | 84 | ------------------------------------------------------------------------------------------- 85 | -- Step 5: Query the Menu table 86 | -- SELECT: https://docs.snowflake.com/en/sql-reference/sql/select 87 | -- TOP : https://docs.snowflake.com/en/sql-reference/constructs/top_n 88 | -- FLATTEN: https://docs.snowflake.com/en/sql-reference/functions/flatten 89 | ------------------------------------------------------------------------------------------- 90 | 91 | ---> how many rows are in the table? 92 | SELECT COUNT(*) AS row_count FROM tasty_bytes_sample_data.raw_pos.menu; 93 | 94 | ---> what do the top 10 rows look like? 95 | SELECT TOP 10 * FROM tasty_bytes_sample_data.raw_pos.menu; 96 | 97 | ---> what menu items does the Freezing Point brand sell? 98 | SELECT 99 | menu_item_name 100 | FROM tasty_bytes_sample_data.raw_pos.menu 101 | WHERE truck_brand_name = 'Freezing Point'; 102 | 103 | ---> what is the profit on Mango Sticky Rice? 104 | SELECT 105 | menu_item_name, 106 | (sale_price_usd - cost_of_goods_usd) AS profit_usd 107 | FROM tasty_bytes_sample_data.raw_pos.menu 108 | WHERE 1=1 109 | AND truck_brand_name = 'Freezing Point' 110 | AND menu_item_name = 'Mango Sticky Rice'; 111 | 112 | ---> to finish, let's extract the Mango Sticky Rice ingredients from the semi-structured column 113 | SELECT 114 | m.menu_item_name, 115 | obj.value:"ingredients" AS ingredients 116 | FROM tasty_bytes_sample_data.raw_pos.menu m, 117 | LATERAL FLATTEN (input => m.menu_item_health_metrics_obj:menu_item_health_metrics) obj 118 | WHERE 1=1 119 | AND truck_brand_name = 'Freezing Point' 120 | AND menu_item_name = 'Mango Sticky Rice'; -------------------------------------------------------------------------------- /tests/fixtures/contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "4ux48jcFhEw": { 3 | "snowflakeRequestId": null, 4 | "snowflakeQueryId": "", 5 | "query": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy).\n • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n", 6 | "queryContext": { 7 | "role": "ACCOUNTADMIN", 8 | "warehouse": "COMPUTE_WH", 9 | "database": "", 10 | "schema": "", 11 | "secondaryRoles": "" 12 | }, 13 | "drafts": { 14 | "147276799089": { 15 | "query": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (⌘-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (⌘-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n • Granting a role to another role creates a “parent-child” relationship between the roles (also referred to as a role hierarchy).\n • Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n • https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n", 16 | "paramRefs": [], 17 | "queryRange": null, 18 | "executionContext": { 19 | "role": "ACCOUNTADMIN", 20 | "warehouse": "COMPUTE_WH", 21 | "database": "", 22 | "schema": "", 23 | "secondaryRoles": "" 24 | }, 25 | "queryLanguage": "sql", 26 | "appSessionId": 0 27 | } 28 | }, 29 | "draftUpdates": { 30 | "147276799089": 1705628729507 31 | }, 32 | "transforms": [], 33 | "queryLanguage": "sql", 34 | "pid": "4ux48jcFhEw", 35 | "name": "[Template] Adding a user and granting roles", 36 | "orgId": "mocked", 37 | "ownerId": "mocked", 38 | "folderId": "Ga8xhnsR", 39 | "visibility": "private", 40 | "modified": "2024-01-19T01:45:29.514886Z", 41 | "version": 2, 42 | "isParamQuery": false, 43 | "projectType": "query", 44 | "executionContext": { 45 | "role": "ACCOUNTADMIN", 46 | "warehouse": "COMPUTE_WH", 47 | "database": "", 48 | "schema": "", 49 | "secondaryRoles": "" 50 | }, 51 | "editable": true, 52 | "runnable": true, 53 | "resultsViewable": true, 54 | "url": "mocked", 55 | "slug": "mocked", 56 | "members": [ 57 | { 58 | "memberType": "user", 59 | "userId": "mocked", 60 | "memberId": "mocked", 61 | "role": "owner", 62 | "hasRole": true 63 | } 64 | ], 65 | "hasRequiredRole": true 66 | }, 67 | "1EDjP07epDt": { 68 | "draftUpdates": { 69 | "147276799089": 1706161972435 70 | }, 71 | "drafts": { 72 | "147276799089": { 73 | "appSessionId": 0, 74 | "executionContext": { 75 | "database": "", 76 | "role": "ACCOUNTADMIN", 77 | "schema": "", 78 | "secondaryRoles": "", 79 | "warehouse": "" 80 | }, 81 | "paramRefs": [], 82 | "query": "SELECT COUNT(*) from pytests;", 83 | "queryLanguage": "sql", 84 | "queryRange": null 85 | } 86 | }, 87 | "editable": true, 88 | "executionContext": { 89 | "database": "", 90 | "role": "ACCOUNTADMIN", 91 | "schema": "", 92 | "secondaryRoles": "", 93 | "warehouse": "" 94 | }, 95 | "folderId": "YPegqcHt", 96 | "hasRequiredRole": true, 97 | "isParamQuery": false, 98 | "members": [ 99 | { 100 | "hasRole": true, 101 | "memberId": "fake", 102 | "memberType": "user", 103 | "role": "owner", 104 | "userId": "fake" 105 | } 106 | ], 107 | "modified": "2024-01-25T05:52:52.471755Z", 108 | "name": "[Tutorial] Sample queries on TPC-DS data", 109 | "orgId": "fake", 110 | "ownerId": "fake", 111 | "pid": "1EDjP07epDt", 112 | "projectType": "query", 113 | "queryContext": { 114 | "database": "", 115 | "role": "ACCOUNTADMIN", 116 | "schema": "", 117 | "secondaryRoles": "", 118 | "warehouse": "" 119 | }, 120 | "queryLanguage": "sql", 121 | "resultsViewable": true, 122 | "runnable": true, 123 | "slug": "fake", 124 | "snowflakeQueryId": "", 125 | "snowflakeRequestId": "", 126 | "startDate": "2024-01-25T04:25:49.861694096Z", 127 | "transforms": [], 128 | "url": "/fake#query", 129 | "version": 1, 130 | "visibility": "private" 131 | } 132 | } -------------------------------------------------------------------------------- /tests/fixtures/entities.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "entityId": "4ux48jcFhEw", 3 | "entityType": "query", 4 | "info": { 5 | "name": "[Template] Adding a user and granting roles", 6 | "slug": "mocked", 7 | "version": 0, 8 | "content": "", 9 | "dashboardRows": [], 10 | "folderId": "Ga8xhnsR", 11 | "folderName": "Getting Started Tutorials", 12 | "folderType": "list", 13 | "folderUrl": "/mocked", 14 | "folderSlug": "getting-started-tutorials-fGa8xhnsR", 15 | "visibility": "private", 16 | "ownerId": 0, 17 | "modified": "2024-01-19T01:45:29.514886Z", 18 | "created": "2024-01-14T08:27:39.503542Z", 19 | "viewed": "2024-01-17T01:03:59.460044Z", 20 | "queryLanguage": "sql", 21 | "role": "ACCOUNTADMIN", 22 | "url": "/mocked" 23 | }, 24 | "match": null 25 | }, 26 | { 27 | "entityId": "Ga8xhnsR", 28 | "entityType": "folder", 29 | "info": { 30 | "name": "Getting Started Tutorials", 31 | "slug": "getting-started-tutorials-fGa8xhnsR", 32 | "version": 0, 33 | "content": "", 34 | "dashboardRows": [], 35 | "folderId": null, 36 | "folderName": "", 37 | "folderType": "list", 38 | "visibility": "private", 39 | "ownerId": 0, 40 | "modified": "2024-01-23T00:56:39.139336Z", 41 | "created": "2024-01-23T00:56:39.139336Z", 42 | "viewed": null, 43 | "queryLanguage": "", 44 | "role": "ACCOUNTADMIN", 45 | "url": "/mocked" 46 | }, 47 | "match": null 48 | }, 49 | { 50 | "entityId": "1EDjP07epDt", 51 | "entityType": "query", 52 | "info": { 53 | "content": "", 54 | "created": "2024-01-25T04:25:49.858888Z", 55 | "dashboardRows": [], 56 | "folderId": "YPegqcHt", 57 | "folderName": "Benchmarking Tutorials", 58 | "folderSlug": "benchmarking-tutorials-fYPegqcHt", 59 | "folderType": "list", 60 | "folderUrl": "/fake/#/benchmarking-tutorials-fYPegqcHt", 61 | "modified": "2024-01-25T05:52:52.471755Z", 62 | "name": "[Tutorial] Sample queries on TPC-DS data", 63 | "ownerId": 0, 64 | "queryLanguage": "sql", 65 | "role": "ACCOUNTADMIN", 66 | "slug": "fake", 67 | "url": "/fake#query", 68 | "version": 0, 69 | "viewed": "2024-01-25T05:03:28.247018Z", 70 | "visibility": "private" 71 | }, 72 | "match": null 73 | }, 74 | { 75 | "entityId": "YPegqcHt", 76 | "entityType": "folder", 77 | "info": { 78 | "content": "", 79 | "created": "2024-01-25T04:25:49.140711Z", 80 | "dashboardRows": [], 81 | "folderId": null, 82 | "folderName": "", 83 | "folderType": "list", 84 | "modified": "2024-01-25T04:25:49.140711Z", 85 | "name": "Benchmarking Tutorials", 86 | "ownerId": 0, 87 | "queryLanguage": "", 88 | "role": "ACCOUNTADMIN", 89 | "slug": "benchmarking-tutorials-fYPegqcHt", 90 | "url": "/fake/#/benchmarking-tutorials-fYPegqcHt", 91 | "version": 0, 92 | "viewed": null, 93 | "visibility": "private" 94 | }, 95 | "match": null 96 | } 97 | ] -------------------------------------------------------------------------------- /tests/fixtures/folders.json: -------------------------------------------------------------------------------- 1 | [{"_id": "Ga8xhnsR", "name": "Getting Started Tutorials", "worksheets": []}] -------------------------------------------------------------------------------- /tests/fixtures/worksheets.json: -------------------------------------------------------------------------------- 1 | [{"_id": "4ux48jcFhEw", "name": "[Template] Adding a user and granting roles", "folder_id": "Ga8xhnsR", "folder_name": "Getting Started Tutorials", "content": "/*--\nIn this Worksheet we will walk through creating a User in Snowflake.\n\nFor the User we will provide grants to a defined default role and default warehouse\nand then walk through viewing all other users and roles in our account.\n\nTo conclude, we will drop the created User.\n--*/\n\n\n-------------------------------------------------------------------------------------------\n -- Step 1: To start, we first must set our Role context\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- System-Defined Roles: https://docs.snowflake.com/en/user-guide/security-access-control-overview#system-defined-roles\n-------------------------------------------------------------------------------------------\n\n--> To run a single query, place your cursor in the query editor and select the Run button (\u2318-Return).\n--> To run the entire worksheet, select 'Run All' from the dropdown next to the Run button (\u2318-Shift-Return).\n\n---> set our Role context\n USE ROLE USERADMIN;\n\n-------------------------------------------------------------------------------------------\n -- Step 2: Create our User\n -- CREATE USER: https://docs.snowflake.com/en/sql-reference/sql/create-user\n-------------------------------------------------------------------------------------------\n\n---> now let's create a User using various available parameters.\n -- NOTE: please fill out each section below before executing the query\n\nCREATE OR REPLACE USER -- adjust user name\nPASSWORD = '' -- add a secure password\nLOGIN_NAME = '' -- add a login name\nFIRST_NAME = '' -- add user's first name\nLAST_NAME = '' -- add user's last name\nEMAIL = '' -- add user's email \nMUST_CHANGE_PASSWORD = true -- ensures a password reset on first login\nDEFAULT_WAREHOUSE = COMPUTE_WH; -- set default warehouse to COMPUTE_WH\n\n \n/*--\nWith the User created, send the following information in a secure manner\nto whomever the User is created for, so that they can access this Snowflake account:\n --> Snowflake Account URL: This is the Snowflake account link that they'll need to login. You can find this link at the top of your browser:(ex: https://app.snowflake.com/xxxxxxx/xxxxxxxx/)\n --> LOGIN_NAME: from above\n --> PASSWORD: from above\n--*/\n\n-------------------------------------------------------------------------------------------\n -- Step 3: Grant access to a Role and Warehouse for our User\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- GRANT ROLE: https://docs.snowflake.com/en/sql-reference/sql/grant-role\n -- GRANT : https://docs.snowflake.com/en/sql-reference/sql/grant-privilege\n-------------------------------------------------------------------------------------------\n\n---> with the User created, let's use our SECURITYADMIN role to grant the SYSADMIN role and COMPUTE_WH warehouse to it\nUSE ROLE SECURITYADMIN;\n\n /*--\n \u2022 Granting a role to another role creates a \u201cparent-child\u201d relationship between the roles (also referred to as a role hierarchy).\n \u2022 Granting a role to a user enables the user to perform all operations allowed by the role (through the access privileges granted to the role).\n\n NOTE: The SYSADMIN role has privileges to create warehouses, databases, and database objects in an account and grant those privileges to other roles.\n Only grant this role to Users who should have these privileges. You can view other system-defined roles in the documentation below:\n \u2022 https://docs.snowflake.com/en/user-guide/security-access-control-overview#label-access-control-overview-roles-system\n --*/\n\n-- grant role SYSADMIN to our User\nGRANT ROLE SYSADMIN TO USER ;\n\n\n-- grant usage on the COMPUTE_WH warehouse to our SYSADMIN role\nGRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE SYSADMIN;\n\n\n-------------------------------------------------------------------------------------------\n -- Step 4: Explore all Users and Roles in our Account\n -- USE ROLE: https://docs.snowflake.com/en/sql-reference/sql/use-role\n -- SHOW USERS: https://docs.snowflake.com/en/sql-reference/sql/show-users\n -- SHOW ROLES: https://docs.snowflake.com/en/sql-reference/sql/show-roles\n-------------------------------------------------------------------------------------------\n\n---> let's now explore all users and roles in our account using our ACCOUNTADMIN role\nUSE ROLE ACCOUNTADMIN;\n\n-- show all users in account\nSHOW USERS;\n\n-- show all roles in account\nSHOW ROLES;\n\n-------------------------------------------------------------------------------------------\n -- Step 5: Drop our created Users\n -- DROP USER: https://docs.snowflake.com/en/sql-reference/sql/drop-user\n-------------------------------------------------------------------------------------------\n\n---> to drop the user, we could execute the following command\nDROP USER ;\n", "content_type": "sql"}] -------------------------------------------------------------------------------- /tests/fixtures/worksheets_update.json: -------------------------------------------------------------------------------- 1 | [{"_id": "4ux48jcFhEw", "name": "[Template] Adding a user and granting roles", "folder_id": "Ga8xhnsR", "folder_name": "Getting Started Tutorials", "content": "// Updated", "content_type": "sql"}] -------------------------------------------------------------------------------- /tests/test_cache.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import sf_git.cache as cache 4 | from sf_git.models import SnowflakeGitError, Worksheet 5 | import sf_git.config as config 6 | 7 | 8 | def test_load_ws_after_init(repo): 9 | 10 | worksheets = cache.load_worksheets_from_cache( 11 | repo, 12 | branch_name='main', 13 | ) 14 | 15 | assert isinstance(worksheets, list) 16 | assert len(worksheets) == 7 17 | 18 | 19 | def test_load_ws_wrong_branch(repo): 20 | 21 | with pytest.raises(SnowflakeGitError): 22 | cache.load_worksheets_from_cache( 23 | repo, 24 | branch_name='non_existing' 25 | ) 26 | 27 | 28 | def test_load_ws_from_existing_folder(repo): 29 | 30 | # Todo: only_folder param is the SF one, not intuitive 31 | worksheets = cache.load_worksheets_from_cache( 32 | repo, 33 | only_folder='Benchmarking Tutorials' 34 | ) 35 | 36 | assert isinstance(worksheets, list) 37 | assert len(worksheets) == 4 38 | 39 | 40 | def test_load_ws_from_non_existing_folder(repo): 41 | worksheets = cache.load_worksheets_from_cache( 42 | repo, 43 | only_folder='Non existing' 44 | ) 45 | 46 | assert isinstance(worksheets, list) 47 | assert len(worksheets) == 0 48 | 49 | 50 | @pytest.mark.parametrize( 51 | "worksheet", 52 | [Worksheet( 53 | _id="worksheet_id_01", 54 | name="test_worksheet_01", 55 | folder_id="folder_id_01", 56 | folder_name="Benchmarking_Tutorials", 57 | content_type="sql", 58 | content="SELECT count(*) from PASSING_TESTS" 59 | )] 60 | ) 61 | def test_save_sql_ws_to_cache_existing_folder(worksheet): 62 | 63 | cache.save_worksheets_to_cache([worksheet]) 64 | 65 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "Benchmarking_Tutorials" / f"{worksheet.name}.sql" 66 | 67 | assert expected_file_path.is_file() 68 | with open(expected_file_path, "r") as f: 69 | assert f.read() == worksheet.content 70 | 71 | 72 | @pytest.mark.parametrize( 73 | "worksheet", 74 | [Worksheet( 75 | _id="worksheet_id_02", 76 | name="test_worksheet_02", 77 | folder_id="folder_id_01", 78 | folder_name="Benchmarking_Tutorials", 79 | content_type="python", 80 | content="import sf_git; print(sf_git.__version__)" 81 | )] 82 | ) 83 | def test_save_py_ws_to_cache_existing_folder( 84 | worksheet 85 | ): 86 | 87 | cache.save_worksheets_to_cache([worksheet]) 88 | 89 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "Benchmarking_Tutorials" / f"{worksheet.name}.py" 90 | 91 | assert expected_file_path.is_file() 92 | with open(expected_file_path, "r") as f: 93 | assert f.read() == worksheet.content 94 | 95 | 96 | @pytest.mark.parametrize( 97 | "worksheet", 98 | [Worksheet( 99 | _id="worksheet_id_03", 100 | name="test_worksheet_03", 101 | folder_id="folder_id_02", 102 | folder_name="new_folder_for_test", 103 | content_type="sql", 104 | content="SELECT count(*) from PASSING_TESTS" 105 | )] 106 | ) 107 | def test_save_sql_ws_to_cache_new_folder(worksheet): 108 | 109 | cache.save_worksheets_to_cache([worksheet]) 110 | 111 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "new_folder_for_test" / f"{worksheet.name}.sql" 112 | 113 | assert expected_file_path.is_file() 114 | with open(expected_file_path, "r") as f: 115 | assert f.read() == worksheet.content 116 | 117 | 118 | @pytest.mark.parametrize( 119 | "worksheet", 120 | [Worksheet( 121 | _id="worksheet_id_04", 122 | name="test_worksheet_04", 123 | folder_id="folder_id_02", 124 | folder_name="new_folder_for_test", 125 | content_type="scala", 126 | content='println("Coming soon");' 127 | )] 128 | ) 129 | def test_save_unsupported_extension_ws_to_cache_new_folder(worksheet): 130 | 131 | cache.save_worksheets_to_cache([worksheet]) 132 | 133 | # Todo: defaults to sql as of now 134 | expected_file_path = config.GLOBAL_CONFIG.worksheets_path / "new_folder_for_test" / f"{worksheet.name}.scala" 135 | 136 | assert not expected_file_path.is_file() 137 | 138 | 139 | @pytest.mark.run(after='test_save_py_ws_to_cache_existing_folder') 140 | def test_load_only_tracked_files( 141 | repo, 142 | ): 143 | 144 | worksheets = cache.load_worksheets_from_cache( 145 | repo, 146 | only_folder="Benchmarking Tutorials" 147 | ) 148 | 149 | assert isinstance(worksheets, list) 150 | assert 'test_worksheet_02' not in [ws.name for ws in worksheets] 151 | 152 | 153 | @pytest.mark.run(after='test_save_sql_ws_to_cache_new_folder') 154 | def test_load_only_tracked_files( 155 | repo, 156 | ): 157 | 158 | worksheets = cache.load_worksheets_from_cache( 159 | repo, 160 | only_folder="new_folder_for_test" 161 | ) 162 | 163 | assert isinstance(worksheets, list) 164 | assert len(worksheets) == 0 165 | -------------------------------------------------------------------------------- /tests/test_commands.py: -------------------------------------------------------------------------------- 1 | import json 2 | import shutil 3 | 4 | import dotenv 5 | import pytest 6 | from click import UsageError 7 | from git.repo import Repo 8 | 9 | import sf_git.commands 10 | from sf_git.models import Worksheet 11 | import sf_git.config as config 12 | 13 | 14 | @pytest.fixture 15 | def worksheets(testing_folder): 16 | with open(testing_folder / "fixtures" / "worksheets.json", "r") as f: 17 | worksheets_as_dict = json.load(f) 18 | return [Worksheet(**w) for w in worksheets_as_dict] 19 | 20 | 21 | @pytest.fixture 22 | def mock_authenticate_to_snowsight(auth_context, monkeypatch): 23 | def get_fake_auth_context(account_id, username, password, auth_mode=None): 24 | return auth_context 25 | 26 | monkeypatch.setattr( 27 | sf_git.commands, "authenticate_to_snowsight", get_fake_auth_context 28 | ) 29 | 30 | 31 | @pytest.fixture 32 | def mock_sf_get_worksheets(worksheets, monkeypatch): 33 | def get_fake_worksheets(auth_context, store_to_cache, only_folder): 34 | return worksheets 35 | 36 | monkeypatch.setattr( 37 | sf_git.commands, "sf_get_worksheets", get_fake_worksheets 38 | ) 39 | 40 | 41 | @pytest.fixture 42 | def mock_load_worksheets_from_cache(worksheets, monkeypatch): 43 | def get_fixture_worksheets(repo, branch_name, only_folder): 44 | return worksheets 45 | 46 | monkeypatch.setattr( 47 | sf_git.commands, "load_worksheets_from_cache", get_fixture_worksheets 48 | ) 49 | 50 | 51 | @pytest.fixture 52 | def no_print(monkeypatch): 53 | def do_nothing(worksheets, logger): 54 | pass 55 | 56 | monkeypatch.setattr(sf_git.commands, "print_worksheets", do_nothing) 57 | 58 | 59 | @pytest.fixture 60 | def no_upload(monkeypatch): 61 | def successful_upload(auth_context, worksheets): 62 | return { 63 | "completed": [worksheet.name for worksheet in worksheets], 64 | "errors": [], 65 | } 66 | 67 | monkeypatch.setattr( 68 | sf_git.commands, "upload_to_snowsight", successful_upload 69 | ) 70 | 71 | 72 | @pytest.fixture 73 | def alternative_repo_path(testing_folder): 74 | return testing_folder / "tmp" / "sf_git_test_alternative" 75 | 76 | 77 | @pytest.fixture(name="alternative_repo") 78 | def setup_and_teardown(alternative_repo_path): 79 | """ 80 | Setup alternative git repo and teardown 81 | """ 82 | 83 | # ---- SETUP ----- 84 | Repo.init(alternative_repo_path, mkdir=True) 85 | 86 | # ---- RUN TESTS ----- 87 | yield 88 | 89 | # ---- TEARDOWN ----- 90 | shutil.rmtree(alternative_repo_path) 91 | 92 | 93 | def test_fetch_worksheets_when_pwd_auth_with_pwd( 94 | mock_authenticate_to_snowsight, 95 | mock_sf_get_worksheets, 96 | no_print, 97 | ): 98 | worksheets = sf_git.commands.fetch_worksheets_procedure( 99 | username=config.GLOBAL_CONFIG.sf_login_name, 100 | account_id=config.GLOBAL_CONFIG.sf_account_id, 101 | auth_mode="PWD", 102 | password=config.GLOBAL_CONFIG.sf_pwd, 103 | store=False, 104 | only_folder="", 105 | logger=lambda x: None, 106 | ) 107 | 108 | assert isinstance(worksheets, list) 109 | assert len(worksheets) == 1 110 | 111 | 112 | def test_fetch_worksheets_when_pwd_auth_without_pwd( 113 | mock_authenticate_to_snowsight, 114 | mock_sf_get_worksheets, 115 | no_print, 116 | ): 117 | with pytest.raises(UsageError): 118 | sf_git.commands.fetch_worksheets_procedure( 119 | username=config.GLOBAL_CONFIG.sf_login_name, 120 | account_id=config.GLOBAL_CONFIG.sf_account_id, 121 | auth_mode="PWD", 122 | password=None, 123 | store=False, 124 | only_folder="", 125 | logger=lambda x: None, 126 | ) 127 | 128 | 129 | def test_push_worksheets_when_pwd_auth_without_pwd( 130 | mock_authenticate_to_snowsight, 131 | mock_load_worksheets_from_cache, 132 | no_upload, 133 | no_print, 134 | ): 135 | with pytest.raises(UsageError): 136 | sf_git.commands.push_worksheets_procedure( 137 | username=config.GLOBAL_CONFIG.sf_login_name, 138 | account_id=config.GLOBAL_CONFIG.sf_account_id, 139 | auth_mode="PWD", 140 | password=None, 141 | only_folder="", 142 | logger=lambda x: None, 143 | ) 144 | 145 | 146 | def test_push_worksheets_when_pwd_auth_with_pwd( 147 | repo, 148 | mock_authenticate_to_snowsight, 149 | mock_load_worksheets_from_cache, 150 | no_upload, 151 | no_print, 152 | ): 153 | report = sf_git.commands.push_worksheets_procedure( 154 | username=config.GLOBAL_CONFIG.sf_login_name, 155 | account_id=config.GLOBAL_CONFIG.sf_account_id, 156 | auth_mode="PWD", 157 | password=config.GLOBAL_CONFIG.sf_pwd, 158 | only_folder="", 159 | logger=lambda x: None, 160 | ) 161 | 162 | assert isinstance(report, dict) 163 | assert len(report["completed"]) == 1 164 | assert len(report["errors"]) == 0 165 | 166 | 167 | @pytest.mark.parametrize( 168 | "key", 169 | [ 170 | "SNOWFLAKE_VERSIONING_REPO", 171 | "WORKSHEETS_PATH", 172 | "SF_ACCOUNT_ID", 173 | "SF_LOGIN_NAME", 174 | "SF_PWD", 175 | ], 176 | ) 177 | def test_get_config_repo_procedure_when_key_exists( 178 | key, 179 | fixture_dotenv_path, 180 | monkeypatch, 181 | ): 182 | dotenv_config = dotenv.dotenv_values(fixture_dotenv_path) 183 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 184 | 185 | value = sf_git.commands.get_config_repo_procedure( 186 | key, logger=lambda x: None 187 | ) 188 | 189 | assert value == dotenv_config[key] 190 | 191 | 192 | def test_get_config_repo_procedure_when_key_doesnt_exists( 193 | fixture_dotenv_path, 194 | monkeypatch, 195 | ): 196 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 197 | 198 | with pytest.raises(UsageError): 199 | sf_git.commands.get_config_repo_procedure( 200 | "NOT_EXISTING", logger=lambda x: None 201 | ) 202 | 203 | 204 | @pytest.mark.parametrize( 205 | "params, expected", 206 | [ 207 | ( 208 | {"username": "new_login", "password": "new_password"}, 209 | {"SF_LOGIN_NAME": "new_login", "SF_PWD": "************"}, 210 | ), 211 | ], 212 | ) 213 | def test_set_config_repo_procedure_when_key_exists( 214 | params, 215 | expected, 216 | fixture_dotenv_path, 217 | repo, 218 | monkeypatch, 219 | ): 220 | dotenv_config = dotenv.dotenv_values(fixture_dotenv_path) 221 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 222 | 223 | updates = sf_git.commands.set_config_repo_procedure(**params) 224 | 225 | assert updates == expected 226 | 227 | 228 | def test_set_config_procedure_when_valid_repo( 229 | alternative_repo_path, 230 | fixture_dotenv_path, 231 | repo, 232 | alternative_repo, 233 | monkeypatch, 234 | ): 235 | params = {"git_repo": str(alternative_repo_path)} 236 | expected = {"SNOWFLAKE_VERSIONING_REPO": str(alternative_repo_path)} 237 | 238 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 239 | 240 | updates = sf_git.commands.set_config_repo_procedure(**params) 241 | 242 | assert updates == expected 243 | 244 | 245 | def test_set_config_procedure_when_invalid_repo( 246 | fixture_dotenv_path, 247 | repo, 248 | testing_folder, 249 | monkeypatch, 250 | ): 251 | params = {"git_repo": str(testing_folder / "not_initialized")} 252 | 253 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 254 | 255 | with pytest.raises(UsageError): 256 | sf_git.commands.set_config_repo_procedure(**params) 257 | 258 | 259 | def test_set_config_repo_ws_path_when_invalid( 260 | alternative_repo_path, 261 | alternative_repo, 262 | fixture_dotenv_path, 263 | repo, 264 | testing_folder, 265 | monkeypatch, 266 | ): 267 | params = { 268 | "git_repo": str(alternative_repo_path), 269 | "save_dir": str(testing_folder / "not_a_subdirectory"), 270 | } 271 | 272 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 273 | 274 | with pytest.raises(UsageError): 275 | sf_git.commands.set_config_repo_procedure(**params) 276 | 277 | 278 | def test_set_config_repo_ws_path_when_valid( 279 | alternative_repo_path, 280 | alternative_repo, 281 | fixture_dotenv_path, 282 | repo, 283 | testing_folder, 284 | monkeypatch, 285 | ): 286 | params = { 287 | "git_repo": str(alternative_repo_path), 288 | "save_dir": str(alternative_repo_path / "a_subdirectory"), 289 | } 290 | expected = { 291 | "SNOWFLAKE_VERSIONING_REPO": str(alternative_repo_path), 292 | "WORKSHEETS_PATH": str(alternative_repo_path / "a_subdirectory"), 293 | } 294 | 295 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 296 | 297 | updates = sf_git.commands.set_config_repo_procedure(**params) 298 | 299 | assert isinstance(updates, dict) 300 | assert updates == expected 301 | 302 | 303 | def test_set_config_ws_path_when_valid( 304 | fixture_dotenv_path, 305 | repo, 306 | repo_root_path, 307 | testing_folder, 308 | monkeypatch, 309 | ): 310 | params = {"save_dir": str(repo_root_path / "a_subdirectory")} 311 | expected = {"WORKSHEETS_PATH": str(repo_root_path / "a_subdirectory")} 312 | 313 | monkeypatch.setattr(sf_git.commands, "DOTENV_PATH", fixture_dotenv_path) 314 | 315 | updates = sf_git.commands.set_config_repo_procedure(**params) 316 | 317 | assert isinstance(updates, dict) 318 | assert updates == expected 319 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | from git import Repo 2 | from sf_git import DOTENV_PATH 3 | from sf_git.config import GLOBAL_CONFIG 4 | 5 | 6 | def test_repo(repo: Repo): 7 | 8 | assert not repo.bare 9 | assert len(repo.head.reference.log()) == 1 10 | 11 | 12 | def test_dotenv_path(): 13 | from dotenv import dotenv_values 14 | values = dotenv_values(DOTENV_PATH) 15 | assert values['SF_LOGIN_NAME'] == GLOBAL_CONFIG.sf_login_name 16 | -------------------------------------------------------------------------------- /tests/test_worksheets_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import json 3 | import requests_mock 4 | import re 5 | 6 | import sf_git.models 7 | import sf_git.worksheets_utils as worksheets_utils 8 | 9 | 10 | @pytest.fixture 11 | def mock_api(): 12 | with requests_mock.Mocker() as m: 13 | yield m 14 | 15 | 16 | @pytest.fixture 17 | def get_worksheets_api_response_with_worksheets(monkeypatch, testing_folder): 18 | with open(testing_folder / "fixtures" / "contents.json") as f: 19 | contents = json.loads(f.read()) 20 | 21 | with open(testing_folder / "fixtures" / "entities.json") as f: 22 | entities = json.loads(f.read()) 23 | 24 | mock_response_body = { 25 | "entities": entities, 26 | "models": {"queries": contents}, 27 | } 28 | 29 | return json.dumps(mock_response_body) 30 | 31 | 32 | @pytest.fixture() 33 | def get_worksheets_api_response_without_worksheets( 34 | monkeypatch, testing_folder 35 | ): 36 | mock_response_body = {"entities": [], "models": {"queries": {}}} 37 | 38 | return json.dumps(mock_response_body) 39 | 40 | 41 | @pytest.fixture() 42 | def mock_get_folders(monkeypatch, testing_folder): 43 | def mocked_folders(auth_context): 44 | with open(testing_folder / "fixtures" / "folders.json", "r") as f: 45 | folders_as_dict = json.load(f) 46 | return [sf_git.models.Folder(**f) for f in folders_as_dict] 47 | 48 | monkeypatch.setattr(worksheets_utils, "get_folders", mocked_folders) 49 | 50 | 51 | @pytest.fixture() 52 | def mock_get_worksheets(monkeypatch, testing_folder): 53 | def mocked_worksheets(auth_context): 54 | with open(testing_folder / "fixtures" / "worksheets.json", "r") as f: 55 | worksheets_as_dict = json.load(f) 56 | return [sf_git.models.Worksheet(**w) for w in worksheets_as_dict] 57 | 58 | monkeypatch.setattr(worksheets_utils, "get_worksheets", mocked_worksheets) 59 | 60 | 61 | @pytest.fixture() 62 | def mock_all_write_to_snowsight(monkeypatch): 63 | monkeypatch.setattr( 64 | worksheets_utils, 65 | "create_folder", 66 | lambda auth_context, folder_name: None, 67 | ) 68 | monkeypatch.setattr( 69 | worksheets_utils, 70 | "create_worksheet", 71 | lambda auth_context, worksheet_name, folder_id: None, 72 | ) 73 | monkeypatch.setattr( 74 | worksheets_utils, 75 | "write_worksheet", 76 | lambda auth_context, worksheet: None, 77 | ) 78 | 79 | 80 | def test_get_worksheets_when_two( 81 | mock_api, get_worksheets_api_response_with_worksheets, auth_context 82 | ): 83 | mock_api.post( 84 | re.compile(auth_context.app_server_url), 85 | text=get_worksheets_api_response_with_worksheets, 86 | status_code=200, 87 | ) 88 | 89 | worksheets = worksheets_utils.get_worksheets( 90 | auth_context=auth_context, 91 | store_to_cache=False, 92 | ) 93 | 94 | assert isinstance(worksheets, list) 95 | assert len(worksheets) == 2 96 | 97 | 98 | def test_get_folders_when_two( 99 | mock_api, get_worksheets_api_response_with_worksheets, auth_context 100 | ): 101 | mock_api.post( 102 | re.compile(auth_context.app_server_url), 103 | text=get_worksheets_api_response_with_worksheets, 104 | status_code=200, 105 | ) 106 | 107 | folders = worksheets_utils.get_folders( 108 | auth_context=auth_context, 109 | ) 110 | 111 | assert isinstance(folders, list) 112 | assert len(folders) == 2 113 | 114 | 115 | def test_get_worksheets_when_none( 116 | mock_api, get_worksheets_api_response_without_worksheets, auth_context 117 | ): 118 | mock_api.post( 119 | re.compile(auth_context.app_server_url), 120 | text=get_worksheets_api_response_without_worksheets, 121 | status_code=200, 122 | ) 123 | 124 | worksheets = worksheets_utils.get_worksheets( 125 | auth_context=auth_context, 126 | store_to_cache=False, 127 | ) 128 | 129 | assert isinstance(worksheets, list) 130 | assert len(worksheets) == 0 131 | 132 | 133 | def test_get_folders_when_none( 134 | mock_api, get_worksheets_api_response_without_worksheets, auth_context 135 | ): 136 | mock_api.post( 137 | re.compile(auth_context.app_server_url), 138 | text=get_worksheets_api_response_without_worksheets, 139 | status_code=200, 140 | ) 141 | 142 | folders = worksheets_utils.get_folders( 143 | auth_context=auth_context, 144 | ) 145 | 146 | assert isinstance(folders, list) 147 | assert len(folders) == 0 148 | 149 | 150 | def test_upload_snowsight_when_one_to_update( 151 | monkeypatch, 152 | mock_get_folders, 153 | mock_get_worksheets, 154 | mock_all_write_to_snowsight, 155 | testing_folder, 156 | auth_context, 157 | ): 158 | with open( 159 | testing_folder / "fixtures" / "worksheets_update.json", "r" 160 | ) as f: 161 | worksheets_as_dicts = json.load(f) 162 | 163 | worksheets = [sf_git.models.Worksheet(**w) for w in worksheets_as_dicts] 164 | upload_report = worksheets_utils.upload_to_snowsight( 165 | auth_context, worksheets 166 | ) 167 | 168 | assert isinstance(upload_report, dict) 169 | assert set(upload_report.keys()) == {"completed", "errors"} 170 | assert len(upload_report["completed"]) == len(worksheets) 171 | assert len(upload_report["errors"]) == 0 172 | 173 | 174 | def test_upload_snowsight_when_none_to_update( 175 | monkeypatch, 176 | mock_get_folders, 177 | mock_get_worksheets, 178 | mock_all_write_to_snowsight, 179 | testing_folder, 180 | auth_context, 181 | ): 182 | with open( 183 | testing_folder / "fixtures" / "worksheets.json", "r" 184 | ) as f: 185 | worksheets_as_dicts = json.load(f) 186 | 187 | worksheets = [sf_git.models.Worksheet(**w) for w in worksheets_as_dicts] 188 | upload_report = worksheets_utils.upload_to_snowsight( 189 | auth_context, worksheets 190 | ) 191 | 192 | assert isinstance(upload_report, dict) 193 | assert set(upload_report.keys()) == {"completed", "errors"} 194 | assert len(upload_report["completed"]) == 0 195 | assert len(upload_report["errors"]) == 0 196 | --------------------------------------------------------------------------------